Merge pull request #56 from FlipsideCrypto/add-latest-models

FSC Services cutover, new models, and updated docs
This commit is contained in:
Mike Stepanovic 2025-04-28 16:33:49 -06:00 committed by GitHub
commit 429d52fdbe
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 649 additions and 15 deletions

View File

@ -0,0 +1,21 @@
{{ config(
materialized = 'view'
) }}
SELECT
amount_e8,
asset,
asset_address,
rune_address,
tx_id,
event_id,
block_timestamp,
__HEVO__DATABASE_NAME,
__HEVO__SCHEMA_NAME,
__HEVO__INGESTED_AT,
__HEVO__LOADED_AT
FROM
{{ source(
'thorchain_midgard',
'midgard_secure_asset_deposit_events'
) }}

View File

@ -0,0 +1,21 @@
{{ config(
materialized = 'view'
) }}
SELECT
amount_e8,
asset,
asset_address,
rune_address,
tx_id,
event_id,
block_timestamp,
__HEVO__DATABASE_NAME,
__HEVO__SCHEMA_NAME,
__HEVO__INGESTED_AT,
__HEVO__LOADED_AT
FROM
{{ source(
'thorchain_midgard',
'midgard_secure_asset_withdraw_events'
) }}

View File

@ -0,0 +1,19 @@
{{ config(
materialized = 'view'
) }}
SELECT
address,
key,
value,
event_id,
block_timestamp,
__HEVO__DATABASE_NAME,
__HEVO__SCHEMA_NAME,
__HEVO__INGESTED_AT,
__HEVO__LOADED_AT
FROM
{{ source(
'thorchain_midgard',
'midgard_set_node_mimir_events'
) }}

View File

@ -0,0 +1,21 @@
{{ config(
materialized = 'view'
) }}
SELECT
tx_id,
contract_address,
contract_type,
sender,
attributes,
event_id,
block_timestamp,
__HEVO__DATABASE_NAME,
__HEVO__SCHEMA_NAME,
__HEVO__INGESTED_AT,
__HEVO__LOADED_AT
FROM
{{ source(
'thorchain_midgard',
'midgard_wasm_contracts_events'
) }}

View File

@ -31,6 +31,7 @@ There is more information on how to use dbt docs in the last section of this doc
- [core.fact_transfers](#!/model/model.thorchain_models.core__fact_transfers)
- [core.fact_tss_keygen_failure_events](#!/model/model.thorchain_models.core__fact_tss_keygen_failure_events)
- [core.fact_tss_keygen_success_events](#!/model/model.thorchain_models.core__fact_tss_keygen_success_events)
- [core.fact_wasm_contracts_events](#!/model/model.thorchain_models.core__fact_wasm_contracts_events)
### Defi Schema
@ -67,6 +68,10 @@ There is more information on how to use dbt docs in the last section of this doc
- [defi.fact_rewards_events](#!/model/model.thorchain_models.defi__fact_rewards_events)
- [defi.fact_rune_pool_deposit_events](#!/model/model.thorchain_models.defi__fact_rune_pool_deposit_events)
- [defi.fact_rune_pool_withdraw_events](#!/model/model.thorchain_models.defi__fact_rune_pool_withdraw_events)
- [defi.fact_scheduled_outbound_events](#!/model/model.thorchain_models.defi__fact_scheduled_outbound_events)
- [defi.fact_secure_asset_deposit_events](#!/model/model.thorchain_models.defi__fact_secure_asset_deposit_events)
- [defi.fact_secure_asset_withdraw_events](#!/model/model.thorchain_models.defi__fact_secure_asset_withdraw_events)
- [defi.fact_send_messages](#!/model/model.thorchain_models.defi__fact_send_messages)
- [defi.fact_stake_events](#!/model/model.thorchain_models.defi__fact_stake_events)
- [defi.fact_streamling_swap_details_events](#!/model/model.thorchain_models.defi__fact_streamling_swap_details_events)
- [defi.fact_swaps](#!/model/model.thorchain_models.defi__fact_swaps)
@ -74,6 +79,8 @@ There is more information on how to use dbt docs in the last section of this doc
- [defi.fact_switch_events](#!/model/model.thorchain_models.defi__fact_switch_events)
- [defi.fact_total_block_rewards](#!/model/model.thorchain_models.defi__fact_total_block_rewards)
- [defi.fact_total_value_locked](#!/model/model.thorchain_models.defi__fact_total_value_locked)
- [defi.fact_trade_account_deposit_events](#!/model/model.thorchain_models.defi__fact_trade_account_deposit_events)
- [defi.fact_trade_account_withdraw_events](#!/model/model.thorchain_models.defi__fact_trade_account_withdraw_events)
- [defi.fact_unstake_events](#!/model/model.thorchain_models.defi__fact_unstake_events)
- [defi.fact_update_node_account_status_events](#!/model/model.thorchain_models.defi__fact_update_node_account_status_events)
- [defi.fact_upgrades](#!/model/model.thorchain_models.defi__fact_upgrades)
@ -84,6 +91,7 @@ There is more information on how to use dbt docs in the last section of this doc
- [gov.fact_new_node_events](#!/model/model.thorchain_models.gov__fact_new_node_events)
- [gov.fact_set_ip_address_events](#!/model/model.thorchain_models.gov__fact_set_ip_address_events)
- [gov.fact_set_node_keys_events](#!/model/model.thorchain_models.gov__fact_set_node_keys_events)
- [gov.fact_set_node_mimir_events](#!/model/model.thorchain_models.gov__fact_set_node_mimir_events)
- [gov.fact_set_version_events](#!/model/model.thorchain_models.gov__fact_set_version_events)
- [gov.fact_slash_amounts](#!/model/model.thorchain_models.gov__fact_slash_amounts)
- [gov.fact_slash_points](#!/model/model.thorchain_models.gov__fact_slash_points)

View File

@ -3,4 +3,4 @@
) }}
SELECT
'2.29.0' AS midgard_version
'2.31.0' AS midgard_version

View File

@ -0,0 +1,58 @@
{{ config(
materialized = 'incremental',
meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} },
unique_key = 'fact_wasm_contracts_events_id',
incremental_strategy = 'merge',
incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'],
cluster_by = ['block_timestamp::DATE']
) }}
WITH base AS (
SELECT
tx_id,
contract_address,
contract_type,
sender,
attributes,
event_id,
block_timestamp,
_INSERTED_TIMESTAMP
FROM
{{ ref('silver__wasm_contracts_events') }}
)
SELECT
{{ dbt_utils.generate_surrogate_key(
['a.tx_id','a.event_id','a.contract_address', 'a.sender']
) }} AS fact_wasm_contracts_events_id,
b.block_timestamp,
COALESCE(
b.dim_block_id,
'-1'
) AS dim_block_id,
tx_id,
contract_address,
contract_type,
sender,
attributes,
event_id,
A._INSERTED_TIMESTAMP,
'{{ invocation_id }}' AS _audit_run_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp
FROM
base A
JOIN {{ ref('core__dim_block') }}
b
ON A.block_timestamp = b.timestamp
{% if is_incremental() %}
WHERE
b.block_timestamp >= (
SELECT
MAX(
block_timestamp - INTERVAL '1 HOUR'
)
FROM
{{ this }}
)
{% endif %}

View File

@ -0,0 +1,27 @@
version: 2
models:
- name: core__fact_wasm_contracts_events
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- TX_ID
- EVENT_ID
- CONTRACT_ADDRESS
- SENDER
columns:
- name: TX_ID
description: "{{ doc('tx_id') }}"
- name: CONTRACT_ADDRESS
description: "Address of the wasm contract"
- name: CONTRACT_TYPE
description: "Type of the wasm contract"
- name: SENDER
description: "Address of the sender"
- name: ATTRIBUTES
description: "Attributes of the wasm contract"
- name: EVENT_ID
description: ""
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
- name: _INSERTED_TIMESTAMP
description: "{{ doc('inserted_timestamp') }}"

View File

@ -11,9 +11,6 @@ WITH base AS (
SELECT
owner,
{# collateral_up,
debt_up,
#}
collateralization_ratio,
collateral_asset,
target_asset,
@ -36,8 +33,6 @@ SELECT
'-1'
) AS dim_block_id,
owner,
NULL AS collateral_up,
NULL AS debt_up,
collateralization_ratio,
collateral_asset,
target_asset,

View File

@ -22,10 +22,6 @@ models:
description: "The loan owner"
tests:
- not_null
- name: collateral_up
description: "DEPRECATING SOON! The collateral amount"
- name: debt_up
description: "DEPRECATING SOON! The debt amount"
- name: collateralization_ratio
description: "The collateralization ratio"
tests:

View File

@ -0,0 +1,66 @@
{{ config(
materialized = 'incremental',
meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} },
unique_key = 'event_id',
incremental_strategy = 'merge',
incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'],
cluster_by = ['block_timestamp::DATE']
) }}
WITH base AS (
SELECT
amount_e8,
asset,
asset_address,
rune_address,
tx_id,
event_id,
block_timestamp,
_INSERTED_TIMESTAMP
FROM
{{ ref('silver__secure_asset_deposit_events') }}
)
SELECT
{{ dbt_utils.generate_surrogate_key(
['a.amount_e8','a.asset','a.asset_address','a.rune_address','a.tx_id','a.event_id','a.block_timestamp']
) }} AS fact_secure_asset_deposit_events_id,
b.block_timestamp,
COALESCE(
b.dim_block_id,
'-1'
) AS dim_block_id,
amount_e8,
asset,
asset_address,
rune_address,
tx_id,
event_id,
A._INSERTED_TIMESTAMP,
'{{ invocation_id }}' AS _audit_run_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp
FROM
base A
JOIN {{ ref('core__dim_block') }}
b
ON A.block_timestamp = b.timestamp
{% if is_incremental() %}
WHERE
b.block_timestamp >= (
SELECT
MAX(
block_timestamp - INTERVAL '1 HOUR'
)
FROM
{{ this }}
)
OR event_id IN (
SELECT
event_id
FROM
{{ this }}
WHERE
dim_block_id = '-1'
)
{% endif %}

View File

@ -0,0 +1,36 @@
version: 2
models:
- name: defi__fact_secure_asset_deposit_events
description: "Fact table that shows secure asset deposits"
columns:
- name: FACT_SECURE_ASSET_DEPOSIT_EVENTS_ID
description: "{{ doc('sk') }}"
tests:
- not_null
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- not_null:
where: DIM_BLOCK_ID not in ('-1','-2')
- name: DIM_BLOCK_ID
description: "FK to DIM_BLOCK table"
tests:
- negative_one:
where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS')
- name: AMOUNT_E8
- name: ASSET
- name: ASSET_ADDRESS
- name: RUNE_ADDRESS
- name: TX_ID
- name: EVENT_ID
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'
tests:
- dbt_constraints.primary_key:
column_name: FACT_SECURE_ASSET_DEPOSIT_EVENTS_ID
- dbt_constraints.foreign_key:
fk_column_name: DIM_BLOCK_ID
pk_table_name: ref('core__dim_block')
pk_column_name: DIM_BLOCK_ID

View File

@ -0,0 +1,66 @@
{{ config(
materialized = 'incremental',
meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} },
unique_key = 'event_id',
incremental_strategy = 'merge',
incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'],
cluster_by = ['block_timestamp::DATE']
) }}
WITH base AS (
SELECT
amount_e8,
asset,
asset_address,
rune_address,
tx_id,
event_id,
block_timestamp,
_INSERTED_TIMESTAMP
FROM
{{ ref('silver__secure_asset_withdraw_events') }}
)
SELECT
{{ dbt_utils.generate_surrogate_key(
['a.amount_e8','a.asset','a.asset_address','a.rune_address','a.tx_id','a.event_id','a.block_timestamp']
) }} AS fact_secure_asset_withdraw_events_id,
b.block_timestamp,
COALESCE(
b.dim_block_id,
'-1'
) AS dim_block_id,
amount_e8,
asset,
asset_address,
rune_address,
tx_id,
event_id,
A._INSERTED_TIMESTAMP,
'{{ invocation_id }}' AS _audit_run_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp
FROM
base A
JOIN {{ ref('core__dim_block') }}
b
ON A.block_timestamp = b.timestamp
{% if is_incremental() %}
WHERE
b.block_timestamp >= (
SELECT
MAX(
block_timestamp - INTERVAL '1 HOUR'
)
FROM
{{ this }}
)
OR event_id IN (
SELECT
event_id
FROM
{{ this }}
WHERE
dim_block_id = '-1'
)
{% endif %}

View File

@ -0,0 +1,36 @@
version: 2
models:
- name: defi__fact_secure_asset_withdraw_events
description: "Fact table that shows secure asset withdrawals"
columns:
- name: FACT_SECURE_ASSET_WITHDRAW_EVENTS_ID
description: "{{ doc('sk') }}"
tests:
- not_null
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- not_null:
where: DIM_BLOCK_ID not in ('-1','-2')
- name: DIM_BLOCK_ID
description: "FK to DIM_BLOCK table"
tests:
- negative_one:
where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS')
- name: AMOUNT_E8
- name: ASSET
- name: ASSET_ADDRESS
- name: RUNE_ADDRESS
- name: TX_ID
- name: EVENT_ID
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'
tests:
- dbt_constraints.primary_key:
column_name: FACT_SECURE_ASSET_WITHDRAW_EVENTS_ID
- dbt_constraints.foreign_key:
fk_column_name: DIM_BLOCK_ID
pk_table_name: ref('core__dim_block')
pk_column_name: DIM_BLOCK_ID

View File

@ -53,7 +53,6 @@ SELECT
failed_swaps,
failed_swaps_reasons,
event_id,
failed_swap_reasons,
A._inserted_timestamp,
'{{ invocation_id }}' AS _audit_run_id,
SYSDATE() AS inserted_timestamp,

View File

@ -42,8 +42,6 @@ models:
description: "the out amount of the swap"
- name: failed_swaps
description: "the count of failed swaps"
- name: failed_swaps_reasons
description: "DEPRECATING SOON! the reason of failed swaps"
- name: event_id
description: ""
- name: failed_swap_reasons

View File

@ -0,0 +1,54 @@
{{ config(
materialized = 'incremental',
meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} },
unique_key = 'fact_set_node_mimir_events_id',
incremental_strategy = 'merge',
incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'],
cluster_by = ['block_timestamp::DATE']
) }}
WITH base AS (
SELECT
address,
key,
value,
event_id,
block_timestamp,
_INSERTED_TIMESTAMP
FROM
{{ ref('silver__set_node_mimir_events') }}
)
SELECT
{{ dbt_utils.generate_surrogate_key(
['a.event_id','a.address','a.key']
) }} AS fact_set_node_mimir_events_id,
b.block_timestamp,
COALESCE(
b.dim_block_id,
'-1'
) AS dim_block_id,
address,
key,
value,
event_id,
A._INSERTED_TIMESTAMP,
'{{ invocation_id }}' AS _audit_run_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp
FROM
base A
JOIN {{ ref('core__dim_block') }}
b
ON A.block_timestamp = b.timestamp
{% if is_incremental() %}
WHERE
b.block_timestamp >= (
SELECT
MAX(
block_timestamp - INTERVAL '1 HOUR'
)
FROM
{{ this }}
)
{% endif %}

View File

@ -0,0 +1,37 @@
version: 2
models:
- name: gov__fact_set_node_mimir_events
description: "Fact table that shows set node mimir events"
columns:
- name: FACT_SET_NODE_MIMIR_EVENTS_ID
description: "{{ doc('sk') }}"
tests:
- dbt_expectations.expect_column_to_exist
- unique
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- not_null:
where: DIM_BLOCK_ID not in ('-1','-2')
- name: DIM_BLOCK_ID
description: "FK to DIM_BLOCK table"
tests:
- negative_one:
where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS')
- name: ADDRESS
description: ""
- name: KEY
description: ""
- name: VALUE
description: ""
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'
tests:
- dbt_constraints.primary_key:
column_name: FACT_SET_NODE_MIMIR_EVENTS_ID
- dbt_constraints.foreign_key:
fk_column_name: DIM_BLOCK_ID
pk_table_name: ref('core__dim_block')
pk_column_name: DIM_BLOCK_ID

View File

@ -0,0 +1,22 @@
{{ config(
materialized = 'view'
) }}
SELECT
amount_e8,
asset,
asset_address,
rune_address,
tx_id,
event_id,
block_timestamp,
DATEADD(
ms,
__HEVO__LOADED_AT,
'1970-01-01'
) AS _INSERTED_TIMESTAMP
FROM
{{ ref('bronze__secure_asset_deposit_events') }}
qualify(ROW_NUMBER() over(PARTITION BY event_id
ORDER BY
__HEVO__LOADED_AT DESC)) = 1

View File

@ -0,0 +1,27 @@
version: 2
models:
- name: silver__secure_asset_deposit_events
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- EVENT_ID
columns:
- name: AMOUNT_E8
tests:
- not_null
- name: ASSET
tests:
- not_null
- name: ASSET_ADDRESS
- name: RUNE_ADDRESS
tests:
- not_null
- name: TX_ID
tests:
- not_null
- name: EVENT_ID
tests:
- not_null
- name: BLOCK_TIMESTAMP
tests:
- not_null

View File

@ -0,0 +1,22 @@
{{ config(
materialized = 'view'
) }}
SELECT
amount_e8,
asset,
asset_address,
rune_address,
tx_id,
event_id,
block_timestamp,
DATEADD(
ms,
__HEVO__LOADED_AT,
'1970-01-01'
) AS _INSERTED_TIMESTAMP
FROM
{{ ref('bronze__secure_asset_withdraw_events') }}
qualify(ROW_NUMBER() over(PARTITION BY event_id
ORDER BY
__HEVO__LOADED_AT DESC)) = 1

View File

@ -0,0 +1,27 @@
version: 2
models:
- name: silver__secure_asset_withdraw_events
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- EVENT_ID
columns:
- name: AMOUNT_E8
tests:
- not_null
- name: ASSET
tests:
- not_null
- name: ASSET_ADDRESS
- name: RUNE_ADDRESS
tests:
- not_null
- name: TX_ID
tests:
- not_null
- name: EVENT_ID
tests:
- not_null
- name: BLOCK_TIMESTAMP
tests:
- not_null

View File

@ -0,0 +1,20 @@
{{ config(
materialized = 'view'
) }}
SELECT
address,
key,
value,
event_id,
block_timestamp,
DATEADD(
ms,
__HEVO__LOADED_AT,
'1970-01-01'
) AS _INSERTED_TIMESTAMP
FROM
{{ ref('bronze__set_node_mimir_events') }}
qualify(ROW_NUMBER() over(PARTITION BY event_id
ORDER BY
__HEVO__LOADED_AT DESC)) = 1

View File

@ -0,0 +1,22 @@
{{ config(
materialized = 'view'
) }}
SELECT
tx_id,
contract_address,
contract_type,
sender,
attributes,
event_id,
block_timestamp,
DATEADD(
ms,
__HEVO__LOADED_AT,
'1970-01-01'
) AS _INSERTED_TIMESTAMP
FROM
{{ ref('bronze__wasm_contracts_events') }}
qualify(ROW_NUMBER() over(PARTITION BY event_id
ORDER BY
__HEVO__LOADED_AT DESC)) = 1

View File

@ -0,0 +1,33 @@
version: 2
models:
- name: silver__wasm_contracts_events
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- TX_ID
- EVENT_ID
- CONTRACT_ADDRESS
- SENDER
columns:
- name: TX_ID
tests:
- not_null
- name: CONTRACT_ADDRESS
tests:
- not_null
- name: CONTRACT_TYPE
tests:
- not_null
- name: SENDER
- name: ATTRIBUTES
tests:
- not_null
- name: EVENT_ID
tests:
- not_null
- name: BLOCK_TIMESTAMP
tests:
- not_null
- name: _INSERTED_TIMESTAMP
tests:
- not_null

View File

@ -3,7 +3,7 @@ version: 2
sources:
- name: thorchain_midgard
database: HEVO
schema: THORCHAIN_MIDGARD_2_29_0_DRAB
schema: THORCHAIN_MIDGARD_METAL
tables:
- name: midgard_active_vault_events
- name: midgard_add_events
@ -20,7 +20,6 @@ sources:
- name: midgard_inactive_vault_events
- name: midgard_loan_open_events
- name: midgard_loan_repayment_events
- name: midgard_message_events
- name: midgard_mint_burn_events
- name: midgard_network_version_events
- name: midgard_new_node_events
@ -36,10 +35,13 @@ sources:
- name: midgard_rune_pool_withdraw_events
- name: midgard_rune_price
- name: midgard_scheduled_outbound_events
- name: midgard_secure_asset_deposit_events
- name: midgard_secure_asset_withdraw_events
- name: midgard_send_messages
- name: midgard_set_ip_address_events
- name: midgard_set_mimir_events
- name: midgard_set_node_keys_events
- name: midgard_set_node_mimir_events
- name: midgard_set_version_events
- name: midgard_slash_events
- name: midgard_slash_points_events
@ -56,6 +58,7 @@ sources:
- name: midgard_unstake_events
- name: midgard_update_node_account_status_events
- name: midgard_validator_request_leave_events
- name: midgard_wasm_contracts_events
- name: midgard_withdraw_events
- name: crosschain
database: "{{ 'crosschain' if target.database == 'THORCHAIN' else 'crosschain_dev' }}"