From 530730057089ff64be0a33b3c909d9797eb65bd3 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Wed, 6 Dec 2023 15:19:30 -0500 Subject: [PATCH] updates --- dbt_project.yml | 4 + models/gold/core/core__dim_labels.sql | 48 +++++++- models/gold/core/core__dim_tokens.sql | 16 ++- models/gold/core/core__ez_icns.sql | 16 ++- models/gold/core/core__fact_airdrop.sql | 16 ++- models/gold/core/core__fact_blocks.sql | 16 ++- .../gold/core/core__fact_daily_balances.sql | 37 +++--- .../gold/core/core__fact_msg_attributes.sql | 16 ++- models/gold/core/core__fact_msgs.sql | 16 ++- models/gold/core/core__fact_token_day.sql | 16 ++- models/gold/core/core__fact_transactions.sql | 85 ++++---------- models/gold/core/core__fact_transfers.sql | 16 ++- models/gold/core/core__fact_tx_body_msgs.sql | 16 ++- .../gold/defi/defi__dim_liquidity_pools.sql | 29 +++++ ...ools.yml => defi__dim_liquidity_pools.yml} | 2 +- .../defi__fact_liquidity_provider_actions.sql | 48 +++++++- .../defi__fact_locked_liquidity_actions.sql | 32 ++++- models/gold/defi/defi__fact_pool_fee_day.sql | 16 ++- models/gold/defi/defi__fact_pool_hour.sql | 16 ++- .../defi/defi__fact_superfluid_staking.sql | 32 ++++- models/gold/defi/defi__fact_swaps.sql | 16 ++- models/gold/defi/gov__dim_liquidity_pools.sql | 15 --- models/gold/gov/gov__dim_vote_options.sql | 7 +- ...gov__fact_governance_proposal_deposits.sql | 18 ++- .../gov__fact_governance_submit_proposal.sql | 18 ++- .../gov__fact_governance_validator_votes.sql | 16 ++- .../gold/gov/gov__fact_governance_votes.sql | 18 ++- models/gold/gov/gov__fact_staking.sql | 18 ++- models/gold/gov/gov__fact_staking_rewards.sql | 18 ++- .../gov/gov__fact_validator_commission.sql | 16 ++- models/gold/gov/gov__fact_validators.sql | 16 ++- .../mars__ez_liquidity_provider_actions.sql | 16 ++- models/gold/mars/mars__ez_pool_fee_day.sql | 16 ++- models/gold/mars/mars__ez_pool_hour.sql | 16 ++- models/gold/mars/mars__ez_redbank_actions.sql | 16 ++- .../mars/mars__ez_redbank_balance_changes.sql | 16 ++- .../mars/mars__ez_redbank_reward_claims.sql | 16 ++- .../mars__ez_redbank_reward_distributions.sql | 16 ++- models/gold/mars/mars__ez_swaps.sql | 16 ++- models/gold/mars/mars__ez_token_day.sql | 16 ++- models/gold/price/price__dim_prices.sql | 62 +++------- models/gold/price/price__ez_prices.sql | 16 ++- models/silver/core/silver__airdrops.sql | 11 +- models/silver/core/silver__asset_metadata.sql | 8 +- models/silver/core/silver__blockchain.sql | 9 +- models/silver/core/silver__blocks.sql | 9 +- models/silver/core/silver__daily_balances.sql | 81 ++++++++----- .../core/silver__daily_balances_final.sql | 36 ++++++ .../core/silver__latest_liquid_balances.sql | 8 +- .../silver/core/silver__liquid_balances.sql | 8 +- models/silver/core/silver__msg_attributes.sql | 9 +- models/silver/core/silver__msgs.sql | 9 +- models/silver/core/silver__transactions.sql | 21 +++- .../core/silver__transactions_final.sql | 81 +++++++++++++ models/silver/core/silver__transfers.sql | 9 +- models/silver/core/silver__tx_body_msgs.sql | 9 +- .../silver/crossdb/silver__croschain_icns.sql | 8 +- .../crossdb/silver__croschain_labels.sql | 8 +- models/silver/defi/silver___early_swaps.sql | 9 +- ...lver__early_liquidity_provider_actions.sql | 9 +- .../silver__liquidity_provider_actions.sql | 9 +- ...ver__liquidity_provider_actions_unpool.sql | 9 +- .../defi/silver__locked_liquidity_actions.sql | 9 +- ..._locked_liquidity_actions_begin_unlock.sql | 9 +- ...lver__locked_liquidity_actions_convert.sql | 9 +- .../silver__locked_liquidity_balances.sql | 9 +- .../silver__locked_liquidity_transactions.sql | 9 +- models/silver/defi/silver__pool_balances.sql | 9 +- .../defi/silver__pool_fee_summary_day.sql | 9 +- models/silver/defi/silver__pool_metadata.sql | 9 +- .../silver/defi/silver__pool_summary_hour.sql | 9 +- models/silver/defi/silver__swaps.sql | 8 +- .../silver/defi/silver__token_summary_day.sql | 9 +- models/silver/defi/silver__token_swapped.sql | 9 +- .../silver__governance_proposal_deposits.sql | 9 +- .../silver__governance_submit_proposal.sql | 9 +- .../silver/gov/silver__governance_votes.sql | 9 +- models/silver/gov/silver__staked_balances.sql | 9 +- models/silver/gov/silver__staking.sql | 9 +- models/silver/gov/silver__staking_rewards.sql | 9 +- .../silver/gov/silver__superfluid_actions.sql | 9 +- .../silver__superfluid_staked_balances.sql | 9 +- .../silver/gov/silver__superfluid_staking.sql | 9 +- .../gov/silver__validator_commission.sql | 9 +- models/silver/gov/silver__validator_memos.sql | 8 +- .../silver/gov/silver__validator_metadata.sql | 8 +- .../silver/mars/silver__red_bank_actions.sql | 9 +- .../prices/silver__pool_token_prices.sql | 9 +- .../prices/silver__pool_token_prices_usd.sql | 82 ++++++++----- .../silver/prices/silver__prices_combined.sql | 110 ++++++++++++++++++ models/silver/prices/silver__prices_swaps.sql | 9 +- ...silver__token_prices_coin_gecko_hourly.sql | 9 +- ...r__token_prices_coin_market_cap_hourly.sql | 9 +- 93 files changed, 1398 insertions(+), 303 deletions(-) create mode 100644 models/gold/defi/defi__dim_liquidity_pools.sql rename models/gold/defi/{gov__dim_liquidity_pools.yml => defi__dim_liquidity_pools.yml} (92%) delete mode 100644 models/gold/defi/gov__dim_liquidity_pools.sql create mode 100644 models/silver/core/silver__daily_balances_final.sql create mode 100644 models/silver/core/silver__transactions_final.sql create mode 100644 models/silver/prices/silver__prices_combined.sql diff --git a/dbt_project.yml b/dbt_project.yml index 766cef5..7f5e941 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -49,6 +49,10 @@ vars: UPDATE_UDFS_AND_SPS: False UPDATE_SNOWFLAKE_TAGS: True START_GHA_TASKS: False + API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' + EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}' + ROLES: | + ["INTERNAL_DEV"] tests: +store_failures: true # all tests diff --git a/models/gold/core/core__dim_labels.sql b/models/gold/core/core__dim_labels.sql index a0d8cb7..4280cd6 100644 --- a/models/gold/core/core__dim_labels.sql +++ b/models/gold/core/core__dim_labels.sql @@ -11,7 +11,21 @@ SELECT label_subtype, label, project_name, - raw_metadata + raw_metadata, + COALESCE( + asset_metadata_id, + {{ dbt_utils.generate_surrogate_key( + ['address'] + ) }} + ) AS dim_labels_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__asset_metadata') }} UNION ALL @@ -23,7 +37,21 @@ SELECT label_subtype, label, project_name, - raw_metadata + raw_metadata, + COALESCE( + validator_metadata_id, + {{ dbt_utils.generate_surrogate_key( + [' _unique_key '] + ) }} + ) AS dim_labels_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__validator_metadata') }} UNION ALL @@ -35,7 +63,21 @@ SELECT label_subtype, address_name AS label, project_name, - NULL AS raw_metadata + NULL AS raw_metadata, + COALESCE( + crosschain_labels_id, + {{ dbt_utils.generate_surrogate_key( + [' address '] + ) }} + ) AS dim_labels_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref( 'silver__croschain_labels' diff --git a/models/gold/core/core__dim_tokens.sql b/models/gold/core/core__dim_tokens.sql index 64bb37d..fb4c801 100644 --- a/models/gold/core/core__dim_tokens.sql +++ b/models/gold/core/core__dim_tokens.sql @@ -13,6 +13,20 @@ SELECT project_name, alias, DECIMAL, - raw_metadata + raw_metadata, + COALESCE( + asset_metadata_id, + {{ dbt_utils.generate_surrogate_key( + ['address'] + ) }} + ) AS dim_tokens_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__asset_metadata') }} diff --git a/models/gold/core/core__ez_icns.sql b/models/gold/core/core__ez_icns.sql index 774df17..f61f241 100644 --- a/models/gold/core/core__ez_icns.sql +++ b/models/gold/core/core__ez_icns.sql @@ -10,7 +10,21 @@ SELECT label, project_name, start_date, - end_date + end_date, + COALESCE( + crosschain_icns_id, + {{ dbt_utils.generate_surrogate_key( + ['address'] + ) }} + ) AS ez_icns_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref( 'silver__croschain_icns' diff --git a/models/gold/core/core__fact_airdrop.sql b/models/gold/core/core__fact_airdrop.sql index 69b1eb6..fcafaf9 100644 --- a/models/gold/core/core__fact_airdrop.sql +++ b/models/gold/core/core__fact_airdrop.sql @@ -13,6 +13,20 @@ SELECT amount, currency, DECIMAL, - receiver + receiver, + COALESCE( + airdrops_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_airdrops_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__airdrops') }} diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index fb4a6ee..c7d7af8 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -9,6 +9,20 @@ SELECT chain_id, tx_count, proposer_address, - validator_hash + validator_hash, + COALESCE( + blocks_id, + {{ dbt_utils.generate_surrogate_key( + ['block_id'] + ) }} + ) AS fact_airdrops_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__blocks') }} diff --git a/models/gold/core/core__fact_daily_balances.sql b/models/gold/core/core__fact_daily_balances.sql index 6d1c16a..6a8aa33 100644 --- a/models/gold/core/core__fact_daily_balances.sql +++ b/models/gold/core/core__fact_daily_balances.sql @@ -1,9 +1,5 @@ {{ config( - materialized = 'incremental', - unique_key = "CONCAT_WS('-', date, address, balance_type, currency)", - incremental_strategy = 'delete+insert', - cluster_by = ['date'], - tags = ['balances'], + materialized = 'view', meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'BALANCES' }} } ) }} @@ -13,19 +9,20 @@ SELECT address, currency, DECIMAL, - balance + balance, + COALESCE( + daily_balances_final_id, + {{ dbt_utils.generate_surrogate_key( + ['date', 'address', 'balance_type', 'currency'] + ) }} + ) AS fact_daily_balances_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM - {{ ref('silver__daily_balances') }} -WHERE - balance > 0 - -{% if is_incremental() %} -AND DATE >= ( - SELECT - MAX( - DATE - ) - FROM - {{ this }} -) - INTERVAL '48 HOURS' -{% endif %} + {{ ref('silver__daily_balances_final') }} diff --git a/models/gold/core/core__fact_msg_attributes.sql b/models/gold/core/core__fact_msg_attributes.sql index 5b119d3..1dcfa70 100644 --- a/models/gold/core/core__fact_msg_attributes.sql +++ b/models/gold/core/core__fact_msg_attributes.sql @@ -17,6 +17,20 @@ SELECT msg_type, attribute_index, attribute_key, - attribute_value + attribute_value, + COALESCE( + msg_attributes_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id', 'msg_index', 'attribute_index'] + ) }} + ) AS fact_msg_attributes_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__msg_attributes') }} diff --git a/models/gold/core/core__fact_msgs.sql b/models/gold/core/core__fact_msgs.sql index 5710235..40600f1 100644 --- a/models/gold/core/core__fact_msgs.sql +++ b/models/gold/core/core__fact_msgs.sql @@ -15,6 +15,20 @@ SELECT ) AS msg_group, msg_index, msg_type, - msg + msg, + COALESCE( + msgs_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id', 'msg_index'] + ) }} + ) AS fact_msgs_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__msgs') }} diff --git a/models/gold/core/core__fact_token_day.sql b/models/gold/core/core__fact_token_day.sql index 803df0f..9a101db 100644 --- a/models/gold/core/core__fact_token_day.sql +++ b/models/gold/core/core__fact_token_day.sql @@ -14,6 +14,20 @@ SELECT liquidity, liquidity_usd, volume, - volume_usd + volume_usd, + COALESCE( + token_summary_day_id, + {{ dbt_utils.generate_surrogate_key( + ['currency', 'block_date'] + ) }} + ) AS fact_token_day_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__token_summary_day') }} diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 3503584..3b4d435 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -1,74 +1,33 @@ {{ config( - materialized = 'incremental', - unique_key = "tx_id", - incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE'], + materialized = 'view', tags = ['core'] ) }} -WITH fee AS ( - - SELECT - tx_id, - attribute_value AS fee - FROM - {{ ref('silver__msg_attributes') }} - WHERE - attribute_key = 'fee' - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= CURRENT_DATE -2 -{% endif %} - -qualify(ROW_NUMBER() over(PARTITION BY tx_id -ORDER BY - msg_index)) = 1 -), -spender AS ( - SELECT - tx_id, - SPLIT_PART( - attribute_value, - '/', - 0 - ) AS tx_from - FROM - {{ ref('silver__msg_attributes') }} - WHERE - attribute_key = 'acc_seq' - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= CURRENT_DATE -2 -{% endif %} - -qualify(ROW_NUMBER() over(PARTITION BY tx_id -ORDER BY - msg_index)) = 1 -) SELECT - t.block_id, - t.block_timestamp, - t.tx_id, - s.tx_from, + block_id, + block_timestamp, + tx_id, + tx_from, tx_succeeded, codespace, - COALESCE( - fee, - '0uosmo' - ) AS fee, + fee, gas_used, gas_wanted, tx_code, - msgs + msgs, + COALESCE( + transactions_final_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} + ) AS fact_transactions_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM - {{ ref('silver__transactions') }} - t - LEFT OUTER JOIN fee f - ON t.tx_id = f.tx_id - LEFT OUTER JOIN spender s - ON t.tx_id = s.tx_id - -{% if is_incremental() %} -WHERE - _inserted_timestamp :: DATE >= CURRENT_DATE -2 -{% endif %} + {{ ref('silver__transactions_final') }} diff --git a/models/gold/core/core__fact_transfers.sql b/models/gold/core/core__fact_transfers.sql index 31c1d52..9e354cd 100644 --- a/models/gold/core/core__fact_transfers.sql +++ b/models/gold/core/core__fact_transfers.sql @@ -15,6 +15,20 @@ SELECT DECIMAL, receiver, foreign_address, - foreign_chain + foreign_chain, + COALESCE( + transfers_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_transfers_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__transfers') }} diff --git a/models/gold/core/core__fact_tx_body_msgs.sql b/models/gold/core/core__fact_tx_body_msgs.sql index de8545b..b0f2fcb 100644 --- a/models/gold/core/core__fact_tx_body_msgs.sql +++ b/models/gold/core/core__fact_tx_body_msgs.sql @@ -10,6 +10,20 @@ SELECT tx_succeeded, msg_group AS msg_index, msg_type, - msg + msg, + COALESCE( + tx_body_msgs_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_tx_body_msgs_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__tx_body_msgs') }} diff --git a/models/gold/defi/defi__dim_liquidity_pools.sql b/models/gold/defi/defi__dim_liquidity_pools.sql new file mode 100644 index 0000000..3e778a5 --- /dev/null +++ b/models/gold/defi/defi__dim_liquidity_pools.sql @@ -0,0 +1,29 @@ +{{ config( + materialized = 'view', + tags = ['noncore'] +) }} + +SELECT + 'osmosis' AS blockchain, + module, + pool_created_block_timestamp, + pool_created_block_id, + pool_id, + pool_address, + assets, + COALESCE( + pool_metadata_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS dim_liquidity_pools_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp +FROM + {{ ref('silver__pool_metadata') }} diff --git a/models/gold/defi/gov__dim_liquidity_pools.yml b/models/gold/defi/defi__dim_liquidity_pools.yml similarity index 92% rename from models/gold/defi/gov__dim_liquidity_pools.yml rename to models/gold/defi/defi__dim_liquidity_pools.yml index 8e21ef3..b6c42c3 100644 --- a/models/gold/defi/gov__dim_liquidity_pools.yml +++ b/models/gold/defi/defi__dim_liquidity_pools.yml @@ -1,6 +1,6 @@ version: 2 models: - - name: gov__dim_liquidity_pools + - name: defi__dim_liquidity_pools description: Contains liquidity pool metadata for assets on the Osmosis blockchain. This table was sourced from an API and may not contain every token. columns: - name: BLOCKCHAIN diff --git a/models/gold/defi/defi__fact_liquidity_provider_actions.sql b/models/gold/defi/defi__fact_liquidity_provider_actions.sql index 72b1d7b..0c0fd8c 100644 --- a/models/gold/defi/defi__fact_liquidity_provider_actions.sql +++ b/models/gold/defi/defi__fact_liquidity_provider_actions.sql @@ -14,7 +14,21 @@ SELECT amount, currency, DECIMAL, - msg_group + msg_group, + COALESCE( + liquidity_provider_actions_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_liquidity_provider_actions_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__liquidity_provider_actions') }} UNION ALL @@ -29,7 +43,21 @@ SELECT amount, currency, DECIMAL, - msg_group + msg_group, + COALESCE( + early_liquidity_provider_actions_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_liquidity_provider_actions_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__early_liquidity_provider_actions') }} UNION ALL @@ -44,6 +72,20 @@ SELECT amount, currency, DECIMAL, - msg_group + msg_group, + COALESCE( + liquidity_provider_actions_unpool_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_liquidity_provider_actions_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__liquidity_provider_actions_unpool') }} diff --git a/models/gold/defi/defi__fact_locked_liquidity_actions.sql b/models/gold/defi/defi__fact_locked_liquidity_actions.sql index 786d426..3b94530 100644 --- a/models/gold/defi/defi__fact_locked_liquidity_actions.sql +++ b/models/gold/defi/defi__fact_locked_liquidity_actions.sql @@ -21,7 +21,21 @@ SELECT A.lock_duration, A.unlock_time, A.is_superfluid, - A.unpool_new_lock_ids + A.unpool_new_lock_ids, + COALESCE( + locked_liquidity_actions_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_locked_liquidity_actions_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__locked_liquidity_actions') }} A UNION ALL @@ -43,6 +57,20 @@ SELECT NULL AS lock_duration, NULL AS unlock_time, A.is_superfluid, - NULL AS unpool_new_lock_ids + NULL AS unpool_new_lock_ids, + COALESCE( + locked_liquidity_actions_convert_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_locked_liquidity_actions_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__locked_liquidity_actions_convert') }} A diff --git a/models/gold/defi/defi__fact_pool_fee_day.sql b/models/gold/defi/defi__fact_pool_fee_day.sql index 47e81e1..eff045b 100644 --- a/models/gold/defi/defi__fact_pool_fee_day.sql +++ b/models/gold/defi/defi__fact_pool_fee_day.sql @@ -11,6 +11,20 @@ SELECT currency, fees, fees_usd, - fee_type + fee_type, + COALESCE( + pool_fee_summary_day_id, + {{ dbt_utils.generate_surrogate_key( + ['pool_id','block_date','currency'] + ) }} + ) AS fact_pool_fee_day_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__pool_fee_summary_day') }} diff --git a/models/gold/defi/defi__fact_pool_hour.sql b/models/gold/defi/defi__fact_pool_hour.sql index 9b74f62..6119a3b 100644 --- a/models/gold/defi/defi__fact_pool_hour.sql +++ b/models/gold/defi/defi__fact_pool_hour.sql @@ -23,6 +23,20 @@ SELECT token_2_denom AS token_2_currency, token_2_amount, token_3_denom AS token_3_currency, - token_3_amount + token_3_amount, + COALESCE( + pool_summary_hour_id, + {{ dbt_utils.generate_surrogate_key( + ['pool_id','block_hour'] + ) }} + ) AS fact_pool_hour_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__pool_summary_hour') }} diff --git a/models/gold/defi/defi__fact_superfluid_staking.sql b/models/gold/defi/defi__fact_superfluid_staking.sql index 1580f0d..07f0c15 100644 --- a/models/gold/defi/defi__fact_superfluid_staking.sql +++ b/models/gold/defi/defi__fact_superfluid_staking.sql @@ -21,7 +21,21 @@ SELECT AND currency IS NOT NULL, TRUE, FALSE - ) AS is_unpool + ) AS is_unpool, + COALESCE( + superfluid_staking_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_superfluid_staking_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__superfluid_staking') }} UNION ALL @@ -37,6 +51,20 @@ SELECT DECIMAL, A.validator_address, A.lock_id, - FALSE AS is_unpool + FALSE AS is_unpool, + COALESCE( + locked_liquidity_actions_convert_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_superfluid_staking_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__locked_liquidity_actions_convert') }} A diff --git a/models/gold/defi/defi__fact_swaps.sql b/models/gold/defi/defi__fact_swaps.sql index d50fe9f..02435df 100644 --- a/models/gold/defi/defi__fact_swaps.sql +++ b/models/gold/defi/defi__fact_swaps.sql @@ -17,6 +17,20 @@ SELECT to_currency, TO_DECIMAL, pool_ids, - _BODY_INDEX + _BODY_INDEX, + COALESCE( + swaps_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','_body_index'] + ) }} + ) AS fact_swaps_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__swaps') }} diff --git a/models/gold/defi/gov__dim_liquidity_pools.sql b/models/gold/defi/gov__dim_liquidity_pools.sql deleted file mode 100644 index 7dd45a6..0000000 --- a/models/gold/defi/gov__dim_liquidity_pools.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config( - materialized = 'view', - tags = ['noncore'] -) }} - -SELECT - 'osmosis' AS blockchain, - module, - pool_created_block_timestamp, - pool_created_block_id, - pool_id, - pool_address, - assets -FROM - {{ ref('silver__pool_metadata') }} diff --git a/models/gold/gov/gov__dim_vote_options.sql b/models/gold/gov/gov__dim_vote_options.sql index dd7c541..ebb855c 100644 --- a/models/gold/gov/gov__dim_vote_options.sql +++ b/models/gold/gov/gov__dim_vote_options.sql @@ -12,6 +12,11 @@ SELECT WHEN vote_option = 3 THEN 'NO' WHEN vote_option = 4 THEN 'NO WITH VETO' ELSE 'NULL' - END AS description + END AS description, + {{ dbt_utils.generate_surrogate_key( + ['vote_option'] + ) }} AS dim_vote_option_id, + '2000-01-01' AS inserted_timestamp, + '2000-01-01' AS modified_timestamp FROM {{ ref('silver__governance_votes') }} diff --git a/models/gold/gov/gov__fact_governance_proposal_deposits.sql b/models/gold/gov/gov__fact_governance_proposal_deposits.sql index c673851..bdcd80d 100644 --- a/models/gold/gov/gov__fact_governance_proposal_deposits.sql +++ b/models/gold/gov/gov__fact_governance_proposal_deposits.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'GOVERNANCE' }}}, + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'GOVERNANCE' }} }, tags = ['noncore'] ) }} @@ -13,6 +13,20 @@ SELECT proposal_id, amount, currency, - DECIMAL + DECIMAL, + COALESCE ( + governance_proposal_deposits_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} + ) AS fact_governance_proposal_deposits_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__governance_proposal_deposits') }} diff --git a/models/gold/gov/gov__fact_governance_submit_proposal.sql b/models/gold/gov/gov__fact_governance_submit_proposal.sql index 9fa6932..f057e79 100644 --- a/models/gold/gov/gov__fact_governance_submit_proposal.sql +++ b/models/gold/gov/gov__fact_governance_submit_proposal.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'GOVERNANCE' }}}, + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'GOVERNANCE' }} }, tags = ['noncore'] ) }} @@ -13,6 +13,20 @@ SELECT proposal_id, proposal_type, proposal_title, - proposal_description + proposal_description, + COALESCE ( + governance_submit_proposal_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} + ) AS fact_governance_submit_proposal_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__governance_submit_proposal') }} diff --git a/models/gold/gov/gov__fact_governance_validator_votes.sql b/models/gold/gov/gov__fact_governance_validator_votes.sql index 5ff3f07..56a1fb1 100644 --- a/models/gold/gov/gov__fact_governance_validator_votes.sql +++ b/models/gold/gov/gov__fact_governance_validator_votes.sql @@ -13,7 +13,21 @@ SELECT voting_power, version, created_at, - updated_at + updated_at, + COALESCE ( + validator_memos_id, + {{ dbt_utils.generate_surrogate_key( + ['proposal_id','validator_address'] + ) }} + ) AS fact_validator_votes_id, + COALESCE( + A.inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + A.modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__validator_memos') }} A LEFT JOIN {{ ref('silver__validator_metadata') }} diff --git a/models/gold/gov/gov__fact_governance_votes.sql b/models/gold/gov/gov__fact_governance_votes.sql index 03cb40b..85cf917 100644 --- a/models/gold/gov/gov__fact_governance_votes.sql +++ b/models/gold/gov/gov__fact_governance_votes.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'GOVERNANCE' }}}, + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'GOVERNANCE' }} }, tags = ['noncore'] ) }} @@ -13,6 +13,20 @@ SELECT proposal_id, vote_option, vote_weight, - memo + memo, + COALESCE( + governance_votes_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','proposal_id','voter'] + ) }} + ) AS fact_governance_votes_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__governance_votes') }} diff --git a/models/gold/gov/gov__fact_staking.sql b/models/gold/gov/gov__fact_staking.sql index f8c0a48..a1e4318 100644 --- a/models/gold/gov/gov__fact_staking.sql +++ b/models/gold/gov/gov__fact_staking.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }}}, + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }} }, tags = ['noncore'] ) }} @@ -17,6 +17,20 @@ SELECT amount, currency, DECIMAL, - completion_time + completion_time, + COALESCE( + staking_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_staking_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__staking') }} diff --git a/models/gold/gov/gov__fact_staking_rewards.sql b/models/gold/gov/gov__fact_staking_rewards.sql index adbe0db..6ab8cfc 100644 --- a/models/gold/gov/gov__fact_staking_rewards.sql +++ b/models/gold/gov/gov__fact_staking_rewards.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }}}, + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }} }, tags = ['noncore'] ) }} @@ -15,6 +15,20 @@ SELECT validator_address, amount, currency, - DECIMAL + DECIMAL, + COALESCE( + staking_rewards_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS fact_staking_rewards_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__staking_rewards') }} diff --git a/models/gold/gov/gov__fact_validator_commission.sql b/models/gold/gov/gov__fact_validator_commission.sql index 0a85630..086ebc9 100644 --- a/models/gold/gov/gov__fact_validator_commission.sql +++ b/models/gold/gov/gov__fact_validator_commission.sql @@ -14,6 +14,20 @@ SELECT amount, currency, validator_address_operator, - validator_address_reward + validator_address_reward, + COALESCE( + validator_commission_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} + ) AS fact_validator_commission_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__validator_commission') }} diff --git a/models/gold/gov/gov__fact_validators.sql b/models/gold/gov/gov__fact_validators.sql index 961e528..d78bcf1 100644 --- a/models/gold/gov/gov__fact_validators.sql +++ b/models/gold/gov/gov__fact_validators.sql @@ -19,6 +19,20 @@ SELECT min_self_delegation, RANK, missed_blocks, - raw_metadata + raw_metadata, + COALESCE( + validator_metadata_id, + {{ dbt_utils.generate_surrogate_key( + ['address','creator','blockchain'] + ) }} + ) AS fact_validators_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__validator_metadata') }} diff --git a/models/gold/mars/mars__ez_liquidity_provider_actions.sql b/models/gold/mars/mars__ez_liquidity_provider_actions.sql index 4bdc88c..d7f8655 100644 --- a/models/gold/mars/mars__ez_liquidity_provider_actions.sql +++ b/models/gold/mars/mars__ez_liquidity_provider_actions.sql @@ -32,7 +32,21 @@ SELECT amount ) AS amount, currency, - b.project_name AS symbol + b.project_name AS symbol, + COALESCE( + liquidity_provider_actions_id, + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} + ) AS ez_liquidity_provider_actions_id, + COALESCE( + A.inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + A.modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__liquidity_provider_actions') }} A LEFT JOIN {{ ref('silver__asset_metadata') }} diff --git a/models/gold/mars/mars__ez_pool_fee_day.sql b/models/gold/mars/mars__ez_pool_fee_day.sql index 06c3daf..7e50615 100644 --- a/models/gold/mars/mars__ez_pool_fee_day.sql +++ b/models/gold/mars/mars__ez_pool_fee_day.sql @@ -22,7 +22,21 @@ SELECT b.project_name AS symbol, fees, fees_usd, - fee_type + fee_type, + COALESCE( + pool_fee_summary_day_id, + {{ dbt_utils.generate_surrogate_key( + ['pool_id','block_date','currency'] + ) }} + ) AS ez_pool_fee_day_id, + COALESCE( + A.inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + A.modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__pool_fee_summary_day') }} A LEFT JOIN {{ ref('silver__asset_metadata') }} diff --git a/models/gold/mars/mars__ez_pool_hour.sql b/models/gold/mars/mars__ez_pool_hour.sql index 59dd8be..4d05a8a 100644 --- a/models/gold/mars/mars__ez_pool_hour.sql +++ b/models/gold/mars/mars__ez_pool_hour.sql @@ -31,7 +31,21 @@ SELECT liquidity, liquidity_usd, volume, - volume_usd + volume_usd, + COALESCE( + pool_summary_hour_id, + {{ dbt_utils.generate_surrogate_key( + ['pool_id','block_hour'] + ) }} + ) AS ez_pool_hour_id, + COALESCE( + A.inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + A.modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__pool_summary_hour') }} A LEFT JOIN {{ ref('silver__asset_metadata') }} diff --git a/models/gold/mars/mars__ez_redbank_actions.sql b/models/gold/mars/mars__ez_redbank_actions.sql index 633e16a..70a9a20 100644 --- a/models/gold/mars/mars__ez_redbank_actions.sql +++ b/models/gold/mars/mars__ez_redbank_actions.sql @@ -20,7 +20,21 @@ SELECT on_behalf_of, denom AS currency, amount :: INT amount, - amount_scaled :: INT amount_scaled + amount_scaled :: INT amount_scaled, + COALESCE( + red_bank_actions_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','msg_index'] + ) }} + ) AS ez_red_bank_actions_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__red_bank_actions') }} WHERE diff --git a/models/gold/mars/mars__ez_redbank_balance_changes.sql b/models/gold/mars/mars__ez_redbank_balance_changes.sql index 4a15381..f803513 100644 --- a/models/gold/mars/mars__ez_redbank_balance_changes.sql +++ b/models/gold/mars/mars__ez_redbank_balance_changes.sql @@ -18,7 +18,21 @@ SELECT USER, asset_index, denom AS currency, - rewards_accrued :: INT AS rewards_accrued + rewards_accrued :: INT AS rewards_accrued, + COALESCE( + red_bank_actions_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','msg_index'] + ) }} + ) AS ez_red_bank_balance_changes_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__red_bank_actions') }} WHERE diff --git a/models/gold/mars/mars__ez_redbank_reward_claims.sql b/models/gold/mars/mars__ez_redbank_reward_claims.sql index c67a2e6..1048776 100644 --- a/models/gold/mars/mars__ez_redbank_reward_claims.sql +++ b/models/gold/mars/mars__ez_redbank_reward_claims.sql @@ -16,7 +16,21 @@ SELECT contract_name, action, USER, - mars_rewards + mars_rewards, + COALESCE( + red_bank_actions_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','msg_index'] + ) }} + ) AS ez_red_bank_reward_claims_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__red_bank_actions') }} WHERE diff --git a/models/gold/mars/mars__ez_redbank_reward_distributions.sql b/models/gold/mars/mars__ez_redbank_reward_distributions.sql index fbad101..0edaa72 100644 --- a/models/gold/mars/mars__ez_redbank_reward_distributions.sql +++ b/models/gold/mars/mars__ez_redbank_reward_distributions.sql @@ -17,7 +17,21 @@ SELECT action, to_address, denom AS currency, - amount :: INT AS amount + amount :: INT AS amount, + COALESCE( + red_bank_actions_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','msg_index'] + ) }} + ) AS ez_red_bank_rewards_distributions_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__red_bank_actions') }} WHERE diff --git a/models/gold/mars/mars__ez_swaps.sql b/models/gold/mars/mars__ez_swaps.sql index 987d0b9..6da46e8 100644 --- a/models/gold/mars/mars__ez_swaps.sql +++ b/models/gold/mars/mars__ez_swaps.sql @@ -34,7 +34,21 @@ SELECT ) AS to_amount, to_currency, t.project_name to_symbol, - pool_ids + pool_ids, + COALESCE( + swaps_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','_body_index'] + ) }} + ) AS ez_swaps_id, + COALESCE( + A.inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + A.modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__swaps') }} A LEFT JOIN {{ ref('silver__asset_metadata') }} diff --git a/models/gold/mars/mars__ez_token_day.sql b/models/gold/mars/mars__ez_token_day.sql index 9af4aee..add2664 100644 --- a/models/gold/mars/mars__ez_token_day.sql +++ b/models/gold/mars/mars__ez_token_day.sql @@ -14,7 +14,21 @@ SELECT liquidity, liquidity_usd, volume, - volume_usd + volume_usd, + COALESCE( + token_summary_day_id, + {{ dbt_utils.generate_surrogate_key( + ['currency', 'block_date'] + ) }} + ) AS ez_token_day_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__token_summary_day') }} WHERE diff --git a/models/gold/price/price__dim_prices.sql b/models/gold/price/price__dim_prices.sql index fe47eb3..c596f89 100644 --- a/models/gold/price/price__dim_prices.sql +++ b/models/gold/price/price__dim_prices.sql @@ -1,55 +1,25 @@ {{ config( - materialized = 'table', + materialized = 'view', meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'PRICES' }} }, tags = ['noncore'] ) }} SELECT - recorded_hour AS recorded_at, + recorded_at, symbol, - NULL AS currency, - CLOSE AS price, - NULL AS total_supply, - NULL AS volume_24h, - 'coin gecko' AS provider -FROM - {{ ref('silver__token_prices_coin_gecko_hourly') }} -UNION ALL -SELECT - recorded_hour AS recorded_at, - symbol, - NULL AS currency, - CLOSE AS price, - NULL AS total_supply, - NULL AS volume_24h, - 'coin market cap' AS provider -FROM - {{ ref('silver__token_prices_coin_market_cap_hourly') }} -UNION ALL -SELECT - block_hour AS recorded_at, - project_name AS symbol, currency, - price_usd AS price, - NULL AS total_supply, - NULL AS volume_24h, - 'swaps' AS provider + price, + total_supply, + volume_24h, + provider, + prices_combined_id AS dim_prices_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM - {{ ref('silver__prices_swaps') }} -UNION ALL -SELECT - DATE_TRUNC( - 'hour', - block_timestamp - ) AS recorded_at, - project_name AS symbol, - token_address AS currency, - price_usd AS price, - NULL AS total_supply, - NULL AS volume_24h, - 'pool balance' AS provider -FROM - {{ ref('silver__pool_token_prices_usd') }} A - LEFT JOIN {{ ref('silver__asset_metadata') }} - b - ON A.token_address = b.address + {{ ref('silver__prices_combined') }} diff --git a/models/gold/price/price__ez_prices.sql b/models/gold/price/price__ez_prices.sql index 0529795..aad3e75 100644 --- a/models/gold/price/price__ez_prices.sql +++ b/models/gold/price/price__ez_prices.sql @@ -23,7 +23,10 @@ WITH p_base AS ( WHEN 'coin market cap' THEN 2 WHEN 'pool balances' THEN 3 ELSE 4 - END AS pro_rank + END AS pro_rank, + dim_prices_id, + inserted_timestamp, + modified_timestamp FROM {{ ref('price__dim_prices') }} A WHERE @@ -37,7 +40,16 @@ SELECT b_1.address, b_2.address ) AS currency, - A.price + A.price, + dim_prices_id AS ez_prices_id, + COALESCE( + A.inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + A.modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM p_base A LEFT JOIN {{ ref('silver__asset_metadata') }} diff --git a/models/silver/core/silver__airdrops.sql b/models/silver/core/silver__airdrops.sql index c6fb2ba..051dda2 100644 --- a/models/silver/core/silver__airdrops.sql +++ b/models/silver/core/silver__airdrops.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], ) }} @@ -173,7 +174,7 @@ SELECT 'AIRDROP' AS transfer_type, r.msg_index, sender, - amount :: NUMBER as amount, + amount :: NUMBER AS amount, currency, DECIMAL, receiver, @@ -183,7 +184,13 @@ SELECT r.tx_id, r.msg_index, currency - ) _unique_key + ) _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS airdrops_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM areceiver r LEFT OUTER JOIN aamount C diff --git a/models/silver/core/silver__asset_metadata.sql b/models/silver/core/silver__asset_metadata.sql index 839825d..64e9828 100644 --- a/models/silver/core/silver__asset_metadata.sql +++ b/models/silver/core/silver__asset_metadata.sql @@ -41,7 +41,13 @@ SELECT address, creator, blockchain - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS asset_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM base qualify(ROW_NUMBER() over(PARTITION BY blockchain, creator, address ORDER BY diff --git a/models/silver/core/silver__blockchain.sql b/models/silver/core/silver__blockchain.sql index 019bdc3..72d54eb 100644 --- a/models/silver/core/silver__blockchain.sql +++ b/models/silver/core/silver__blockchain.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "block_id", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -63,7 +64,13 @@ SELECT block_header, block_id AS block_id_object, num_txs, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['height'] + ) }} AS blockchain_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM fin qualify(ROW_NUMBER() over(PARTITION BY height ORDER BY diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index 4e9a727..0c0a9f2 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ['chain_id','block_id'], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['core'] ) }} @@ -14,7 +15,13 @@ SELECT COALESCE(ARRAY_SIZE(DATA :result :block :data :txs) :: NUMBER, 0) AS tx_count, DATA :result :block :header: proposer_address :: STRING AS proposer_address, DATA :result :block :header: validators_hash :: STRING AS validator_hash, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_id'] + ) }} AS blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {% if is_incremental() %} diff --git a/models/silver/core/silver__daily_balances.sql b/models/silver/core/silver__daily_balances.sql index 38df63b..bcaea4d 100644 --- a/models/silver/core/silver__daily_balances.sql +++ b/models/silver/core/silver__daily_balances.sql @@ -284,39 +284,40 @@ balance_temp AS ( AND d.address = b.address AND d.currency = b.currency AND d.balance_type = b.balance_type -) -SELECT - DATE, - balance_type, - address, - currency, - DECIMAL, - LAST_VALUE( - balance ignore nulls - ) over( - PARTITION BY address, +), +fin AS ( + SELECT + DATE, + balance_type, + address, currency, - balance_type - ORDER BY - DATE ASC rows unbounded preceding - ) AS balance -FROM - balance_temp -WHERE - balance_type <> 'pool' -UNION ALL -SELECT - A.block_timestamp :: DATE AS DATE, - 'pool' AS balance_type, - A.pool_address AS address, - token_0_denom currency, - DECIMAL, - token_0_amount balance -FROM - {{ ref('silver__pool_balances') }} A - LEFT JOIN {{ ref('silver__asset_metadata') }} - am - ON A.token_0_denom = am.address + DECIMAL, + LAST_VALUE( + balance ignore nulls + ) over( + PARTITION BY address, + currency, + balance_type + ORDER BY + DATE ASC rows unbounded preceding + ) AS balance + FROM + balance_temp + WHERE + balance_type <> 'pool' + UNION ALL + SELECT + A.block_timestamp :: DATE AS DATE, + 'pool' AS balance_type, + A.pool_address AS address, + token_0_denom currency, + DECIMAL, + token_0_amount balance + FROM + {{ ref('silver__pool_balances') }} A + LEFT JOIN {{ ref('silver__asset_metadata') }} + am + ON A.token_0_denom = am.address {% if is_incremental() %} WHERE @@ -410,3 +411,19 @@ AND A.block_timestamp :: DATE >= ( qualify(ROW_NUMBER() over(PARTITION BY block_timestamp :: DATE, pool_id, token_3_denom ORDER BY block_timestamp DESC) = 1) +) +SELECT + DATE, + balance_type, + address, + balance, + currency, + DECIMAL, + {{ dbt_utils.generate_surrogate_key( + ['date', 'address', 'balance_type', 'currency'] + ) }} AS daily_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + fin diff --git a/models/silver/core/silver__daily_balances_final.sql b/models/silver/core/silver__daily_balances_final.sql new file mode 100644 index 0000000..659ae00 --- /dev/null +++ b/models/silver/core/silver__daily_balances_final.sql @@ -0,0 +1,36 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', date, address, balance_type, currency)", + incremental_strategy = 'delete+insert', + cluster_by = ['date'], + tags = ['balances'] +) }} + +SELECT + DATE, + balance_type, + address, + currency, + DECIMAL, + balance, + {{ dbt_utils.generate_surrogate_key( + ['date', 'address', 'balance_type', 'currency'] + ) }} AS daily_balances_final_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('silver__daily_balances') }} +WHERE + balance > 0 + +{% if is_incremental() %} +AND DATE >= ( + SELECT + MAX( + DATE + ) + FROM + {{ this }} +) - INTERVAL '48 HOURS' +{% endif %} diff --git a/models/silver/core/silver__latest_liquid_balances.sql b/models/silver/core/silver__latest_liquid_balances.sql index 2cbd99a..6a992f0 100644 --- a/models/silver/core/silver__latest_liquid_balances.sql +++ b/models/silver/core/silver__latest_liquid_balances.sql @@ -24,7 +24,13 @@ SELECT b.block_timestamp, b.decimal, b.currency, - b._inserted_timestamp + b._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['b.address', 'b.balance_type'] + ) }} AS latest_liquid_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('silver__liquid_balances') }} b diff --git a/models/silver/core/silver__liquid_balances.sql b/models/silver/core/silver__liquid_balances.sql index 3951a7c..384fc9b 100644 --- a/models/silver/core/silver__liquid_balances.sql +++ b/models/silver/core/silver__liquid_balances.sql @@ -210,7 +210,13 @@ SELECT CASE WHEN currency LIKE 'gamm/pool/%' THEN 18 ELSE b.decimal - END AS DECIMAL + END AS DECIMAL, + {{ dbt_utils.generate_surrogate_key( + ['block_id', 'address', 'currency'] + ) }} AS liquid_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM joined A LEFT OUTER JOIN {{ ref('silver__asset_metadata') }} diff --git a/models/silver/core/silver__msg_attributes.sql b/models/silver/core/silver__msg_attributes.sql index 930d9c5..0602d80 100644 --- a/models/silver/core/silver__msg_attributes.sql +++ b/models/silver/core/silver__msg_attributes.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", tags = ['core'] @@ -29,7 +30,13 @@ SELECT tx_id, msg_index, attribute_index - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS msg_attributes_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('silver__msgs') }} A, LATERAL FLATTEN( diff --git a/models/silver/core/silver__msgs.sql b/models/silver/core/silver__msgs.sql index ba9b088..0929b9a 100644 --- a/models/silver/core/silver__msgs.sql +++ b/models/silver/core/silver__msgs.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", tags = ['core'] @@ -148,7 +149,13 @@ SELECT '-', A.tx_id, A.msg_index - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS msgs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM prefinal A LEFT JOIN grp b diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index 8b13071..a49848f 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_id", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_id)", tags = ['core'] @@ -138,7 +139,25 @@ combo AS ( {% endif %} ) SELECT - * + source, + block_id, + block_timestamp, + codespace, + gas_used, + gas_wanted, + tx_id, + tx_succeeded, + tx_code, + msgs, + auth_info, + tx_body, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} AS transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM combo qualify(ROW_NUMBER() over(PARTITION BY tx_id ORDER BY diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql new file mode 100644 index 0000000..6fe5558 --- /dev/null +++ b/models/silver/core/silver__transactions_final.sql @@ -0,0 +1,81 @@ +{{ config( + materialized = 'incremental', + unique_key = "tx_id", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE'], + tags = ['core'] +) }} + +WITH fee AS ( + + SELECT + tx_id, + attribute_value AS fee + FROM + {{ ref('silver__msg_attributes') }} + WHERE + attribute_key = 'fee' + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= CURRENT_DATE -2 +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY tx_id +ORDER BY + msg_index)) = 1 +), +spender AS ( + SELECT + tx_id, + SPLIT_PART( + attribute_value, + '/', + 0 + ) AS tx_from + FROM + {{ ref('silver__msg_attributes') }} + WHERE + attribute_key = 'acc_seq' + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= CURRENT_DATE -2 +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY tx_id +ORDER BY + msg_index)) = 1 +) +SELECT + t.block_id, + t.block_timestamp, + t.tx_id, + s.tx_from, + tx_succeeded, + codespace, + COALESCE( + fee, + '0uosmo' + ) AS fee, + gas_used, + gas_wanted, + tx_code, + msgs, + {{ dbt_utils.generate_surrogate_key( + ['t.tx_id'] + ) }} AS transactions_final_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('silver__transactions') }} + t + LEFT OUTER JOIN fee f + ON t.tx_id = f.tx_id + LEFT OUTER JOIN spender s + ON t.tx_id = s.tx_id + +{% if is_incremental() %} +WHERE + _inserted_timestamp :: DATE >= CURRENT_DATE -2 +{% endif %} diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index 2dd8461..84e4969 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -3,6 +3,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], tags = ['core'] ) }} @@ -466,7 +467,13 @@ SELECT C.foreign_chain ) AS foreign_chain, A._inserted_timestamp, - A._unique_key + A._unique_key, + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM fin A LEFT JOIN links b diff --git a/models/silver/core/silver__tx_body_msgs.sql b/models/silver/core/silver__tx_body_msgs.sql index 5c0a72e..15f774c 100644 --- a/models/silver/core/silver__tx_body_msgs.sql +++ b/models/silver/core/silver__tx_body_msgs.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], tags = ['core'] ) }} @@ -48,6 +49,12 @@ SELECT tx_id, msg_group ) AS _unique_key, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS tx_body_msgs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM b diff --git a/models/silver/crossdb/silver__croschain_icns.sql b/models/silver/crossdb/silver__croschain_icns.sql index 4b2ff1d..d811276 100644 --- a/models/silver/crossdb/silver__croschain_icns.sql +++ b/models/silver/crossdb/silver__croschain_icns.sql @@ -10,7 +10,13 @@ SELECT tag_name AS label, tag_type AS project_name, start_date, - end_date + end_date, + {{ dbt_utils.generate_surrogate_key( + ['address'] + ) }} AS crosschain_icns_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ source( 'crosschain', diff --git a/models/silver/crossdb/silver__croschain_labels.sql b/models/silver/crossdb/silver__croschain_labels.sql index 770cd4a..efec95c 100644 --- a/models/silver/crossdb/silver__croschain_labels.sql +++ b/models/silver/crossdb/silver__croschain_labels.sql @@ -12,7 +12,13 @@ SELECT label_type, label_subtype, address_name, - project_name + project_name, + {{ dbt_utils.generate_surrogate_key( + ['address'] + ) }} AS crosschain_labels_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ source( 'crosschain', diff --git a/models/silver/defi/silver___early_swaps.sql b/models/silver/defi/silver___early_swaps.sql index 12b4a5d..89ff25e 100644 --- a/models/silver/defi/silver___early_swaps.sql +++ b/models/silver/defi/silver___early_swaps.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_id", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'] ) }} @@ -195,7 +196,13 @@ SELECT ELSE tok.to_decimal :: NUMBER END AS TO_DECIMAL, NULL :: ARRAY AS pool_ids, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['e.tx_id'] + ) }} AS _early_swaps_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM early_swaps e LEFT OUTER JOIN {{ ref('silver__transactions') }} diff --git a/models/silver/defi/silver__early_liquidity_provider_actions.sql b/models/silver/defi/silver__early_liquidity_provider_actions.sql index 8857673..ff188be 100644 --- a/models/silver/defi/silver__early_liquidity_provider_actions.sql +++ b/models/silver/defi/silver__early_liquidity_provider_actions.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'] ) }} @@ -319,7 +320,13 @@ SELECT d.tx_id, msg_index, currency - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS early_liquidity_provider_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM decimals d JOIN pools p diff --git a/models/silver/defi/silver__liquidity_provider_actions.sql b/models/silver/defi/silver__liquidity_provider_actions.sql index 3e388ee..9fbf3ee 100644 --- a/models/silver/defi/silver__liquidity_provider_actions.sql +++ b/models/silver/defi/silver__liquidity_provider_actions.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -334,7 +335,13 @@ SELECT d.msg_group, d.msg_sub_group, d.currency - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS liquidity_provider_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM decimals d JOIN lper l diff --git a/models/silver/defi/silver__liquidity_provider_actions_unpool.sql b/models/silver/defi/silver__liquidity_provider_actions_unpool.sql index 6800d6b..90e6e49 100644 --- a/models/silver/defi/silver__liquidity_provider_actions_unpool.sql +++ b/models/silver/defi/silver__liquidity_provider_actions_unpool.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -328,7 +329,13 @@ SELECT d.tx_id, d.msg_index, d.currency - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS liquidity_provider_actions_unpool_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM decimals d JOIN act diff --git a/models/silver/defi/silver__locked_liquidity_actions.sql b/models/silver/defi/silver__locked_liquidity_actions.sql index 92814d5..51b6d56 100644 --- a/models/silver/defi/silver__locked_liquidity_actions.sql +++ b/models/silver/defi/silver__locked_liquidity_actions.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], tags = ['noncore'] ) }} @@ -394,6 +395,12 @@ SELECT is_superfluid, unpool_new_lock_ids, _unique_key, - _INSERTED_TIMESTAMP + _INSERTED_TIMESTAMP, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS locked_liquidity_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM fin diff --git a/models/silver/defi/silver__locked_liquidity_actions_begin_unlock.sql b/models/silver/defi/silver__locked_liquidity_actions_begin_unlock.sql index 82b5056..0c0c706 100644 --- a/models/silver/defi/silver__locked_liquidity_actions_begin_unlock.sql +++ b/models/silver/defi/silver__locked_liquidity_actions_begin_unlock.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], tags = ['noncore'] ) }} @@ -229,6 +230,12 @@ SELECT ), locker ) AS _unique_key, - _INSERTED_TIMESTAMP + _INSERTED_TIMESTAMP, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS locked_liquidity_actions_begin_unlock_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL diff --git a/models/silver/defi/silver__locked_liquidity_actions_convert.sql b/models/silver/defi/silver__locked_liquidity_actions_convert.sql index 0b8e317..63bf01c 100644 --- a/models/silver/defi/silver__locked_liquidity_actions_convert.sql +++ b/models/silver/defi/silver__locked_liquidity_actions_convert.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], tags = ['noncore'] ) }} @@ -199,7 +200,13 @@ SELECT -1 ) ) AS _unique_key, - _INSERTED_TIMESTAMP + _INSERTED_TIMESTAMP, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS locked_liquidity_actions_convert_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL A JOIN lper b diff --git a/models/silver/defi/silver__locked_liquidity_balances.sql b/models/silver/defi/silver__locked_liquidity_balances.sql index 67950ef..f29df5b 100644 --- a/models/silver/defi/silver__locked_liquidity_balances.sql +++ b/models/silver/defi/silver__locked_liquidity_balances.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -226,6 +227,12 @@ SELECT lock_id, currency ) AS _unique_key, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS locked_liquidity_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM combine diff --git a/models/silver/defi/silver__locked_liquidity_transactions.sql b/models/silver/defi/silver__locked_liquidity_transactions.sql index f8aee17..a7ca637 100644 --- a/models/silver/defi/silver__locked_liquidity_transactions.sql +++ b/models/silver/defi/silver__locked_liquidity_transactions.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['_inserted_timestamp::DATE'], tags = ['noncore'] ) }} @@ -136,7 +137,13 @@ SELECT tx_id, msg_group, lock_id - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS locked_liquidity_transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM base WHERE diff --git a/models/silver/defi/silver__pool_balances.sql b/models/silver/defi/silver__pool_balances.sql index 42e12b6..19d0a05 100644 --- a/models/silver/defi/silver__pool_balances.sql +++ b/models/silver/defi/silver__pool_balances.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ["pool_id","block_id"], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp'], tags = ['noncore'] ) }} @@ -95,7 +96,13 @@ SELECT ) :: bigint AS total_weight, b.value :scaling_factor_controller :: STRING AS scaling_factor_controller, b.value :scaling_factors AS scaling_factors, - _INSERTED_DATE AS _inserted_timestamp + _INSERTED_DATE AS _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['pool_id','a.block_id'] + ) }} AS pool_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ source( 'bronze_streamline', diff --git a/models/silver/defi/silver__pool_fee_summary_day.sql b/models/silver/defi/silver__pool_fee_summary_day.sql index a10e843..9cad419 100644 --- a/models/silver/defi/silver__pool_fee_summary_day.sql +++ b/models/silver/defi/silver__pool_fee_summary_day.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ["block_date","pool_id","currency"], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_date'], tags = ['noncore'] ) }} @@ -162,7 +163,13 @@ SELECT SUM(fees) AS fees, SUM(fees_usd) AS fees_usd, fee_type, - MAX(_inserted_timestamp) AS _inserted_timestamp + MAX(_inserted_timestamp) AS _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['pool_id','block_date','currency'] + ) }} AS pool_fee_summary_day_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM pre_agg A GROUP BY diff --git a/models/silver/defi/silver__pool_metadata.sql b/models/silver/defi/silver__pool_metadata.sql index e76e443..d7cca5f 100644 --- a/models/silver/defi/silver__pool_metadata.sql +++ b/models/silver/defi/silver__pool_metadata.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], tags = ['noncore'] ) }} @@ -223,7 +224,13 @@ SELECT module, pool_id ) AS _unique_key, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS pool_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM e JOIN pool_creation_txs C diff --git a/models/silver/defi/silver__pool_summary_hour.sql b/models/silver/defi/silver__pool_summary_hour.sql index 5621050..ae1f174 100644 --- a/models/silver/defi/silver__pool_summary_hour.sql +++ b/models/silver/defi/silver__pool_summary_hour.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ["pool_id","block_hour"], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp'], tags = ['noncore'] ) }} @@ -322,7 +323,13 @@ SELECT token_2_amount, token_3_denom, token_3_amount, - A._inserted_timestamp + A._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['a.pool_id','block_hour'] + ) }} AS pool_summary_hour_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM adjust A LEFT JOIN pool_token_prices b diff --git a/models/silver/defi/silver__swaps.sql b/models/silver/defi/silver__swaps.sql index 32c61ee..1a6450c 100644 --- a/models/silver/defi/silver__swaps.sql +++ b/models/silver/defi/silver__swaps.sql @@ -368,6 +368,12 @@ SELECT TO_DECIMAL, pool_ids, _inserted_timestamp, - _body_index + _body_index, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','_body_index'] + ) }} AS swaps_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM pre_final2 diff --git a/models/silver/defi/silver__token_summary_day.sql b/models/silver/defi/silver__token_summary_day.sql index d8b9069..af9cee7 100644 --- a/models/silver/defi/silver__token_summary_day.sql +++ b/models/silver/defi/silver__token_summary_day.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ["currency","block_date"], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_date'], tags = ['noncore'] ) }} @@ -31,7 +32,13 @@ SELECT MAX(liquidity_usd) liquidity_usd, SUM(volume) volume, SUM(volume_usd) volume_usd, - MAX(_inserted_timestamp) AS _inserted_timestamp + MAX(_inserted_timestamp) AS _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['currency','block_date'] + ) }} AS token_summary_day_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM last_block_of_day blc JOIN {{ ref('silver__pool_summary_hour') }} A diff --git a/models/silver/defi/silver__token_swapped.sql b/models/silver/defi/silver__token_swapped.sql index 140f90b..5c98ebf 100644 --- a/models/silver/defi/silver__token_swapped.sql +++ b/models/silver/defi/silver__token_swapped.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ["tx_id","msg_index"], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -93,7 +94,13 @@ SELECT ' ', 0 ) :: FLOAT AS to_amount, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','msg_index'] + ) }} AS token_swapped_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM token_swapped A JOIN bhour b diff --git a/models/silver/gov/silver__governance_proposal_deposits.sql b/models/silver/gov/silver__governance_proposal_deposits.sql index a7a222f..2a88234 100644 --- a/models/silver/gov/silver__governance_proposal_deposits.sql +++ b/models/silver/gov/silver__governance_proposal_deposits.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_id", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -116,7 +117,13 @@ SELECT v.amount, v.currency, DECIMAL, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['p.tx_id'] + ) }} AS governance_proposal_deposits_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM deposit_value v INNER JOIN proposal_ids p diff --git a/models/silver/gov/silver__governance_submit_proposal.sql b/models/silver/gov/silver__governance_submit_proposal.sql index b4edd98..f4c2042 100644 --- a/models/silver/gov/silver__governance_submit_proposal.sql +++ b/models/silver/gov/silver__governance_submit_proposal.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_id", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -106,7 +107,13 @@ SELECT tx_body :messages [0] :content :description, tx_body :messages [0] :msgs [0] :content :description ) :: STRING AS proposal_description, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['p.tx_id'] + ) }} AS governance_submit_proposal_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM proposal_ids p INNER JOIN proposal_type y diff --git a/models/silver/gov/silver__governance_votes.sql b/models/silver/gov/silver__governance_votes.sql index 48155b0..14190d3 100644 --- a/models/silver/gov/silver__governance_votes.sql +++ b/models/silver/gov/silver__governance_votes.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -141,7 +142,13 @@ SELECT vote_weight, memo, _inserted_timestamp, - _unique_key + _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS governance_votes_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM pre_final p LEFT OUTER JOIN memo_text m diff --git a/models/silver/gov/silver__staked_balances.sql b/models/silver/gov/silver__staked_balances.sql index a797b49..760c7c3 100644 --- a/models/silver/gov/silver__staked_balances.sql +++ b/models/silver/gov/silver__staked_balances.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', + unique_key = ['block_id', 'address', 'currency'], cluster_by = ['block_timestamp'], tags = ['noncore'] ) }} @@ -73,6 +74,12 @@ SELECT ORDER BY block_timestamp ASC rows unbounded preceding ) AS balance, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_id', 'address', 'currency'] + ) }} AS staked_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM all_staked diff --git a/models/silver/gov/silver__staking.sql b/models/silver/gov/silver__staking.sql index 75f9546..bba783e 100644 --- a/models/silver/gov/silver__staking.sql +++ b/models/silver/gov/silver__staking.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -390,7 +391,13 @@ SELECT currency, delegator_address, validator_address - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS staking_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM add_dec A LEFT OUTER JOIN {{ ref('silver__asset_metadata') }} diff --git a/models/silver/gov/silver__staking_rewards.sql b/models/silver/gov/silver__staking_rewards.sql index be553d0..a1bcbf6 100644 --- a/models/silver/gov/silver__staking_rewards.sql +++ b/models/silver/gov/silver__staking_rewards.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -596,7 +597,13 @@ SELECT tbd.delegator_address ), A.validator_address - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS staking_rewards_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM prefinal A LEFT OUTER JOIN {{ ref('silver__asset_metadata') }} diff --git a/models/silver/gov/silver__superfluid_actions.sql b/models/silver/gov/silver__superfluid_actions.sql index 86579a5..bc0838f 100644 --- a/models/silver/gov/silver__superfluid_actions.sql +++ b/models/silver/gov/silver__superfluid_actions.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['_inserted_timestamp::DATE'], tags = ['noncore'] ) }} @@ -28,7 +29,13 @@ SELECT A.tx_id, A.msg_group ) AS _unique_key, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS superfluid_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('silver__tx_body_msgs') }} A LEFT JOIN ( diff --git a/models/silver/gov/silver__superfluid_staked_balances.sql b/models/silver/gov/silver__superfluid_staked_balances.sql index b51115c..48b31a7 100644 --- a/models/silver/gov/silver__superfluid_staked_balances.sql +++ b/models/silver/gov/silver__superfluid_staked_balances.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -67,6 +68,12 @@ SELECT lock_id, currency ) AS _unique_key, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS superfluid_staked_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM lp_balances diff --git a/models/silver/gov/silver__superfluid_staking.sql b/models/silver/gov/silver__superfluid_staking.sql index 8d0738f..0c550d4 100644 --- a/models/silver/gov/silver__superfluid_staking.sql +++ b/models/silver/gov/silver__superfluid_staking.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -142,7 +143,13 @@ SELECT -1 ), msg_action_description - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS superfluid_staking_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM base_txn A LEFT JOIN vals b diff --git a/models/silver/gov/silver__validator_commission.sql b/models/silver/gov/silver__validator_commission.sql index 9176681..a36f86c 100644 --- a/models/silver/gov/silver__validator_commission.sql +++ b/models/silver/gov/silver__validator_commission.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ["tx_id","msg_group","msg_sub_group"], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], tags = ['noncore'] ) }} @@ -188,7 +189,13 @@ SELECT END AS currency, A.validator_address_operator, d.validator_address_reward, - b._inserted_timestamp + b._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['a.tx_id','a.msg_group','a.msg_sub_group'] + ) }} AS validator_commission_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM combo A JOIN LATERAL SPLIT_TO_TABLE( diff --git a/models/silver/gov/silver__validator_memos.sql b/models/silver/gov/silver__validator_memos.sql index ef6a812..9561142 100644 --- a/models/silver/gov/silver__validator_memos.sql +++ b/models/silver/gov/silver__validator_memos.sql @@ -35,7 +35,13 @@ SELECT i.value :version :: INT AS version, i.value :created_at :: datetime AS created_at, i.value :updated_at :: datetime AS updated_at, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['proposal_id','validator_address'] + ) }} AS validator_memos_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM base, LATERAL FLATTEN( diff --git a/models/silver/gov/silver__validator_metadata.sql b/models/silver/gov/silver__validator_metadata.sql index 2f587f1..cd5ff9c 100644 --- a/models/silver/gov/silver__validator_metadata.sql +++ b/models/silver/gov/silver__validator_metadata.sql @@ -26,7 +26,13 @@ SELECT creator, blockchain ) AS _unique_key, - VALUE :update_time :: TIMESTAMP AS _inserted_timestamp + VALUE :update_time :: TIMESTAMP AS _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS validator_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ source( 'bronze_streamline', diff --git a/models/silver/mars/silver__red_bank_actions.sql b/models/silver/mars/silver__red_bank_actions.sql index bfb4514..0e8ca40 100644 --- a/models/silver/mars/silver__red_bank_actions.sql +++ b/models/silver/mars/silver__red_bank_actions.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ["tx_id","msg_index"], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','action'], tags = ['noncore'] ) }} @@ -93,7 +94,13 @@ SELECT attributes :borrow_rate :: FLOAT AS borrow_rate, attributes :liquidity_index :: STRING AS liquidity_index, attributes :liquidity_rate :: FLOAT AS liquidity_rate, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['a.tx_id','a.msg_index'] + ) }} AS red_bank_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('silver__msg_attributes') }} A JOIN tx b diff --git a/models/silver/prices/silver__pool_token_prices.sql b/models/silver/prices/silver__pool_token_prices.sql index c25a6b3..5d59943 100644 --- a/models/silver/prices/silver__pool_token_prices.sql +++ b/models/silver/prices/silver__pool_token_prices.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ["token_address","pool_id","block_id"], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['_inserted_timestamp::DATE'], tags = ['noncore'] ) }} @@ -112,6 +113,12 @@ SELECT pool_total DESC ) AS token_pool_rank, pool_type, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['token_address','block_id','pool_id'] + ) }} AS pool_token_prices_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM fin diff --git a/models/silver/prices/silver__pool_token_prices_usd.sql b/models/silver/prices/silver__pool_token_prices_usd.sql index 5142f6f..9593477 100644 --- a/models/silver/prices/silver__pool_token_prices_usd.sql +++ b/models/silver/prices/silver__pool_token_prices_usd.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ["token_address","block_id"], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['_inserted_timestamp::DATE'], tags = ['noncore'] ) }} @@ -273,43 +274,60 @@ stable_prices AS ( ON b.block_id = sf2.block_id AND b.pool_id = sf2.pool_id AND b.token_address = sf2.denom +), +fin AS ( + SELECT + block_id, + block_timestamp, + token_address, + osmo_price AS price_usd, + pool_id, + _inserted_timestamp + FROM + osmo_price + UNION ALL + SELECT + block_id, + block_timestamp, + token_address, + price_usd, + pool_id, + _inserted_timestamp + FROM + osmo_pools + UNION ALL + SELECT + block_id, + block_timestamp, + token_address, + price_usd, + pool_id, + _inserted_timestamp + FROM + non_osmo_pools + UNION ALL + SELECT + block_id, + block_timestamp, + token_address, + price_usd, + pool_id, + _inserted_timestamp + FROM + stable_prices ) -SELECT - block_id, - block_timestamp, - token_address, - osmo_price AS price_usd, - pool_id, - _inserted_timestamp -FROM - osmo_price -UNION ALL SELECT block_id, block_timestamp, token_address, price_usd, pool_id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['token_address','block_id'] + ) }} AS pool_token_prices_usd_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM - osmo_pools -UNION ALL -SELECT - block_id, - block_timestamp, - token_address, - price_usd, - pool_id, - _inserted_timestamp -FROM - non_osmo_pools -UNION ALL -SELECT - block_id, - block_timestamp, - token_address, - price_usd, - pool_id, - _inserted_timestamp -FROM - stable_prices + fin diff --git a/models/silver/prices/silver__prices_combined.sql b/models/silver/prices/silver__prices_combined.sql new file mode 100644 index 0000000..91dd873 --- /dev/null +++ b/models/silver/prices/silver__prices_combined.sql @@ -0,0 +1,110 @@ +{{ config( + materialized = 'table', + tags = ['noncore'] +) }} + +SELECT + recorded_hour AS recorded_at, + symbol, + NULL AS currency, + CLOSE AS price, + NULL AS total_supply, + NULL AS volume_24h, + 'coin gecko' AS provider, + COALESCE( + token_prices_coin_gecko_hourly_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS prices_combined_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp +FROM + {{ ref('silver__token_prices_coin_gecko_hourly') }} +UNION ALL +SELECT + recorded_hour AS recorded_at, + symbol, + NULL AS currency, + CLOSE AS price, + NULL AS total_supply, + NULL AS volume_24h, + 'coin market cap' AS provider, + COALESCE( + token_prices_coin_market_cap_hourly_id, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} + ) AS prices_combined_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp +FROM + {{ ref('silver__token_prices_coin_market_cap_hourly') }} +UNION ALL +SELECT + block_hour AS recorded_at, + project_name AS symbol, + currency, + price_usd AS price, + NULL AS total_supply, + NULL AS volume_24h, + 'swaps' AS provider, + COALESCE( + prices_swaps_id, + {{ dbt_utils.generate_surrogate_key( + ['block_hour','currency'] + ) }} + ) AS prices_combined_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp +FROM + {{ ref('silver__prices_swaps') }} +UNION ALL +SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS recorded_at, + project_name AS symbol, + token_address AS currency, + price_usd AS price, + NULL AS total_supply, + NULL AS volume_24h, + 'pool balance' AS provider, + COALESCE( + pool_token_prices_usd_id, + {{ dbt_utils.generate_surrogate_key( + ['token_address','block_id'] + ) }} + ) AS prices_combined_id, + COALESCE( + A.inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + A.modified_timestamp, + '2000-01-01' + ) AS modified_timestamp +FROM + {{ ref('silver__pool_token_prices_usd') }} A + LEFT JOIN {{ ref('silver__asset_metadata') }} + b + ON A.token_address = b.address diff --git a/models/silver/prices/silver__prices_swaps.sql b/models/silver/prices/silver__prices_swaps.sql index 72af4fa..0cb7bdf 100644 --- a/models/silver/prices/silver__prices_swaps.sql +++ b/models/silver/prices/silver__prices_swaps.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ["block_hour", "currency"], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_hour::DATE'], tags = ['noncore'] ) }} @@ -540,7 +541,13 @@ SELECT max_price_usd_hour, volatility_measure, swaps_in_hour, - volume_in_hour * price AS volume_usd_in_hour + volume_in_hour * price AS volume_usd_in_hour, + {{ dbt_utils.generate_surrogate_key( + ['block_hour','currency'] + ) }} AS prices_swaps_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM fill_in_the_blanks_temp {# WHERE project_name IN ( diff --git a/models/silver/prices/silver__token_prices_coin_gecko_hourly.sql b/models/silver/prices/silver__token_prices_coin_gecko_hourly.sql index 1b5f0b2..5d86726 100644 --- a/models/silver/prices/silver__token_prices_coin_gecko_hourly.sql +++ b/models/silver/prices/silver__token_prices_coin_gecko_hourly.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['recorded_hour::DATE'], tags = ['noncore'] ) }} @@ -369,7 +370,13 @@ SELECT '-', recorded_hour, symbol - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS token_prices_coin_gecko_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM imputed_prices p WHERE diff --git a/models/silver/prices/silver__token_prices_coin_market_cap_hourly.sql b/models/silver/prices/silver__token_prices_coin_market_cap_hourly.sql index 4025b57..f8d0040 100644 --- a/models/silver/prices/silver__token_prices_coin_market_cap_hourly.sql +++ b/models/silver/prices/silver__token_prices_coin_market_cap_hourly.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['recorded_hour::DATE'], tags = ['noncore'] ) }} @@ -340,7 +341,13 @@ SELECT '-', recorded_hour, id - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS token_prices_coin_market_cap_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM imputed_prices p WHERE