diff --git a/dbt_project.yml b/dbt_project.yml index ecd9b4e..917b906 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -30,6 +30,7 @@ models: +persist_docs: relation: true columns: true + +on_schema_change: "append_new_columns" tests: +store_failures: true # all tests diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index 7412289..f726c1d 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -116,7 +116,13 @@ SELECT ) END ) AS test_failure_details, - SYSDATE() AS test_timestamp + SYSDATE() AS test_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['SYSDATE()'] + ) }} AS blocks_completeness_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM block_gen A LEFT JOIN source b diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql index 8cbc11b..0c00de6 100644 --- a/models/silver/_observability/silver_observability__transactions_completeness.sql +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -250,7 +250,13 @@ SELECT ORDER BY A.block_id ) AS test_failure_details, - SYSDATE() AS test_timestamp + SYSDATE() AS test_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['SYSDATE()'] + ) }} AS transactions_completeness_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM bronze_api A LEFT JOIN bronze_count b diff --git a/models/silver/core/silver__blockchain.sql b/models/silver/core/silver__blockchain.sql index 2b3f2ce..4564e17 100644 --- a/models/silver/core/silver__blockchain.sql +++ b/models/silver/core/silver__blockchain.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "block_id", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], ) }} @@ -59,7 +60,13 @@ SELECT block_header, block_id AS block_id_object, num_txs, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['height'] + ) }} AS blockchain_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM fin qualify(ROW_NUMBER() over(PARTITION BY height ORDER BY diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index c3492b7..4dbcb8e 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -37,12 +37,18 @@ SELECT DATA :result :block :header, DATA :block :header ) AS header, - _inserted_timestamp, concat_ws( '-', chain_id, block_id - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['chain_id','block_id'] + ) }} AS blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__blocks') }} WHERE diff --git a/models/silver/core/silver__daily_balances.sql b/models/silver/core/silver__daily_balances.sql index 4b3c5f6..802e5f9 100644 --- a/models/silver/core/silver__daily_balances.sql +++ b/models/silver/core/silver__daily_balances.sql @@ -172,6 +172,12 @@ SELECT balance_type ORDER BY DATE ASC rows unbounded preceding - ) AS balance + ) AS balance, + {{ dbt_utils.generate_surrogate_key( + ['date','address','balance_type', 'currency'] + ) }} AS daily_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM balance_temp diff --git a/models/silver/core/silver__link_events.sql b/models/silver/core/silver__link_events.sql index aa769e0..f6d2f5a 100644 --- a/models/silver/core/silver__link_events.sql +++ b/models/silver/core/silver__link_events.sql @@ -72,7 +72,13 @@ SELECT destination_chain, destination_address, asset, + j, + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} AS link_events_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, _inserted_timestamp, - j + '{{ invocation_id }}' AS _invocation_id FROM fin diff --git a/models/silver/core/silver__msg_attributes.sql b/models/silver/core/silver__msg_attributes.sql index 218c0bc..efbdf4a 100644 --- a/models/silver/core/silver__msg_attributes.sql +++ b/models/silver/core/silver__msg_attributes.sql @@ -27,7 +27,13 @@ SELECT msg_index, attribute_index ) AS _unique_key, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['tx_id','msg_index','attribute_index'] + ) }} AS msg_attributes_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('silver__msgs') }} A, LATERAL FLATTEN( @@ -39,7 +45,7 @@ FROM WHERE _inserted_timestamp :: DATE >= ( SELECT - MAX(_inserted_timestamp) :: DATE - 2 + MAX(_inserted_timestamp) :: DATE - 2 FROM {{ this }} ) diff --git a/models/silver/core/silver__msgs.sql b/models/silver/core/silver__msgs.sql index 4746b41..2c43d20 100644 --- a/models/silver/core/silver__msgs.sql +++ b/models/silver/core/silver__msgs.sql @@ -118,7 +118,13 @@ FINAL AS ( A.tx_id, A.msg_index ) AS _unique_key, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['a.tx_id','a.msg_index'] + ) }} AS msgs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM base A LEFT JOIN GROUPING b diff --git a/models/silver/core/silver__staked_balances.sql b/models/silver/core/silver__staked_balances.sql index e093f17..31ed011 100644 --- a/models/silver/core/silver__staked_balances.sql +++ b/models/silver/core/silver__staked_balances.sql @@ -59,6 +59,12 @@ SELECT ORDER BY block_timestamp ASC rows unbounded preceding ) AS balance, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['block_id','address','currency'] + ) }} AS staked_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM all_staked diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index 5e2c022..6042985 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -51,7 +51,13 @@ SELECT tx_code, msgs, tx_log :: STRING AS tx_log, - b._inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} AS transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + b._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM base_table b LEFT OUTER JOIN {{ ref('silver__blocks') }} diff --git a/models/silver/core/silver__transfer_details.sql b/models/silver/core/silver__transfer_details.sql index ede56f9..85f80df 100644 --- a/models/silver/core/silver__transfer_details.sql +++ b/models/silver/core/silver__transfer_details.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ["tx_id","msg_group","msg_sub_group"], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE' ) }} @@ -202,6 +203,12 @@ SELECT tokenAddress AS token_address, txID AS txid, confHeight AS conf_height, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['tx_id','msg_group','msg_sub_group'] + ) }} AS transfer_details_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM fin diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index b392cce..6624c5b 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ['tx_id','msg_index'], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'] ) }} @@ -146,7 +147,13 @@ SELECT A.msg_index, b.destination_address AS foreign_address, b.destination_chain AS foreign_chain, - A._inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['a.tx_id','a.msg_index'] + ) }} AS transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM fin A LEFT JOIN decimals d diff --git a/models/silver/core/silver__transfers_ExecutePendingTransfers.sql b/models/silver/core/silver__transfers_ExecutePendingTransfers.sql index cd0d1bd..fb27e79 100644 --- a/models/silver/core/silver__transfers_ExecutePendingTransfers.sql +++ b/models/silver/core/silver__transfers_ExecutePendingTransfers.sql @@ -189,8 +189,14 @@ SELECT transfer_id, snd.deposit_address AS foreign_address, snd.source_chain AS foreign_chain, + _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','msg_index'] + ) }} AS transfer_executependingtransfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, _inserted_timestamp, - _unique_key + '{{ invocation_id }}' AS _invocation_id FROM txs_final i LEFT JOIN decimals d diff --git a/models/silver/core/silver__transfers_ibc.sql b/models/silver/core/silver__transfers_ibc.sql index b118bf4..30f13b3 100644 --- a/models/silver/core/silver__transfers_ibc.sql +++ b/models/silver/core/silver__transfers_ibc.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ['tx_id','msg_index'], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'] ) }} @@ -290,7 +291,13 @@ SELECT currency, b.destination_address AS foreign_address, b.destination_chain AS foreign_chain, - i._inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['i.tx_id','i.msg_index'] + ) }} AS transfers_ibc_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM new_fin i LEFT JOIN decimals d diff --git a/models/silver/core/silver__transfers_ibc_packets.sql b/models/silver/core/silver__transfers_ibc_packets.sql index d3cf81f..ca14bed 100644 --- a/models/silver/core/silver__transfers_ibc_packets.sql +++ b/models/silver/core/silver__transfers_ibc_packets.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ['tx_id','msg_index'], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'] ) }} @@ -87,6 +88,12 @@ SELECT packet_sequence, packet_src_channel, packet_dst_channel, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['tx_id','msg_index'] + ) }} AS transfers_ibc_packets_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM all_data diff --git a/models/silver/defi/satellite/silver__satellite_arbitrum.sql b/models/silver/defi/satellite/silver__satellite_arbitrum.sql index 50b5e84..33a7fa0 100644 --- a/models/silver/defi/satellite/silver__satellite_arbitrum.sql +++ b/models/silver/defi/satellite/silver__satellite_arbitrum.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -117,7 +118,13 @@ SELECT #} tokenaddress AS token_address, raw_amount, - depositaddress AS deposit_address + depositaddress AS deposit_address, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS satellite_arbitrum_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM links JOIN labeled_transfer_amount USING (depositAddress) diff --git a/models/silver/defi/satellite/silver__satellite_avalanche.sql b/models/silver/defi/satellite/silver__satellite_avalanche.sql index 61118b3..a2f4d1f 100644 --- a/models/silver/defi/satellite/silver__satellite_avalanche.sql +++ b/models/silver/defi/satellite/silver__satellite_avalanche.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -117,7 +118,13 @@ SELECT #} tokenaddress AS token_address, raw_amount, - depositaddress AS deposit_address + depositaddress AS deposit_address, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS satellite_avalanche_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM links JOIN labeled_transfer_amount USING (depositAddress) diff --git a/models/silver/defi/satellite/silver__satellite_axelar.sql b/models/silver/defi/satellite/silver__satellite_axelar.sql index 0d1abda..d4fc6ad 100644 --- a/models/silver/defi/satellite/silver__satellite_axelar.sql +++ b/models/silver/defi/satellite/silver__satellite_axelar.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -86,7 +87,13 @@ SELECT '-', A.tx_hash, A.msg_index - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS satellite_axelar_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM xfer_base A LEFT JOIN transfer_det b diff --git a/models/silver/defi/satellite/silver__satellite_bsc.sql b/models/silver/defi/satellite/silver__satellite_bsc.sql index 34cead9..5b79651 100644 --- a/models/silver/defi/satellite/silver__satellite_bsc.sql +++ b/models/silver/defi/satellite/silver__satellite_bsc.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -117,7 +118,13 @@ SELECT #} tokenaddress AS token_address, raw_amount, - depositaddress AS deposit_address + depositaddress AS deposit_address, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS satellite_bsc_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM links JOIN labeled_transfer_amount USING (depositAddress) diff --git a/models/silver/defi/satellite/silver__satellite_ethereum.sql b/models/silver/defi/satellite/silver__satellite_ethereum.sql index 505198d..b13643a 100644 --- a/models/silver/defi/satellite/silver__satellite_ethereum.sql +++ b/models/silver/defi/satellite/silver__satellite_ethereum.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -117,7 +118,13 @@ SELECT #} tokenaddress AS token_address, raw_amount, - depositaddress AS deposit_address + depositaddress AS deposit_address, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS satellite_ethereum_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM links JOIN labeled_transfer_amount USING (depositAddress) diff --git a/models/silver/defi/satellite/silver__satellite_polygon.sql b/models/silver/defi/satellite/silver__satellite_polygon.sql index 34ff9d2..f027c05 100644 --- a/models/silver/defi/satellite/silver__satellite_polygon.sql +++ b/models/silver/defi/satellite/silver__satellite_polygon.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -117,7 +118,13 @@ SELECT #} tokenaddress AS token_address, raw_amount, - depositaddress AS deposit_address + depositaddress AS deposit_address, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS satellite_polygon_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM links JOIN labeled_transfer_amount USING (depositAddress) diff --git a/models/silver/defi/silver__staking.sql b/models/silver/defi/silver__staking.sql index 997513f..9ad9310 100644 --- a/models/silver/defi/silver__staking.sql +++ b/models/silver/defi/silver__staking.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'] ) }} @@ -284,7 +285,6 @@ SELECT A.redelegate_source_validator_address, A.completion_time :: datetime completion_time, A.msg_group, - A._inserted_timestamp, concat_ws( '-', A.tx_id, @@ -293,7 +293,14 @@ SELECT currency, delegator_address, validator_address - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['A.tx_id','A.msg_group','A.action','currency','A.delegator_address','A.validator_address'] + ) }} AS staking_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM prefinal A JOIN LATERAL SPLIT_TO_TABLE( diff --git a/models/silver/defi/silver__staking_rewards.sql b/models/silver/defi/silver__staking_rewards.sql index 794edd4..77cdced 100644 --- a/models/silver/defi/silver__staking_rewards.sql +++ b/models/silver/defi/silver__staking_rewards.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'] ) }} @@ -406,7 +407,6 @@ SELECT ELSE 'uaxl' END AS currency, A.validator_address, - b._inserted_timestamp, concat_ws( '-', A.tx_id, @@ -415,7 +415,14 @@ SELECT currency, A.delegator_address, A.validator_address - ) AS _unique_key + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['A.tx_id','A.msg_group','A.action','currency','A.delegator_address','A.validator_address'] + ) }} AS staking_rewards_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + b._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM combo_all A JOIN LATERAL SPLIT_TO_TABLE( @@ -437,4 +444,5 @@ GROUP BY A.delegator_address, currency, A.validator_address, - b._inserted_timestamp + b._inserted_timestamp, + staking_rewards_id diff --git a/models/silver/defi/silver__validator_commission.sql b/models/silver/defi/silver__validator_commission.sql index 48cfec4..ab04b76 100644 --- a/models/silver/defi/silver__validator_commission.sql +++ b/models/silver/defi/silver__validator_commission.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_id", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'] ) }} @@ -117,7 +118,13 @@ SELECT 'uaxl' AS currency, A.validator_address_operator, A.validator_address_reward, - b._inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['a.tx_id'] + ) }} AS staking_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + b._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM combo A JOIN block_tx_inserted b diff --git a/models/silver/defi/silver__validators.sql b/models/silver/defi/silver__validators.sql index 39e70f8..ed6b08f 100644 --- a/models/silver/defi/silver__validators.sql +++ b/models/silver/defi/silver__validators.sql @@ -27,6 +27,12 @@ SELECT address, creator, blockchain - ) AS unique_key + ) AS unique_key, + {{ dbt_utils.generate_surrogate_key( + ['address','creator','blockchain'] + ) }} AS validators_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze_api__get_validator_metadata') }} diff --git a/models/silver/defi/squid/silver__squid_arbitrum.sql b/models/silver/defi/squid/silver__squid_arbitrum.sql index 49165e3..9998e7c 100644 --- a/models/silver/defi/squid/silver__squid_arbitrum.sql +++ b/models/silver/defi/squid/silver__squid_arbitrum.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -191,6 +192,11 @@ nonevm_fix_data AS ( ) AS token_symbol, LOWER(REGEXP_REPLACE(destination_chain, '[^a-zA-Z0-9]+')) AS destination_chain, LOWER(receiver) AS receiver, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS squid_arbitrum_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM arb_result diff --git a/models/silver/defi/squid/silver__squid_avalanche.sql b/models/silver/defi/squid/silver__squid_avalanche.sql index bee3a37..807ad99 100644 --- a/models/silver/defi/squid/silver__squid_avalanche.sql +++ b/models/silver/defi/squid/silver__squid_avalanche.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -194,6 +195,11 @@ nonevm_fix_data AS ( ELSE LOWER(REGEXP_REPLACE(destination_chain, '[^a-zA-Z0-9]+')) END AS destination_chain, LOWER(receiver) AS receiver, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS squid_avalanche_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM arb_result A diff --git a/models/silver/defi/squid/silver__squid_base.sql b/models/silver/defi/squid/silver__squid_base.sql index f8fb32b..2bc50b5 100644 --- a/models/silver/defi/squid/silver__squid_base.sql +++ b/models/silver/defi/squid/silver__squid_base.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -191,6 +192,11 @@ nonevm_fix_data AS ( ) AS token_symbol, LOWER(REGEXP_REPLACE(destination_chain, '[^a-zA-Z0-9]+')) AS destination_chain, LOWER(receiver) AS receiver, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS squid_base_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM base_result diff --git a/models/silver/defi/squid/silver__squid_bsc.sql b/models/silver/defi/squid/silver__squid_bsc.sql index e707b8b..7e06e31 100644 --- a/models/silver/defi/squid/silver__squid_bsc.sql +++ b/models/silver/defi/squid/silver__squid_bsc.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -195,6 +196,11 @@ nonevm_fix_data AS ( ) AS token_symbol, LOWER(REGEXP_REPLACE(destination_chain, '[^a-zA-Z0-9]+')) AS destination_chain, LOWER(receiver) AS receiver, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS squid_bsc_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM arb_result A diff --git a/models/silver/defi/squid/silver__squid_ethereum.sql b/models/silver/defi/squid/silver__squid_ethereum.sql index 80eb992..2c6409c 100644 --- a/models/silver/defi/squid/silver__squid_ethereum.sql +++ b/models/silver/defi/squid/silver__squid_ethereum.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -111,7 +112,12 @@ SELECT ), sender ) AS receiver, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['a.tx_hash'] + ) }} AS squid_ethereum_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM dec_logs_base A JOIN squid_to_gateway b diff --git a/models/silver/defi/squid/silver__squid_optimism.sql b/models/silver/defi/squid/silver__squid_optimism.sql index b14960e..457a261 100644 --- a/models/silver/defi/squid/silver__squid_optimism.sql +++ b/models/silver/defi/squid/silver__squid_optimism.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -191,6 +192,11 @@ nonevm_fix_data AS ( ) AS token_symbol, LOWER(REGEXP_REPLACE(destination_chain, '[^a-zA-Z0-9]+')) AS destination_chain, LOWER(receiver) AS receiver, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS squid_optimism_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM op_result diff --git a/models/silver/defi/squid/silver__squid_polygon.sql b/models/silver/defi/squid/silver__squid_polygon.sql index fdccd54..7935813 100644 --- a/models/silver/defi/squid/silver__squid_polygon.sql +++ b/models/silver/defi/squid/silver__squid_polygon.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_hash", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'block_timestamp::DATE', ) }} @@ -195,6 +196,11 @@ nonevm_fix_data AS ( ) AS token_symbol, LOWER(REGEXP_REPLACE(destination_chain, '[^a-zA-Z0-9]+')) AS destination_chain, LOWER(receiver) AS receiver, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS squid_polygon_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM arb_result A diff --git a/models/silver/gov/silver__governance_proposal_deposits.sql b/models/silver/gov/silver__governance_proposal_deposits.sql index cfa9922..5d261b2 100644 --- a/models/silver/gov/silver__governance_proposal_deposits.sql +++ b/models/silver/gov/silver__governance_proposal_deposits.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_id", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'] ) }} @@ -104,7 +105,13 @@ SELECT p.proposal_id :: NUMBER AS proposal_id, v.amount :: NUMBER AS amount, v.currency, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['p.tx_id'] + ) }} AS governance_proposal_deposits_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM deposit_value v INNER JOIN proposal_ids p diff --git a/models/silver/gov/silver__governance_submit_proposal.sql b/models/silver/gov/silver__governance_submit_proposal.sql index 559b9af..272cbdb 100644 --- a/models/silver/gov/silver__governance_submit_proposal.sql +++ b/models/silver/gov/silver__governance_submit_proposal.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "tx_id", incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'] ) }} @@ -86,7 +87,13 @@ SELECT proposer, p.proposal_id :: NUMBER AS proposal_id, y.proposal_type, - _inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['p.tx_id'] + ) }} AS governance_submit_proposal_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM proposal_ids p INNER JOIN proposal_type y diff --git a/models/silver/gov/silver__governance_votes.sql b/models/silver/gov/silver__governance_votes.sql index 89ed4bd..1a9d1eb 100644 --- a/models/silver/gov/silver__governance_votes.sql +++ b/models/silver/gov/silver__governance_votes.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = ['tx_id','proposal_id','voter'], incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'] ) }} @@ -93,7 +94,13 @@ SELECT ELSE A.vote_option :: INT END AS vote_option, A.vote_weight, - A._inserted_timestamp + {{ dbt_utils.generate_surrogate_key( + ['a.tx_id','a.proposal_id','b.voter'] + ) }} AS governance_votes_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM fin A LEFT JOIN (