From 3225d392604fa75250ce345dfd68ed18a66b79a6 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Mon, 4 Aug 2025 17:02:29 -0400 Subject: [PATCH] upgrades for coinstore deprecations --- .../gold/core/core__ez_native_transfers.sql | 20 ++++- models/gold/core/core__fact_transfers.sql | 2 +- models/silver/core/silver__transfers.sql | 3 +- .../silver/core/silver__transfers_native.sql | 5 +- .../silver__transfers_native_fungible.sql | 78 +++++++++++++++++++ .../core/silver__transfers_native_vw.sql | 25 ++++++ models/silver/core/silver__transfers_vw.sql | 27 +++++++ .../transfers/test_silver__transfers_full.sql | 3 +- .../test_silver__transfers_recent.sql | 3 +- .../test_silver__transfers_native_full.sql | 3 +- .../test_silver__transfers_native_recent.sql | 3 +- .../defi/dex/silver__dex_swaps_thala_v0.sql | 4 +- .../nft/mints/silver__nft_mints_combined.sql | 20 ++++- .../silver/nft/mints/silver__nft_mints_v2.sql | 21 ++++- .../nft/sales/silver__nft_sales_combined.sql | 22 +++++- models/sources.yml | 5 ++ 16 files changed, 230 insertions(+), 14 deletions(-) create mode 100644 models/silver/core/silver__transfers_native_fungible.sql create mode 100644 models/silver/core/silver__transfers_native_vw.sql create mode 100644 models/silver/core/silver__transfers_vw.sql diff --git a/models/gold/core/core__ez_native_transfers.sql b/models/gold/core/core__ez_native_transfers.sql index c76ab9b..d19fa31 100644 --- a/models/gold/core/core__ez_native_transfers.sql +++ b/models/gold/core/core__ez_native_transfers.sql @@ -18,5 +18,23 @@ SELECT modified_timestamp FROM {{ ref( - 'silver__transfers_native' + 'silver__transfers_native_fungible' + ) }} +UNION ALL +SELECT + block_number, + block_timestamp, + tx_hash, + version, + success, + from_address, + to_address, + amount, + token_address, + transfers_native_id AS ez_native_transfers_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref( + 'silver__transfers_native_vw' ) }} diff --git a/models/gold/core/core__fact_transfers.sql b/models/gold/core/core__fact_transfers.sql index 742900c..d37e3cf 100644 --- a/models/gold/core/core__fact_transfers.sql +++ b/models/gold/core/core__fact_transfers.sql @@ -26,7 +26,7 @@ SELECT SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, FROM - {{ ref('silver__transfers') }} + {{ ref('silver__transfers_vw') }} WHERE amount <> 0 diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index ea81881..b6ba9b8 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -4,7 +4,8 @@ incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], - tags = ['core','full_test'] + tags = ['core','full_test'], + enabled = false ) }} WITH events AS ( diff --git a/models/silver/core/silver__transfers_native.sql b/models/silver/core/silver__transfers_native.sql index f05d6a5..bb2622a 100644 --- a/models/silver/core/silver__transfers_native.sql +++ b/models/silver/core/silver__transfers_native.sql @@ -5,7 +5,8 @@ merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash, version, from_address, to_address);", - tags = ['core','full_test'] + tags = ['core','full_test'], + enabled = false ) }} WITH xfer AS ( @@ -106,4 +107,4 @@ WHERE AND ( wth.event_index + 1 = dep.event_index OR reg.event_index + 1 = dep.event_index - ) \ No newline at end of file + ) diff --git a/models/silver/core/silver__transfers_native_fungible.sql b/models/silver/core/silver__transfers_native_fungible.sql new file mode 100644 index 0000000..d72c6de --- /dev/null +++ b/models/silver/core/silver__transfers_native_fungible.sql @@ -0,0 +1,78 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_hash','_transfer_key','block_timestamp::DATE'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE'], + tags = ['core','full_test'] +) }} +{# post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash, version, from_address, to_address);", #} +WITH xfer AS ( + + SELECT + block_number, + block_timestamp, + tx_hash, + version, + success, + event_index, + transfer_event, + owner_address account_address, + amount, + metadata_address AS token_address + FROM + {{ ref('silver__transfers_fungible') }} + WHERE + amount > 0 + AND metadata_address = '0xa' + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +wth AS ( + SELECT + * + FROM + xfer + WHERE + transfer_event = 'WithdrawEvent' +), +dep AS ( + SELECT + * + FROM + xfer + WHERE + transfer_event = 'DepositEvent' +) +SELECT + wth.block_number, + wth.block_timestamp, + wth.tx_hash, + wth.version, + wth.success, + wth.account_address AS from_address, + dep.account_address AS to_address, + wth.amount, + wth.token_address, + wth.event_index || ':' || dep.event_index AS _transfer_key, + {{ dbt_utils.generate_surrogate_key( + ['wth.tx_hash','wth.event_index','dep.event_index'] + ) }} AS transfers_native_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + wth + JOIN dep + ON wth.tx_hash = dep.tx_hash + AND wth.amount = dep.amount + AND wth.event_index + 1 = dep.event_index +WHERE + wth.account_address <> dep.account_address diff --git a/models/silver/core/silver__transfers_native_vw.sql b/models/silver/core/silver__transfers_native_vw.sql new file mode 100644 index 0000000..ecae4ae --- /dev/null +++ b/models/silver/core/silver__transfers_native_vw.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + version, + success, + from_address, + to_address, + amount, + token_address, + _transfer_key, + transfers_native_id, + inserted_timestamp, + modified_timestamp, + _inserted_timestamp, + _invocation_id +FROM + {{ source( + 'silver', + 'transfers_native' + ) }} diff --git a/models/silver/core/silver__transfers_vw.sql b/models/silver/core/silver__transfers_vw.sql new file mode 100644 index 0000000..80b8d2c --- /dev/null +++ b/models/silver/core/silver__transfers_vw.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + version, + success, + event_index, + creation_number, + transfer_event, + account_address, + amount, + token_inoken_address, + transfers_id, + inserted_timestamp, + modified_timestamp, + _inserted_timestamp, + _invocation_id +FROM + {{ source( + 'silver', + 'transfers' + ) }} + e diff --git a/models/silver/core/tests/transfers/test_silver__transfers_full.sql b/models/silver/core/tests/transfers/test_silver__transfers_full.sql index df0197c..253d6a6 100644 --- a/models/silver/core/tests/transfers/test_silver__transfers_full.sql +++ b/models/silver/core/tests/transfers/test_silver__transfers_full.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'view', - tags = ['full_test'] + tags = ['full_test'], + enabled = false ) }} SELECT diff --git a/models/silver/core/tests/transfers/test_silver__transfers_recent.sql b/models/silver/core/tests/transfers/test_silver__transfers_recent.sql index 32af55a..6e12a02 100644 --- a/models/silver/core/tests/transfers/test_silver__transfers_recent.sql +++ b/models/silver/core/tests/transfers/test_silver__transfers_recent.sql @@ -1,6 +1,7 @@ {{ config ( materialized = 'view', - tags = ['recent_test'] + tags = ['recent_test'], + enabled = false ) }} WITH last_3_days AS ( diff --git a/models/silver/core/tests/transfers_native/test_silver__transfers_native_full.sql b/models/silver/core/tests/transfers_native/test_silver__transfers_native_full.sql index 6237f4d..57404e0 100644 --- a/models/silver/core/tests/transfers_native/test_silver__transfers_native_full.sql +++ b/models/silver/core/tests/transfers_native/test_silver__transfers_native_full.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'view', - tags = ['full_test'] + tags = ['full_test'], + enabled = false ) }} SELECT diff --git a/models/silver/core/tests/transfers_native/test_silver__transfers_native_recent.sql b/models/silver/core/tests/transfers_native/test_silver__transfers_native_recent.sql index a12bce9..837bb6c 100644 --- a/models/silver/core/tests/transfers_native/test_silver__transfers_native_recent.sql +++ b/models/silver/core/tests/transfers_native/test_silver__transfers_native_recent.sql @@ -1,6 +1,7 @@ {{ config ( materialized = 'view', - tags = ['recent_test'] + tags = ['recent_test'], + enabled = false ) }} WITH last_3_days AS ( diff --git a/models/silver/defi/dex/silver__dex_swaps_thala_v0.sql b/models/silver/defi/dex/silver__dex_swaps_thala_v0.sql index 2038d6c..6d7a402 100644 --- a/models/silver/defi/dex/silver__dex_swaps_thala_v0.sql +++ b/models/silver/defi/dex/silver__dex_swaps_thala_v0.sql @@ -4,7 +4,7 @@ incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['modified_timestamp::DATE'], - tags = ['noncore'] + tags = ['noncore_retired'] ) }} WITH tx AS ( @@ -56,7 +56,7 @@ xfers AS ( token_address, transfer_event FROM - {{ ref('silver__transfers') }} + {{ ref('silver__transfers_vw') }} WHERE success AND block_timestamp :: DATE <= '2023-04-05' diff --git a/models/silver/nft/mints/silver__nft_mints_combined.sql b/models/silver/nft/mints/silver__nft_mints_combined.sql index 7e914a0..12a5c34 100644 --- a/models/silver/nft/mints/silver__nft_mints_combined.sql +++ b/models/silver/nft/mints/silver__nft_mints_combined.sql @@ -46,10 +46,28 @@ xfers AS ( event_index, token_address FROM - {{ ref('silver__transfers') }} + {{ ref('silver__transfers_vw') }} WHERE success +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} +) +{% endif %} +UNION ALL +SELECT + tx_hash, + event_index, + metadata_address AS token_address +FROM + {{ ref('silver__transfers_fungible') }} +WHERE + success + {% if is_incremental() %} AND _inserted_timestamp >= ( SELECT diff --git a/models/silver/nft/mints/silver__nft_mints_v2.sql b/models/silver/nft/mints/silver__nft_mints_v2.sql index 3b7447f..434e540 100644 --- a/models/silver/nft/mints/silver__nft_mints_v2.sql +++ b/models/silver/nft/mints/silver__nft_mints_v2.sql @@ -77,10 +77,29 @@ xfers AS ( transfer_event, token_address FROM - {{ ref('silver__transfers') }} + {{ ref('silver__transfers_vw') }} WHERE success +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} +) +{% endif %} +UNION ALL +SELECT + tx_hash, + owner_address AS account_address, + transfer_event, + metadata_address AS token_address +FROM + {{ ref('silver__transfers_fungible') }} +WHERE + success + {% if is_incremental() %} AND _inserted_timestamp >= ( SELECT diff --git a/models/silver/nft/sales/silver__nft_sales_combined.sql b/models/silver/nft/sales/silver__nft_sales_combined.sql index 262949b..9edb734 100644 --- a/models/silver/nft/sales/silver__nft_sales_combined.sql +++ b/models/silver/nft/sales/silver__nft_sales_combined.sql @@ -51,10 +51,30 @@ xfers AS ( event_index, transfer_event FROM - {{ ref('silver__transfers') }} + {{ ref('silver__transfers_vw') }} WHERE success +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} +) +{% endif %} +UNION ALL +SELECT + tx_hash, + owner_address AS account_address, + metadata_address AS token_address, + event_index, + transfer_event +FROM + {{ ref('silver__transfers_fungible') }} +WHERE + success + {% if is_incremental() %} AND _inserted_timestamp >= ( SELECT diff --git a/models/sources.yml b/models/sources.yml index cd4f5d8..160558f 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -25,6 +25,11 @@ sources: schema: bronze tables: - name: lq_blocks_txs + - name: silver + schema: silver + tables: + - name: transfers + - name: transfers_native - name: github_actions database: aptos schema: github_actions