From ecaacc008e44e81935af3d4c497ecb3741fa2b50 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Fri, 13 Sep 2024 20:35:38 +0300 Subject: [PATCH] breakdown --- models/gold/core/core__ez_token_transfers.sql | 2 +- .../curated/silver__token_transfers.sql | 171 ----------------- .../silver__token_transfer_native.sql} | 35 +++- .../silver__token_transfer_base.sql | 10 +- ...ver__token_transfer_ft_transfers_event.sql | 28 ++- ...er__token_transfer_ft_transfers_method.sql | 14 +- .../silver__token_transfer_liquidity.sql | 16 +- .../silver__token_transfer_mints.sql | 15 +- ...er__token_transfer_non_native_complete.sql | 175 ++++++++++++++++++ .../silver__token_transfer_orders.sql | 16 +- .../silver__token_transfers_complete.sql | 92 +++++++++ .../silver__token_transfers_complete.yml} | 2 +- 12 files changed, 371 insertions(+), 205 deletions(-) delete mode 100644 models/silver/curated/silver__token_transfers.sql rename models/silver/curated/token_transfers/{silver__token_transfer_base_native.sql => native/silver__token_transfer_native.sql} (64%) rename models/silver/curated/token_transfers/{ => non_native}/silver__token_transfer_base.sql (83%) rename models/silver/curated/token_transfers/{ => non_native}/silver__token_transfer_ft_transfers_event.sql (79%) rename models/silver/curated/token_transfers/{ => non_native}/silver__token_transfer_ft_transfers_method.sql (82%) rename models/silver/curated/token_transfers/{ => non_native}/silver__token_transfer_liquidity.sql (81%) rename models/silver/curated/token_transfers/{ => non_native}/silver__token_transfer_mints.sql (85%) create mode 100644 models/silver/curated/token_transfers/non_native/silver__token_transfer_non_native_complete.sql rename models/silver/curated/token_transfers/{ => non_native}/silver__token_transfer_orders.sql (82%) create mode 100644 models/silver/curated/token_transfers/silver__token_transfers_complete.sql rename models/silver/curated/{silver__token_transfers.yml => token_transfers/silver__token_transfers_complete.yml} (97%) diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index 87b975c..7324c4c 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -60,7 +60,7 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__token_transfers') }} + {{ ref('silver__token_transfers_complete') }} t LEFT JOIN hourly_prices p ON t.contract_address = p.token_address diff --git a/models/silver/curated/silver__token_transfers.sql b/models/silver/curated/silver__token_transfers.sql deleted file mode 100644 index 61a7aa4..0000000 --- a/models/silver/curated/silver__token_transfers.sql +++ /dev/null @@ -1,171 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], - merge_exclude_columns = ["inserted_timestamp"], - cluster_by = ['block_timestamp::DATE','modified_timestamp::Date'], - unique_key = 'transfers_id', - incremental_strategy = 'merge', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,action_id,contract_address,from_address,to_address);", - tags = ['curated','scheduled_non_core'] -) }} -{# Note - multisource model #} --- Curation Challenge - 'https://flipsidecrypto.xyz/Hossein/transfer-sector-of-near-curation-challenge-zgM44F' - ------------------------------- NEAR Tokens (NEP 141) -------------------------------- -WITH orders_final AS ( - SELECT - * - FROM - {{ref('silver__token_transfer_orders')}} -), -add_liquidity AS ( - SELECT - * - FROM - {{ref('silver__token_transfer_liquidity')}} -), -ft_transfers_method AS ( - SELECT - * - FROM - {{ref('silver__token_transfer_ft_transfers_method')}} -), -ft_transfers_event AS ( - SELECT - * - FROM - {{ref('silver__token_transfer_ft_transfers_event')}} -), -ft_mints_final AS ( - SELECT - * - FROM - {{ref('silver__token_transfer_mints')}} -), -nep_transfers AS ( - SELECT - * - FROM - ft_transfers_method - UNION ALL - SELECT - * - FROM - ft_transfers_event - UNION ALL - SELECT - block_id, - block_timestamp, - tx_hash, - action_id, - contract_address, - from_address, - to_address, - amount_unadjusted, - memo, - rn, - _inserted_timestamp, - _modified_timestamp, - _partition_by_block_number - FROM - ft_mints_final - UNION ALL - SELECT - * - FROM - orders_final - UNION ALL - SELECT - * - FROM - add_liquidity -), -native_transfers AS ( - SELECT - * - FROM - {{ref('silver__token_transfer_base_native')}} -), ------------------------------- MODELS -------------------------------- -native_final AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - action_id, - 'wrap.near' AS contract_address, - from_address :: STRING, - to_address :: STRING, - NULL AS memo, - '0' AS rn, - 'native' AS transfer_type, - amount_unadjusted :: STRING AS amount_raw, - amount_unadjusted :: FLOAT AS amount_raw_precise, - _inserted_timestamp, - _modified_timestamp, - _partition_by_block_number - FROM - native_transfers -), -nep_final AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - action_id, - contract_address, - from_address, - to_address, - memo, - rn :: STRING AS rn, - 'nep141' AS transfer_type, - amount_unadjusted :: STRING AS amount_raw, - amount_unadjusted :: FLOAT AS amount_raw_precise, - _inserted_timestamp, - _modified_timestamp, - _partition_by_block_number - FROM - nep_transfers -), ------------------------------- FINAL -------------------------------- -transfer_union AS ( - SELECT - * - FROM - nep_final - UNION ALL - SELECT - * - FROM - native_final -), -FINAL AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - action_id, - rn, - contract_address, - from_address, - to_address, - memo, - amount_raw, - amount_raw_precise, - transfer_type, - _inserted_timestamp, - _modified_timestamp, - _partition_by_block_number - FROM - transfer_union -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] - ) }} AS transfers_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL diff --git a/models/silver/curated/token_transfers/silver__token_transfer_base_native.sql b/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql similarity index 64% rename from models/silver/curated/token_transfers/silver__token_transfer_base_native.sql rename to models/silver/curated/token_transfers/native/silver__token_transfer_native.sql index 4f72efe..039d546 100644 --- a/models/silver/curated/token_transfers/silver__token_transfer_base_native.sql +++ b/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql @@ -3,7 +3,7 @@ incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_modified_timestamp::Date'], - unique_key = 'action_id', + unique_key = 'transfers_native_id', incremental_strategy = 'merge', tags = ['curated','scheduled_non_core'] ) }} @@ -37,7 +37,34 @@ WITH native_transfers AS ( {{ this }} ) {% endif %} -) +), +FINAL AS ( SELECT - * -FROM native_transfers \ No newline at end of file + block_id, + block_timestamp, + tx_hash, + action_id, + 'wrap.near' AS contract_address, + from_address :: STRING AS from_address, + to_address :: STRING AS to_address, + NULL AS memo, + '0' AS rn, + 'native' AS transfer_type, + amount_unadjusted :: STRING AS amount_raw, + amount_unadjusted :: FLOAT AS amount_raw_precise, + _inserted_timestamp, + _modified_timestamp, + _partition_by_block_number +FROM + native_transfers +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['action_id'] + ) }} AS transfers_native_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL \ No newline at end of file diff --git a/models/silver/curated/token_transfers/silver__token_transfer_base.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_base.sql similarity index 83% rename from models/silver/curated/token_transfers/silver__token_transfer_base.sql rename to models/silver/curated/token_transfers/non_native/silver__token_transfer_base.sql index 44b89d0..bb90bbf 100644 --- a/models/silver/curated/token_transfers/silver__token_transfer_base.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_base.sql @@ -3,7 +3,7 @@ incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_modified_timestamp::Date'], - unique_key = 'action_id', + unique_key = 'transfers_base_id', incremental_strategy = 'merge', tags = ['curated','scheduled_non_core'] ) }} @@ -46,6 +46,12 @@ WITH nep141 AS ( {% endif %} ) SELECT - * + *, + {{ dbt_utils.generate_surrogate_key( + ['action_id'] + ) }} AS transfers_base_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM nep141 \ No newline at end of file diff --git a/models/silver/curated/token_transfers/silver__token_transfer_ft_transfers_event.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql similarity index 79% rename from models/silver/curated/token_transfers/silver__token_transfer_ft_transfers_event.sql rename to models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql index bc6b523..330e08a 100644 --- a/models/silver/curated/token_transfers/silver__token_transfer_ft_transfers_event.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql @@ -3,7 +3,7 @@ incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_modified_timestamp::Date'], - unique_key = 'action_id', + unique_key = 'transfers_event_id', incremental_strategy = 'merge', tags = ['curated','scheduled_non_core'] ) }} @@ -24,17 +24,19 @@ WITH actions_events AS ( logs, receipt_succeeded, _inserted_timestamp, - _modified_timestamp, + modified_timestamp as _modified_timestamp, _partition_by_block_number FROM {{ ref('silver__token_transfer_base') }} - {% if is_incremental() %} - WHERE _modified_timestamp >= ( - SELECT - MAX(_modified_timestamp) - FROM - {{ this }} - ) + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} + WHERE _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) {% endif %} ), ft_transfers_event AS ( @@ -89,6 +91,12 @@ ft_transfers_final AS ( amount_unadjusted > 0 ) SELECT - * + *, + {{ dbt_utils.generate_surrogate_key( + ['action_id'] + ) }} AS transfers_event_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM ft_transfers_final diff --git a/models/silver/curated/token_transfers/silver__token_transfer_ft_transfers_method.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql similarity index 82% rename from models/silver/curated/token_transfers/silver__token_transfer_ft_transfers_method.sql rename to models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql index 080b578..fd33b99 100644 --- a/models/silver/curated/token_transfers/silver__token_transfer_ft_transfers_method.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql @@ -25,11 +25,13 @@ WITH actions_events AS ( logs, receipt_succeeded, _inserted_timestamp, - _modified_timestamp, + modified_timestamp as _modified_timestamp, _partition_by_block_number FROM {{ ref('silver__token_transfer_base') }} - {% if is_incremental() %} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} WHERE _modified_timestamp >= ( SELECT MAX(_modified_timestamp) @@ -82,6 +84,12 @@ ft_transfers_method AS ( AND amount_unadjusted IS NOT NULL ) SELECT - * + *, + {{ dbt_utils.generate_surrogate_key( + ['action_id'] + ) }} AS transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM ft_transfers_method \ No newline at end of file diff --git a/models/silver/curated/token_transfers/silver__token_transfer_liquidity.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql similarity index 81% rename from models/silver/curated/token_transfers/silver__token_transfer_liquidity.sql rename to models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql index f8078e4..6d5dbf9 100644 --- a/models/silver/curated/token_transfers/silver__token_transfer_liquidity.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql @@ -3,7 +3,7 @@ incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_modified_timestamp::Date'], - unique_key = 'action_id', + unique_key = 'transfers_liquidity_id', incremental_strategy = 'merge', tags = ['curated','scheduled_non_core'] ) }} @@ -24,11 +24,13 @@ WITH actions_events AS ( logs, receipt_succeeded, _inserted_timestamp, - _modified_timestamp, + modified_timestamp as _modified_timestamp, _partition_by_block_number FROM {{ ref('silver__token_transfer_base') }} - {% if is_incremental() %} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} WHERE _modified_timestamp >= ( SELECT MAX(_modified_timestamp) @@ -81,6 +83,12 @@ add_liquidity AS ( logs [0] LIKE 'Liquidity added [%minted % shares' ) SELECT - * + *, + {{ dbt_utils.generate_surrogate_key( + ['action_id'] + ) }} AS transfers_liquidity_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM add_liquidity \ No newline at end of file diff --git a/models/silver/curated/token_transfers/silver__token_transfer_mints.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql similarity index 85% rename from models/silver/curated/token_transfers/silver__token_transfer_mints.sql rename to models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql index 7f09774..553ca9c 100644 --- a/models/silver/curated/token_transfers/silver__token_transfer_mints.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql @@ -23,11 +23,13 @@ WITH actions_events AS ( logs, receipt_succeeded, _inserted_timestamp, - _modified_timestamp, + modified_timestamp as _modified_timestamp, _partition_by_block_number FROM {{ ref('silver__token_transfer_base') }} - {% if is_incremental() %} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} WHERE _modified_timestamp >= ( SELECT MAX(_modified_timestamp) @@ -89,10 +91,13 @@ ft_mints_final AS ( amount_unadjusted > 0 ) SELECT - {{ dbt_utils.generate_surrogate_key( + *, + {{ dbt_utils.generate_surrogate_key( ['action_id','rn'] - ) }} AS mint_id, - * + )}} AS mint_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM ft_mints_final WHERE diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_non_native_complete.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_non_native_complete.sql new file mode 100644 index 0000000..c7a5e38 --- /dev/null +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_non_native_complete.sql @@ -0,0 +1,175 @@ +{{ config( + materialized = 'incremental', + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','_modified_timestamp::Date'], + unique_key = 'transfers_non_native_id', + incremental_strategy = 'merge', + tags = ['curated','scheduled_non_core'] +) }} +WITH nep_transfers AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + action_id, + contract_address, + from_address, + to_address, + amount_unadjusted, + memo, + rn, + _inserted_timestamp, + modified_timestamp as _modified_timestamp, + _partition_by_block_number + FROM + {{ref('silver__token_transfer_ft_transfers_method')}} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} + WHERE _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + UNION ALL + SELECT + block_id, + block_timestamp, + tx_hash, + action_id, + contract_address, + from_address, + to_address, + amount_unadjusted, + memo, + rn, + _inserted_timestamp, + modified_timestamp as _modified_timestamp, + _partition_by_block_number + FROM + {{ref('silver__token_transfer_ft_transfers_event')}} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} + WHERE _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + UNION ALL + SELECT + block_id, + block_timestamp, + tx_hash, + action_id, + contract_address, + from_address, + to_address, + amount_unadjusted, + memo, + rn, + _inserted_timestamp, + modified_timestamp as _modified_timestamp, + _partition_by_block_number + FROM + {{ref('silver__token_transfer_mints')}} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} + WHERE _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + UNION ALL + SELECT + block_id, + block_timestamp, + tx_hash, + action_id, + contract_address, + from_address, + to_address, + amount_unadjusted, + memo, + rn, + _inserted_timestamp, + modified_timestamp as _modified_timestamp, + _partition_by_block_number + FROM + {{ref('silver__token_transfer_orders')}} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} + WHERE _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + UNION ALL + SELECT + block_id, + block_timestamp, + tx_hash, + action_id, + contract_address, + from_address, + to_address, + amount_unadjusted, + memo, + rn, + _inserted_timestamp, + modified_timestamp as _modified_timestamp, + _partition_by_block_number + FROM + {{ref('silver__token_transfer_liquidity')}} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} + WHERE _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) + {% endif %} +), +FINAL AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + action_id, + contract_address, + from_address, + to_address, + memo, + rn :: STRING AS rn, + 'nep141' AS transfer_type, + amount_unadjusted :: STRING AS amount_raw, + amount_unadjusted :: FLOAT AS amount_raw_precise, + _inserted_timestamp, + _modified_timestamp, + _partition_by_block_number + FROM + nep_transfers +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['action_id'] + ) }} AS transfers_non_native_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL \ No newline at end of file diff --git a/models/silver/curated/token_transfers/silver__token_transfer_orders.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql similarity index 82% rename from models/silver/curated/token_transfers/silver__token_transfer_orders.sql rename to models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql index f69da33..eac15a5 100644 --- a/models/silver/curated/token_transfers/silver__token_transfer_orders.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql @@ -3,7 +3,7 @@ incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_modified_timestamp::Date'], - unique_key = 'action_id', + unique_key = 'transfers_orders_id', incremental_strategy = 'merge', tags = ['curated','scheduled_non_core'] ) }} @@ -23,11 +23,13 @@ WITH actions_events AS ( logs, receipt_succeeded, _inserted_timestamp, - _modified_timestamp, + modified_timestamp as _modified_timestamp, _partition_by_block_number FROM {{ ref('silver__token_transfer_base') }} - {% if is_incremental() %} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} WHERE _modified_timestamp >= ( SELECT MAX(_modified_timestamp) @@ -83,6 +85,12 @@ orders_final AS ( amount_unadjusted > 0 ) SELECT - * + *, + {{ dbt_utils.generate_surrogate_key( + ['action_id'] + ) }} AS transfers_orders_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM orders_final \ No newline at end of file diff --git a/models/silver/curated/token_transfers/silver__token_transfers_complete.sql b/models/silver/curated/token_transfers/silver__token_transfers_complete.sql new file mode 100644 index 0000000..2f0e65f --- /dev/null +++ b/models/silver/curated/token_transfers/silver__token_transfers_complete.sql @@ -0,0 +1,92 @@ +{{ config( + materialized = 'incremental', + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','modified_timestamp::Date'], + unique_key = 'transfers_id', + incremental_strategy = 'merge', + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,action_id,contract_address,from_address,to_address);", + tags = ['curated','scheduled_non_core'] +) }} + +WITH native_transfers AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + action_id, + rn, + contract_address, + from_address, + to_address, + memo, + amount_raw, + amount_raw_precise, + transfer_type, + _inserted_timestamp, + modified_timestamp as _modified_timestamp, + _partition_by_block_number + FROM + {{ref('silver__token_transfer_native')}} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} + WHERE _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) + {% endif %} +), +non_native AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + action_id, + rn, + contract_address, + from_address, + to_address, + memo, + amount_raw, + amount_raw_precise, + transfer_type, + _inserted_timestamp, + modified_timestamp as _modified_timestamp, + _partition_by_block_number + FROM + {{ref('silver__token_transfer_non_native_complete')}} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} + WHERE _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) + {% endif %} +), +FINAL AS ( + SELECT + * + FROM + non_native + UNION ALL + SELECT + * + FROM + native_transfers +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] + ) }} AS transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL diff --git a/models/silver/curated/silver__token_transfers.yml b/models/silver/curated/token_transfers/silver__token_transfers_complete.yml similarity index 97% rename from models/silver/curated/silver__token_transfers.yml rename to models/silver/curated/token_transfers/silver__token_transfers_complete.yml index 35635d7..5c1a9ac 100644 --- a/models/silver/curated/silver__token_transfers.yml +++ b/models/silver/curated/token_transfers/silver__token_transfers_complete.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: silver__token_transfers + - name: silver__token_transfers_complete tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: