From 6b2ba63fccc607ee2a1b2664bbb7db64d50c272d Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Fri, 20 Sep 2024 12:09:22 +0300 Subject: [PATCH 1/4] rn --- .../non_native/silver__token_transfer_ft_transfers_event.sql | 5 +++-- .../silver__token_transfer_ft_transfers_method.sql | 5 +++-- .../non_native/silver__token_transfer_liquidity.sql | 5 +++-- .../non_native/silver__token_transfer_mints.sql | 3 ++- .../non_native/silver__token_transfer_orders.sql | 5 +++-- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql index 3f962cb..9840f45 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql @@ -1,9 +1,10 @@ {{ config( materialized = 'incremental', + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_modified_timestamp::Date'], unique_key = 'transfers_event_id', - incremental_strategy = 'delete+insert', + incremental_strategy = 'merge', tags = ['curated','scheduled_non_core'] ) }} @@ -92,7 +93,7 @@ ft_transfers_final AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['action_id'] + ['action_id','rn'] ) }} AS transfers_event_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql index f023b37..1074fa6 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql @@ -1,9 +1,10 @@ {{ config( materialized = 'incremental', + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_modified_timestamp::Date'], unique_key = 'transfers_id', - incremental_strategy = 'delete+insert', + incremental_strategy = 'merge', tags = ['curated','scheduled_non_core'] ) }} @@ -83,7 +84,7 @@ ft_transfers_method AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['action_id'] + ['action_id','rn'] ) }} AS transfers_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql index 8a74553..bab8f58 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql @@ -1,9 +1,10 @@ {{ config( materialized = 'incremental', + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_modified_timestamp::Date'], unique_key = 'transfers_liquidity_id', - incremental_strategy = 'delete+insert', + incremental_strategy = 'merge', tags = ['curated','scheduled_non_core'] ) }} @@ -85,7 +86,7 @@ add_liquidity AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['action_id'] + ['action_id','rn'] ) }} AS transfers_liquidity_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql index 9896871..ca91e9f 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql @@ -1,9 +1,10 @@ {{ config( materialized = 'incremental', merge_exclude_columns = ["inserted_timestamp"], + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], cluster_by = ['block_timestamp::DATE','_modified_timestamp::Date'], unique_key = 'mint_id', - incremental_strategy = 'delete+insert', + incremental_strategy = 'merge', tags = ['curated','scheduled_non_core'] ) }} diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql index f7f0a50..dab62d2 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql @@ -1,9 +1,10 @@ {{ config( materialized = 'incremental', merge_exclude_columns = ["inserted_timestamp"], + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], cluster_by = ['block_timestamp::DATE','_modified_timestamp::Date'], unique_key = 'transfers_orders_id', - incremental_strategy = 'delete+insert', + incremental_strategy = 'merge', tags = ['curated','scheduled_non_core'] ) }} @@ -86,7 +87,7 @@ orders_final AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['action_id'] + ['action_id','rn'] ) }} AS transfers_orders_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, From 63c54a1774c540cfe34eebb5d402d89052fc5ec7 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Fri, 20 Sep 2024 12:11:48 +0300 Subject: [PATCH 2/4] key --- .../non_native/silver__token_transfer_ft_transfers_event.sql | 2 +- .../non_native/silver__token_transfer_ft_transfers_method.sql | 2 +- .../non_native/silver__token_transfer_liquidity.sql | 2 +- .../token_transfers/non_native/silver__token_transfer_mints.sql | 2 +- .../non_native/silver__token_transfer_orders.sql | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql index 9840f45..c1333a2 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql @@ -93,7 +93,7 @@ ft_transfers_final AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['action_id','rn'] + ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] ) }} AS transfers_event_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql index 1074fa6..2871c64 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql @@ -84,7 +84,7 @@ ft_transfers_method AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['action_id','rn'] + ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] ) }} AS transfers_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql index bab8f58..bfacb70 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql @@ -86,7 +86,7 @@ add_liquidity AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['action_id','rn'] + 'tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] ) }} AS transfers_liquidity_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql index ca91e9f..83185e7 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql @@ -93,7 +93,7 @@ ft_mints_final AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['action_id','rn'] + ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] )}} AS mint_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql index dab62d2..818d19a 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql @@ -87,7 +87,7 @@ orders_final AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['action_id','rn'] + ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] ) }} AS transfers_orders_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, From c5d17abc4454e3994a24efd755c68f225c6a3a15 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Fri, 20 Sep 2024 12:12:21 +0300 Subject: [PATCH 3/4] fix list --- .../non_native/silver__token_transfer_liquidity.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql index bfacb70..fe14d10 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql @@ -86,7 +86,7 @@ add_liquidity AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - 'tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] + ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] ) }} AS transfers_liquidity_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, From 10858d96c4aab17b5a5c19ea98fcf8943ec50589 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Fri, 20 Sep 2024 12:54:33 +0300 Subject: [PATCH 4/4] keys --- .../non_native/silver__token_transfer_ft_transfers_event.sql | 2 +- .../non_native/silver__token_transfer_ft_transfers_method.sql | 2 +- .../non_native/silver__token_transfer_liquidity.sql | 2 +- .../token_transfers/non_native/silver__token_transfer_mints.sql | 2 +- .../non_native/silver__token_transfer_orders.sql | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql index c1333a2..35837fa 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql @@ -93,7 +93,7 @@ ft_transfers_final AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] + ['tx_hash', 'action_id','contract_address','amount_unadj','from_address','to_address','memo','rn'] ) }} AS transfers_event_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql index 2871c64..047c76b 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql @@ -84,7 +84,7 @@ ft_transfers_method AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] + ['tx_hash', 'action_id','contract_address','amount_unadj','from_address','to_address','memo','rn'] ) }} AS transfers_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql index fe14d10..4c98c00 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql @@ -86,7 +86,7 @@ add_liquidity AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] + ['tx_hash', 'action_id','contract_address','amount_unadj','from_address','to_address','memo','rn'] ) }} AS transfers_liquidity_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql index 83185e7..84c7462 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql @@ -93,7 +93,7 @@ ft_mints_final AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] + ['tx_hash', 'action_id','contract_address','amount_unadj','from_address','to_address','memo','rn'] )}} AS mint_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql b/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql index 818d19a..0aef75b 100644 --- a/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql +++ b/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql @@ -87,7 +87,7 @@ orders_final AS ( SELECT *, {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'action_id','contract_address','amount_raw','from_address','to_address','memo','rn'] + ['tx_hash', 'action_id','contract_address','amount_unadj','from_address','to_address','memo','rn'] ) }} AS transfers_orders_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp,