From 26f91269eff2e845d2afba10ad892c388a4fa82a Mon Sep 17 00:00:00 2001 From: Jack Forgash Date: Tue, 24 Jan 2023 23:32:46 -0700 Subject: [PATCH] macro to curated models --- .../silver__actions_events_addkey_s3.sql | 8 +- ...ilver__actions_events_function_call_s3.sql | 8 +- models/silver/silver__actions_events_s3.sql | 10 ++- models/silver/silver__dex_swaps_s3.sql | 8 +- models/silver/silver__nft_mints_s3.sql | 22 ++++-- models/silver/silver__prices_oracle_s3.sql | 14 ++-- models/silver/silver__staking_actions_s3.sql | 75 ++++++++++++++++--- models/silver/silver__staking_pools_s3.sql | 18 ++++- models/silver/silver__transfers_s3.sql | 18 ++++- models/silver/silver__usn_supply_s3.sql | 10 ++- 10 files changed, 152 insertions(+), 39 deletions(-) diff --git a/models/silver/silver__actions_events_addkey_s3.sql b/models/silver/silver__actions_events_addkey_s3.sql index 88e733c..e39a9a3 100644 --- a/models/silver/silver__actions_events_addkey_s3.sql +++ b/models/silver/silver__actions_events_addkey_s3.sql @@ -13,8 +13,12 @@ WITH action_events AS ( FROM {{ ref('silver__actions_events_s3') }} WHERE - action_name = 'AddKey' - AND {{ incremental_load_filter('_load_timestamp') }} + action_name = 'AddKey' + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + AND {{ incremental_load_filter('_load_timestamp') }} + {% endif %} ), addkey_events AS ( SELECT diff --git a/models/silver/silver__actions_events_function_call_s3.sql b/models/silver/silver__actions_events_function_call_s3.sql index 91ed982..34851b6 100644 --- a/models/silver/silver__actions_events_function_call_s3.sql +++ b/models/silver/silver__actions_events_function_call_s3.sql @@ -13,8 +13,12 @@ WITH action_events AS ( FROM {{ ref('silver__actions_events_s3') }} WHERE - {{ incremental_load_filter('_load_timestamp') }} - AND action_name = 'FunctionCall' + action_name = 'FunctionCall' + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + AND {{ incremental_load_filter('_load_timestamp') }} + {% endif %} ), decoding AS ( SELECT diff --git a/models/silver/silver__actions_events_s3.sql b/models/silver/silver__actions_events_s3.sql index ff1299c..d58cc54 100644 --- a/models/silver/silver__actions_events_s3.sql +++ b/models/silver/silver__actions_events_s3.sql @@ -25,8 +25,14 @@ WITH txs AS ( _partition_by_block_number FROM {{ ref('silver__streamline_transactions_final') }} - WHERE - {{ incremental_load_filter('_load_timestamp') }} + + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + WHERE + {{ incremental_load_filter('_load_timestamp') }} + {% endif %} ), actions AS ( SELECT diff --git a/models/silver/silver__dex_swaps_s3.sql b/models/silver/silver__dex_swaps_s3.sql index d7b123e..bbe95a0 100644 --- a/models/silver/silver__dex_swaps_s3.sql +++ b/models/silver/silver__dex_swaps_s3.sql @@ -22,8 +22,12 @@ WITH base_swap_calls AS ( method_name IN ( 'swap', 'ft_transfer_call' - ) - AND {{ incremental_load_filter('_load_timestamp') }} + ) + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + AND {{ incremental_load_filter('_load_timestamp') }} + {% endif %} ), base_swaps AS ( SELECT diff --git a/models/silver/silver__nft_mints_s3.sql b/models/silver/silver__nft_mints_s3.sql index 51c461a..288ff6b 100644 --- a/models/silver/silver__nft_mints_s3.sql +++ b/models/silver/silver__nft_mints_s3.sql @@ -41,8 +41,11 @@ WITH function_call AS ( method_name IN ( 'nft_mint', 'nft_mint_batch' - ) - AND {{ incremental_load_filter("_load_timestamp") }} + ) {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + AND {{ incremental_load_filter("_load_timestamp") }} + {% endif %} ), --Data Pulled from Transaction mint_transactions AS ( @@ -64,8 +67,11 @@ mint_transactions AS ( FROM function_call ) - AND tx_status = 'Success' - AND {{ incremental_load_filter("_load_timestamp") }} + AND tx_status = 'Success' {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + AND {{ incremental_load_filter("_load_timestamp") }} + {% endif %} ), --Data pulled from Receipts Table receipts_data AS ( @@ -84,8 +90,12 @@ receipts_data AS ( DISTINCT tx_hash FROM function_call - ) - AND {{ incremental_load_filter("_load_timestamp") }} + ) + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + AND {{ incremental_load_filter("_load_timestamp") }} + {% endif %} ) SELECT DISTINCT action_id, diff --git a/models/silver/silver__prices_oracle_s3.sql b/models/silver/silver__prices_oracle_s3.sql index 77fd9bc..91d4169 100644 --- a/models/silver/silver__prices_oracle_s3.sql +++ b/models/silver/silver__prices_oracle_s3.sql @@ -3,9 +3,7 @@ unique_key = 'block_id', incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE', '_load_timestamp::DATE'], - tags = ['curated', 's3_curated'] - - + tags = ['curated', 's3_curated'] ) }} WITH txs AS ( @@ -14,8 +12,14 @@ WITH txs AS ( * FROM {{ ref('silver__streamline_transactions_final') }} - WHERE - {{ incremental_load_filter('_load_timestamp') }} + + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + WHERE + {{ incremental_load_filter('_load_timestamp') }} + {% endif %} ), token_labels AS ( SELECT diff --git a/models/silver/silver__staking_actions_s3.sql b/models/silver/silver__staking_actions_s3.sql index e9e842e..fe43b8e 100644 --- a/models/silver/silver__staking_actions_s3.sql +++ b/models/silver/silver__staking_actions_s3.sql @@ -15,21 +15,31 @@ WITH actions_events_function_call AS ( FROM {{ ref('silver__actions_events_function_call_s3') }} WHERE - {{ incremental_load_filter('_load_timestamp') }} - AND method_name IN ( + method_name IN ( 'deposit_and_stake', 'stake', 'unstake', 'unstake_all' - ) + ) + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + AND {{ incremental_load_filter('_load_timestamp') }} + {% endif %} ), base_txs AS ( SELECT * FROM {{ ref('silver__streamline_transactions_final') }} - WHERE - {{ incremental_load_filter('_load_timestamp') }} + + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + WHERE + {{ incremental_load_filter('_load_timestamp') }} + {% endif %} ), txs AS ( SELECT @@ -62,7 +72,16 @@ deposit_and_stake_txs AS ( block_timestamp, tx_receiver AS pool_address, tx_signer, - REGEXP_SUBSTR(ARRAY_TO_STRING(tx :receipt [0] :outcome :logs, ','), 'staking (\\d+)', 1, 1, 'e') :: NUMBER AS stake_amount, + REGEXP_SUBSTR( + ARRAY_TO_STRING( + tx :receipt [0] :outcome :logs, + ',' + ), + 'staking (\\d+)', + 1, + 1, + 'e' + ) :: NUMBER AS stake_amount, 'Stake' AS action, _load_timestamp FROM @@ -77,7 +96,16 @@ stake_txs AS ( block_timestamp, tx_receiver AS pool_address, tx_signer, - REGEXP_SUBSTR(ARRAY_TO_STRING(tx :receipt [0] :outcome :logs, ','), 'staking (\\d+)', 1, 1, 'e') :: NUMBER AS stake_amount, + REGEXP_SUBSTR( + ARRAY_TO_STRING( + tx :receipt [0] :outcome :logs, + ',' + ), + 'staking (\\d+)', + 1, + 1, + 'e' + ) :: NUMBER AS stake_amount, 'Stake' AS action, _load_timestamp FROM @@ -92,7 +120,16 @@ stake_all_txs AS ( block_timestamp, tx_receiver AS pool_address, tx_signer, - REGEXP_SUBSTR(ARRAY_TO_STRING(tx :receipt [0] :outcome :logs, ','), 'staking (\\d+)', 1, 1, 'e') :: NUMBER AS stake_amount, + REGEXP_SUBSTR( + ARRAY_TO_STRING( + tx :receipt [0] :outcome :logs, + ',' + ), + 'staking (\\d+)', + 1, + 1, + 'e' + ) :: NUMBER AS stake_amount, 'Stake' AS action, _load_timestamp FROM @@ -107,7 +144,16 @@ unstake_txs AS ( block_timestamp, tx_receiver AS pool_address, tx_signer, - REGEXP_SUBSTR(ARRAY_TO_STRING(tx :receipt [0] :outcome :logs, ','), 'unstaking (\\d+)', 1, 1, 'e') :: NUMBER AS stake_amount, + REGEXP_SUBSTR( + ARRAY_TO_STRING( + tx :receipt [0] :outcome :logs, + ',' + ), + 'unstaking (\\d+)', + 1, + 1, + 'e' + ) :: NUMBER AS stake_amount, 'Unstake' AS action, _load_timestamp FROM @@ -122,7 +168,16 @@ unstake_all_txs AS ( block_timestamp, tx_receiver AS pool_address, tx_signer, - REGEXP_SUBSTR(ARRAY_TO_STRING(tx :receipt [0] :outcome :logs, ','), 'unstaking (\\d+)', 1, 1, 'e') :: NUMBER AS stake_amount, + REGEXP_SUBSTR( + ARRAY_TO_STRING( + tx :receipt [0] :outcome :logs, + ',' + ), + 'unstaking (\\d+)', + 1, + 1, + 'e' + ) :: NUMBER AS stake_amount, 'Unstake' AS action, _load_timestamp FROM diff --git a/models/silver/silver__staking_pools_s3.sql b/models/silver/silver__staking_pools_s3.sql index 086a302..647e2a1 100644 --- a/models/silver/silver__staking_pools_s3.sql +++ b/models/silver/silver__staking_pools_s3.sql @@ -17,8 +17,14 @@ WITH txs AS ( _load_timestamp FROM {{ ref('silver__streamline_transactions_final') }} - WHERE - {{ incremental_load_filter('_load_timestamp') }} + + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + WHERE + {{ incremental_load_filter('_load_timestamp') }} + {% endif %} ), function_calls AS ( SELECT @@ -32,8 +38,12 @@ function_calls AS ( method_name IN ( 'create_staking_pool', 'update_reward_fee_fraction' - ) - AND {{ incremental_load_filter('_load_timestamp') }} + ) + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + AND {{ incremental_load_filter('_load_timestamp') }} + {% endif %} ), pool_txs AS ( SELECT diff --git a/models/silver/silver__transfers_s3.sql b/models/silver/silver__transfers_s3.sql index 9011de0..ad9ca8b 100644 --- a/models/silver/silver__transfers_s3.sql +++ b/models/silver/silver__transfers_s3.sql @@ -15,8 +15,12 @@ WITH action_events AS( FROM {{ ref('silver__actions_events_s3') }} WHERE - action_name = 'Transfer' - AND {{ incremental_load_filter("_load_timestamp") }} + action_name = 'Transfer' + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + AND {{ incremental_load_filter("_load_timestamp") }} + {% endif %} ), txs AS ( SELECT @@ -35,8 +39,14 @@ txs AS ( _load_timestamp FROM {{ ref('silver__streamline_transactions_final') }} - WHERE - {{ incremental_load_filter("_load_timestamp") }} + + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + WHERE + {{ incremental_load_filter("_load_timestamp") }} + {% endif %} ), receipts AS ( SELECT diff --git a/models/silver/silver__usn_supply_s3.sql b/models/silver/silver__usn_supply_s3.sql index f4239f3..4b8caf3 100644 --- a/models/silver/silver__usn_supply_s3.sql +++ b/models/silver/silver__usn_supply_s3.sql @@ -12,8 +12,14 @@ WITH txs AS ( * FROM {{ ref('silver__streamline_receipts_final') }} - WHERE - {{ incremental_load_filter('_load_timestamp') }} + + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + WHERE + {{ incremental_load_filter('_load_timestamp') }} + {% endif %} ), logs AS ( SELECT