From c3a3141fa866fab959dedf8ffd5c1f14f90c4b40 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 2 May 2025 10:33:29 -0600 Subject: [PATCH 1/2] fix incr logic - ft transfers method+ --- .../silver/lockup/silver__lockup_actions.sql | 98 ++++++++++++++----- ...er__token_transfer_ft_transfers_method.sql | 70 +++++++++++-- 2 files changed, 138 insertions(+), 30 deletions(-) diff --git a/models/silver/lockup/silver__lockup_actions.sql b/models/silver/lockup/silver__lockup_actions.sql index aeea3dd..2d8f8ed 100644 --- a/models/silver/lockup/silver__lockup_actions.sql +++ b/models/silver/lockup/silver__lockup_actions.sql @@ -8,6 +8,60 @@ tags = ['curated', 'scheduled_non_core'], ) }} +{% if execute %} + + {% if is_incremental() and not var("MANUAL_FIX") %} + {% do log("Incremental and not MANUAL_FIX", info=True) %} + {% set max_mod_query %} + + SELECT + MAX(modified_timestamp) modified_timestamp + FROM + {{ this }} + + {% endset %} + + {% set max_mod = run_query(max_mod_query) [0] [0] %} + {% if not max_mod or max_mod == 'None' %} + {% set max_mod = '2099-01-01' %} + {% endif %} + + {% do log("max_mod: " ~ max_mod, info=True) %} + + {% set min_block_date_query %} + SELECT + MIN( + block_timestamp :: DATE + ) + FROM + ( + SELECT + MIN(block_timestamp) block_timestamp + FROM + {{ ref('core__ez_actions') }} A + WHERE + modified_timestamp >= '{{max_mod}}' + UNION ALL + SELECT + MIN(block_timestamp) block_timestamp + FROM + {{ ref('silver__token_transfer_native') }} A + WHERE + modified_timestamp >= '{{max_mod}}' + ) + {% endset %} + + {% set min_bd = run_query(min_block_date_query) [0] [0] %} + {% if not min_bd or min_bd == 'None' %} + {% set min_bd = '2099-01-01' %} + {% endif %} + + {% do log("min_bd: " ~ min_bd, info=True) %} + + {% endif %} + +{% endif %} + WITH lockup_actions AS ( SELECT @@ -21,7 +75,8 @@ lockup_actions AS ( action_data, tx_succeeded, receipt_succeeded, - _partition_by_block_number + _partition_by_block_number, + modified_timestamp FROM {{ ref('core__ez_actions') }} WHERE @@ -42,13 +97,8 @@ lockup_actions AS ( {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} - {% if is_incremental() %} - AND modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) + {% if is_incremental() %} + AND block_timestamp :: DATE >= '{{min_bd}}' {% endif %} {% endif %} @@ -61,20 +111,15 @@ xfers AS ( block_id, amount_unadj :: INT AS deposit, _partition_by_block_number, - _inserted_timestamp + modified_timestamp FROM {{ ref('silver__token_transfer_native') }} {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} - {% if is_incremental() %} - WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) + {% if is_incremental() %} + AND block_timestamp :: DATE >= '{{min_bd}}' {% endif %} {% endif %} ), @@ -98,7 +143,8 @@ agg_arguments AS ( COUNT( DISTINCT action_data :method_name :: STRING ) AS method_count, - MIN(_partition_by_block_number) AS _partition_by_block_number + MIN(_partition_by_block_number) AS _partition_by_block_number, + MIN(modified_timestamp) AS modified_timestamp FROM lockup_actions GROUP BY @@ -112,16 +158,13 @@ lockup_xfers AS ( block_id, deposit, _partition_by_block_number, - _inserted_timestamp + xfers.modified_timestamp FROM xfers + LEFT JOIN lockup_actions l + ON xfers.tx_hash = l.tx_hash WHERE - tx_hash IN ( - SELECT - DISTINCT tx_hash - FROM - lockup_actions - ) + l.tx_hash IS NOT NULL ), parse_args_json AS ( SELECT @@ -164,6 +207,13 @@ parse_args_json AS ( agg_arguments A LEFT JOIN lockup_xfers x ON A.receipt_ids :new :: STRING = x.receipt_id + {% if is_incremental() and not var("MANUAL_FIX") %} + WHERE + GREATEST( + COALESCE(A.modified_timestamp, '1970-01-01'), + COALESCE(x.modified_timestamp, '1970-01-01') + ) >= '{{max_mod}}' + {% endif %} ), FINAL AS ( SELECT diff --git a/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql index c6eaf1b..dad2b64 100644 --- a/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql +++ b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql @@ -8,6 +8,61 @@ tags = ['curated','scheduled_non_core'] ) }} + +{% if execute %} + + {% if is_incremental() and not var("MANUAL_FIX") %} + {% do log("Incremental and not MANUAL_FIX", info=True) %} + {% set max_mod_query %} + + SELECT + MAX(modified_timestamp) modified_timestamp + FROM + {{ this }} + + {% endset %} + + {% set max_mod = run_query(max_mod_query) [0] [0] %} + {% if not max_mod or max_mod == 'None' %} + {% set max_mod = '2099-01-01' %} + {% endif %} + + {% do log("max_mod: " ~ max_mod, info=True) %} + + {% set min_block_date_query %} + SELECT + MIN( + block_timestamp :: DATE + ) + FROM + ( + SELECT + MIN(block_timestamp) block_timestamp + FROM + {{ ref('core__ez_actions') }} A + WHERE + modified_timestamp >= '{{max_mod}}' + UNION ALL + SELECT + MIN(block_timestamp) block_timestamp + FROM + {{ ref('silver__logs_s3') }} A + WHERE + modified_timestamp >= '{{max_mod}}' + ) + {% endset %} + + {% set min_bd = run_query(min_block_date_query) [0] [0] %} + {% if not min_bd or min_bd == 'None' %} + {% set min_bd = '2099-01-01' %} + {% endif %} + + {% do log("min_bd: " ~ min_bd, info=True) %} + + {% endif %} + +{% endif %} + WITH ft_transfer_actions AS ( SELECT block_id, @@ -31,12 +86,7 @@ WITH ft_transfer_actions AS ( AND {{ partition_load_manual('no_buffer') }} {% else %} {% if is_incremental() %} - AND modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) + WHERE block_timestamp :: DATE >= '{{min_bd}}' {% endif %} {% endif %} ), @@ -60,6 +110,14 @@ ft_transfer_logs AS ( AND l.receipt_id = a.receipt_id WHERE l.receipt_succeeded + {% if var("MANUAL_FIX") %} + AND + {{ partition_load_manual('no_buffer') }} + {% else %} + {% if is_incremental() %} + AND block_timestamp :: DATE >= '{{min_bd}}' + {% endif %} + {% endif %} ), ft_transfers_final AS ( SELECT From 6533d5d54a71f000f1108e4bcd3bee03292615f8 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 2 May 2025 10:40:48 -0600 Subject: [PATCH 2/2] typos and fixes --- .../silver/lockup/silver__lockup_actions.sql | 14 +++--- ...er__token_transfer_ft_transfers_method.sql | 49 +++++++++++++------ 2 files changed, 42 insertions(+), 21 deletions(-) diff --git a/models/silver/lockup/silver__lockup_actions.sql b/models/silver/lockup/silver__lockup_actions.sql index 2d8f8ed..12d70d2 100644 --- a/models/silver/lockup/silver__lockup_actions.sql +++ b/models/silver/lockup/silver__lockup_actions.sql @@ -119,7 +119,7 @@ xfers AS ( WHERE {{ partition_load_manual('no_buffer') }} {% else %} {% if is_incremental() %} - AND block_timestamp :: DATE >= '{{min_bd}}' + WHERE block_timestamp :: DATE >= '{{min_bd}}' {% endif %} {% endif %} ), @@ -152,12 +152,12 @@ agg_arguments AS ( ), lockup_xfers AS ( SELECT - tx_hash, - receipt_id, - block_timestamp, - block_id, - deposit, - _partition_by_block_number, + xfers.tx_hash, + xfers.receipt_id, + xfers.block_timestamp, + xfers.block_id, + xfers.deposit, + xfers._partition_by_block_number, xfers.modified_timestamp FROM xfers diff --git a/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql index dad2b64..db0764b 100644 --- a/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql +++ b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql @@ -74,7 +74,8 @@ WITH ft_transfer_actions AS ( receipt_predecessor_id AS predecessor_id, receipt_signer_id AS signer_id, receipt_succeeded, - _partition_by_block_number + _partition_by_block_number, + modified_timestamp FROM {{ ref('core__ez_actions') }} WHERE @@ -86,7 +87,30 @@ WITH ft_transfer_actions AS ( AND {{ partition_load_manual('no_buffer') }} {% else %} {% if is_incremental() %} - WHERE block_timestamp :: DATE >= '{{min_bd}}' + AND block_timestamp :: DATE >= '{{min_bd}}' + {% endif %} + {% endif %} +), +logs AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + receipt_id, + log_index, + clean_log AS log_value, + _partition_by_block_number, + modified_timestamp + FROM + {{ ref('silver__logs_s3') }} + WHERE + receipt_succeeded + {% if var("MANUAL_FIX") %} + AND + {{ partition_load_manual('no_buffer') }} + {% else %} + {% if is_incremental() %} + AND block_timestamp :: DATE >= '{{min_bd}}' {% endif %} {% endif %} ), @@ -97,26 +121,23 @@ ft_transfer_logs AS ( l.tx_hash, l.receipt_id, l.log_index, - l.clean_log AS log_value, + l.log_value, l._partition_by_block_number, a.contract_address, a.predecessor_id, a.signer_id, a.action_index FROM - {{ ref('silver__logs_s3') }} l - INNER JOIN ft_transfer_actions a + logs l + INNER JOIN ft_transfer_actions a ON l.tx_hash = a.tx_hash AND l.receipt_id = a.receipt_id - WHERE - l.receipt_succeeded - {% if var("MANUAL_FIX") %} - AND - {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - AND block_timestamp :: DATE >= '{{min_bd}}' - {% endif %} + {% if is_incremental() and not var("MANUAL_FIX") %} + WHERE + GREATEST( + COALESCE(l.modified_timestamp, '1970-01-01'), + COALESCE(a.modified_timestamp, '1970-01-01') + ) >= '{{max_mod}}' {% endif %} ), ft_transfers_final AS (