diff --git a/macros/tests/tx_gaps.sql b/macros/tests/tx_gaps.sql index a9d93df..ed52562 100644 --- a/macros/tests/tx_gaps.sql +++ b/macros/tests/tx_gaps.sql @@ -6,7 +6,7 @@ block_number, tx_count FROM - {{ ref('silver__blocks') }} + {{ ref('test_silver__blocks_full') }} ), model_name AS ( SELECT @@ -29,5 +29,51 @@ FROM LEFT JOIN model_name ON block_base.block_number = model_name.block_number WHERE - tx_count <> model_tx_count + ( + tx_count <> model_tx_count + ) + OR ( + model_tx_count IS NULL + AND tx_count <> 0 + ) +{% endmacro %} + +{% macro recent_tx_gaps( + model + ) %} + WITH block_base AS ( + SELECT + block_number, + tx_count + FROM + {{ ref('test_silver__blocks_recent') }} + ), + model_name AS ( + SELECT + block_number, + COUNT( + DISTINCT tx_hash + ) AS model_tx_count + FROM + {{ model }} + GROUP BY + block_number + ) +SELECT + block_base.block_number, + tx_count, + model_name.block_number AS model_block_number, + model_tx_count +FROM + block_base + LEFT JOIN model_name + ON block_base.block_number = model_name.block_number +WHERE + ( + tx_count <> model_tx_count + ) + OR ( + model_tx_count IS NULL + AND tx_count <> 0 + ) {% endmacro %} diff --git a/models/silver/core/silver__logs.sql b/models/silver/core/silver__logs.sql index 8efb83e..8238cc3 100644 --- a/models/silver/core/silver__logs.sql +++ b/models/silver/core/silver__logs.sql @@ -82,14 +82,12 @@ new_records AS ( FROM flat_logs l LEFT OUTER JOIN {{ ref('silver__transactions') }} - txs USING ( - block_number, - tx_hash - ) + txs + ON l.block_number = txs.block_number + AND l.tx_hash = txs.tx_hash {% if is_incremental() %} -WHERE - txs._INSERTED_TIMESTAMP >= '{{ lookback() }}' +AND txs._INSERTED_TIMESTAMP >= '{{ lookback() }}' {% endif %} ) diff --git a/models/silver/core/silver__traces.sql b/models/silver/core/silver__traces.sql index aacb861..a065048 100644 --- a/models/silver/core/silver__traces.sql +++ b/models/silver/core/silver__traces.sql @@ -252,7 +252,11 @@ flattened_traces AS ( t ON f.tx_position = t.position AND f.block_number = t.block_number - ) + +{% if is_incremental() %} +AND t._INSERTED_TIMESTAMP >= '{{ lookback() }}' +{% endif %} +) {% if is_incremental() %}, missing_data AS ( diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index 2ea711c..f1da7b5 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -117,16 +117,13 @@ new_records AS ( r ON A.block_number = r.block_number AND A.data :hash :: STRING = r.tx_hash - LEFT OUTER JOIN {{ ref('silver__blocks') }} - b - ON A.block_number = b.block_number {% if is_incremental() %} -WHERE - r._INSERTED_TIMESTAMP >= '{{ lookback() }}' +AND r._INSERTED_TIMESTAMP >= '{{ lookback() }}' {% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY A.data :hash :: STRING +LEFT OUTER JOIN {{ ref('silver__blocks') }} +b +ON A.block_number = b.block_number qualify(ROW_NUMBER() over (PARTITION BY A.data :hash :: STRING ORDER BY A._inserted_timestamp DESC)) = 1 ) diff --git a/tests/polygon/test_silver__missing_txs.sql b/tests/polygon/test_silver__missing_txs.sql new file mode 100644 index 0000000..499466f --- /dev/null +++ b/tests/polygon/test_silver__missing_txs.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__blocks_full') }} +{{ tx_gaps(ref("test_silver__transactions_full")) }} diff --git a/tests/polygon/test_silver__recent_missing_txs.sql b/tests/polygon/test_silver__recent_missing_txs.sql new file mode 100644 index 0000000..3d02c92 --- /dev/null +++ b/tests/polygon/test_silver__recent_missing_txs.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__blocks_recent') }} +{{ recent_tx_gaps(ref("test_silver__transactions_recent")) }}