refine test and lookback (#174)

macros/tests/tx_gaps.sql

@@ -6,7 +6,7 @@
             block_number,
             tx_count
         FROM
-            {{ ref('silver__blocks') }}
+            {{ ref('test_silver__blocks_full') }}
     ),
     model_name AS (
         SELECT
@@ -29,5 +29,51 @@ FROM
     LEFT JOIN model_name
     ON block_base.block_number = model_name.block_number
 WHERE
-    tx_count <> model_tx_count
+    (
+        tx_count <> model_tx_count
+    )
+    OR (
+        model_tx_count IS NULL
+        AND tx_count <> 0
+    )
+{% endmacro %}
+
+{% macro recent_tx_gaps(
+        model
+    ) %}
+    WITH block_base AS (
+        SELECT
+            block_number,
+            tx_count
+        FROM
+            {{ ref('test_silver__blocks_recent') }}
+    ),
+    model_name AS (
+        SELECT
+            block_number,
+            COUNT(
+                DISTINCT tx_hash
+            ) AS model_tx_count
+        FROM
+            {{ model }}
+        GROUP BY
+            block_number
+    )
+SELECT
+    block_base.block_number,
+    tx_count,
+    model_name.block_number AS model_block_number,
+    model_tx_count
+FROM
+    block_base
+    LEFT JOIN model_name
+    ON block_base.block_number = model_name.block_number
+WHERE
+    (
+        tx_count <> model_tx_count
+    )
+    OR (
+        model_tx_count IS NULL
+        AND tx_count <> 0
+    )
 {% endmacro %}

models/silver/core/silver__logs.sql

@@ -82,14 +82,12 @@ new_records AS (
     FROM
         flat_logs l
         LEFT OUTER JOIN {{ ref('silver__transactions') }}
-        txs USING (
-            block_number,
-            tx_hash
-        )
+        txs
+        ON l.block_number = txs.block_number
+        AND l.tx_hash = txs.tx_hash
 
 {% if is_incremental() %}
-WHERE
-    txs._INSERTED_TIMESTAMP >= '{{ lookback() }}'
+AND txs._INSERTED_TIMESTAMP >= '{{ lookback() }}'
 {% endif %}
 )
 

models/silver/core/silver__traces.sql

@@ -252,7 +252,11 @@ flattened_traces AS (
                 t
                 ON f.tx_position = t.position
                 AND f.block_number = t.block_number
-        )
+
+{% if is_incremental() %}
+AND t._INSERTED_TIMESTAMP >= '{{ lookback() }}'
+{% endif %}
+)
 
 {% if is_incremental() %},
 missing_data AS (

models/silver/core/silver__transactions.sql

@@ -117,16 +117,13 @@ new_records AS (
         r
         ON A.block_number = r.block_number
         AND A.data :hash :: STRING = r.tx_hash
-        LEFT OUTER JOIN {{ ref('silver__blocks') }}
-        b
-        ON A.block_number = b.block_number
 
 {% if is_incremental() %}
-WHERE
-    r._INSERTED_TIMESTAMP >= '{{ lookback() }}'
+AND r._INSERTED_TIMESTAMP >= '{{ lookback() }}'
 {% endif %}
-
-qualify(ROW_NUMBER() over (PARTITION BY A.data :hash :: STRING
+LEFT OUTER JOIN {{ ref('silver__blocks') }}
+b
+ON A.block_number = b.block_number qualify(ROW_NUMBER() over (PARTITION BY A.data :hash :: STRING
 ORDER BY
     A._inserted_timestamp DESC)) = 1
 )

tests/polygon/test_silver__missing_txs.sql

@@ -0,0 +1,2 @@
+-- depends_on: {{ ref('test_silver__blocks_full') }}
+{{ tx_gaps(ref("test_silver__transactions_full")) }}

tests/polygon/test_silver__recent_missing_txs.sql

@@ -0,0 +1,2 @@
+-- depends_on: {{ ref('test_silver__blocks_recent') }}
+{{ recent_tx_gaps(ref("test_silver__transactions_recent")) }}