From e08975a5e32cbca1f8614cd185dd6ccfe42b376d Mon Sep 17 00:00:00 2001
From: Jack Forgash <58153492+forgxyz@users.noreply.github.com>
Date: Tue, 6 Feb 2024 08:30:57 -0700
Subject: [PATCH] upd block gap test, and migration lookback in 2 load models

---
 models/silver/streamline/silver__streamline_blocks.sql | 6 ++++--
 models/silver/streamline/silver__streamline_shards.sql | 6 ++++--
 tests/tests__streamline_block_gaps.sql                 | 2 ++
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/models/silver/streamline/silver__streamline_blocks.sql b/models/silver/streamline/silver__streamline_blocks.sql
index e09954d..3632c0e 100644
--- a/models/silver/streamline/silver__streamline_blocks.sql
+++ b/models/silver/streamline/silver__streamline_blocks.sql
@@ -74,11 +74,13 @@ blocks AS (
         )
     {% if var('IS_MIGRATION', False) %}
     {# Can quickly delete after migration. But, data in other tables is older blocks 
-    ingested more recently. So, simply doing >= inserted timestamp will cause a large gap. #}
+    ingested more recently. So, simply doing >= inserted timestamp will cause a large gap.
+    Lookback, here, should probably be min 4 hours.
+     #}
     WHERE
         _inserted_timestamp >= (
             SELECT 
-                MAX(_inserted_timestamp) - INTERVAL '6 hours'
+                MAX(_inserted_timestamp) - INTERVAL '{{ var('STREAMLINE_LOAD_LOOKBACK_HOURS') }} hours'
             FROM {{ this }}
         )
     {% else %}
diff --git a/models/silver/streamline/silver__streamline_shards.sql b/models/silver/streamline/silver__streamline_shards.sql
index c7617a8..054797f 100644
--- a/models/silver/streamline/silver__streamline_shards.sql
+++ b/models/silver/streamline/silver__streamline_shards.sql
@@ -67,11 +67,13 @@ shards AS (
         LEFT JOIN meta m USING (_filename)
     {% if var('IS_MIGRATION', False) %}
     {# Can quickly delete after migration. But, data in other tables is older blocks 
-    ingested more recently. So, simply doing >= inserted timestamp will cause a large gap. #}
+    ingested more recently. So, simply doing >= inserted timestamp will cause a large gap.
+    Lookback, here, should probably be min 4 hours.
+     #}
     WHERE
         _inserted_timestamp >= (
             SELECT 
-                MAX(_inserted_timestamp) - INTERVAL '6 hours'
+                MAX(_inserted_timestamp) - INTERVAL '{{ var('STREAMLINE_LOAD_LOOKBACK_HOURS') }} hours'
             FROM {{ this }}
         )
     {% else %}
diff --git a/tests/tests__streamline_block_gaps.sql b/tests/tests__streamline_block_gaps.sql
index 17b4d93..f98702c 100644
--- a/tests/tests__streamline_block_gaps.sql
+++ b/tests/tests__streamline_block_gaps.sql
@@ -37,3 +37,5 @@ FROM
   silver_blocks
 WHERE
   prior_hash <> prev_hash
+  {# Filter out false positive from blocks at start of window (whose parent hash was cut off) #}
+  AND (_inserted_timestamp > SYSDATE() - INTERVAL '7 days' + INTERVAL '1 hour')
\ No newline at end of file