diff --git a/models/silver/streamline/silver__streamline_blocks.sql b/models/silver/streamline/silver__streamline_blocks.sql index e09954d..3632c0e 100644 --- a/models/silver/streamline/silver__streamline_blocks.sql +++ b/models/silver/streamline/silver__streamline_blocks.sql @@ -74,11 +74,13 @@ blocks AS ( ) {% if var('IS_MIGRATION', False) %} {# Can quickly delete after migration. But, data in other tables is older blocks - ingested more recently. So, simply doing >= inserted timestamp will cause a large gap. #} + ingested more recently. So, simply doing >= inserted timestamp will cause a large gap. + Lookback, here, should probably be min 4 hours. + #} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '6 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var('STREAMLINE_LOAD_LOOKBACK_HOURS') }} hours' FROM {{ this }} ) {% else %} diff --git a/models/silver/streamline/silver__streamline_shards.sql b/models/silver/streamline/silver__streamline_shards.sql index c7617a8..054797f 100644 --- a/models/silver/streamline/silver__streamline_shards.sql +++ b/models/silver/streamline/silver__streamline_shards.sql @@ -67,11 +67,13 @@ shards AS ( LEFT JOIN meta m USING (_filename) {% if var('IS_MIGRATION', False) %} {# Can quickly delete after migration. But, data in other tables is older blocks - ingested more recently. So, simply doing >= inserted timestamp will cause a large gap. #} + ingested more recently. So, simply doing >= inserted timestamp will cause a large gap. + Lookback, here, should probably be min 4 hours. + #} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '6 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var('STREAMLINE_LOAD_LOOKBACK_HOURS') }} hours' FROM {{ this }} ) {% else %} diff --git a/tests/tests__streamline_block_gaps.sql b/tests/tests__streamline_block_gaps.sql index 17b4d93..f98702c 100644 --- a/tests/tests__streamline_block_gaps.sql +++ b/tests/tests__streamline_block_gaps.sql @@ -37,3 +37,5 @@ FROM silver_blocks WHERE prior_hash <> prev_hash + {# Filter out false positive from blocks at start of window (whose parent hash was cut off) #} + AND (_inserted_timestamp > SYSDATE() - INTERVAL '7 days' + INTERVAL '1 hour') \ No newline at end of file