flip/near-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/near-models.git synced 2026-02-06 13:41:54 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

upd block gap test, and migration lookback in 2 load models

Browse Source
This commit is contained in:
Jack Forgash 2024-02-06 08:30:57 -07:00
parent 71b41749a7
commit e08975a5e3
3 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
models/silver/streamline/silver__streamline_blocks.sql
View File

@ -74,11 +74,13 @@ blocks AS (
)
{% if var('IS_MIGRATION', False) %}
{# Can quickly delete after migration. But, data in other tables is older blocks
ingested more recently. So, simply doing >= inserted timestamp will cause a large gap. #}
ingested more recently. So, simply doing >= inserted timestamp will cause a large gap.
Lookback, here, should probably be min 4 hours.
#}
WHERE
_inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp) - INTERVAL '6 hours'
MAX(_inserted_timestamp) - INTERVAL '{{ var('STREAMLINE_LOAD_LOOKBACK_HOURS') }} hours'
FROM {{ this }}
)
{% else %}

6
models/silver/streamline/silver__streamline_shards.sql
View File

@ -67,11 +67,13 @@ shards AS (
LEFT JOIN meta m USING (_filename)
{% if var('IS_MIGRATION', False) %}
{# Can quickly delete after migration. But, data in other tables is older blocks
ingested more recently. So, simply doing >= inserted timestamp will cause a large gap. #}
ingested more recently. So, simply doing >= inserted timestamp will cause a large gap.
Lookback, here, should probably be min 4 hours.
#}
WHERE
_inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp) - INTERVAL '6 hours'
MAX(_inserted_timestamp) - INTERVAL '{{ var('STREAMLINE_LOAD_LOOKBACK_HOURS') }} hours'
FROM {{ this }}
)
{% else %}

2
tests/tests__streamline_block_gaps.sql
View File

@ -37,3 +37,5 @@ FROM
silver_blocks
WHERE
prior_hash <> prev_hash
{# Filter out false positive from blocks at start of window (whose parent hash was cut off) #}
AND (_inserted_timestamp > SYSDATE() - INTERVAL '7 days' + INTERVAL '1 hour')