better gap logic

fix dupes in tx
rm duplicate unique test
This commit is contained in:
Eric Laurello 2024-10-08 15:22:34 -04:00
parent 96755a007d
commit bd4a65da23
3 changed files with 30 additions and 10 deletions

View File

@ -330,4 +330,6 @@ SELECT
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL
FINAL qualify(ROW_NUMBER() over (PARTITION BY tx_hash
ORDER BY
_inserted_timestamp DESC) = 1)

View File

@ -124,7 +124,6 @@ models:
description: '{{ doc("pk_id") }}'
tests:
- not_null
- unique
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP

View File

@ -3,13 +3,12 @@
severity = 'error'
) }}
WITH blocks AS (
WITH MIN AS (
SELECT
block_number,
block_hash,
parent_hash,
_inserted_timestamp
MIN(
block_timestamp :: DATE
) bd
FROM
{{ ref('silver_evm__blocks') }}
@ -21,6 +20,22 @@ WITH blocks AS (
_inserted_timestamp >= SYSDATE() - INTERVAL '7 days'
{% endif %}
),
blocks AS (
SELECT
block_number,
block_hash,
parent_hash,
_inserted_timestamp
FROM
{{ ref('silver_evm__blocks') }}
WHERE
block_timestamp :: DATE >= (
SELECT
bd
FROM
MIN
)
),
check_orphan AS (
SELECT
child.block_number,
@ -46,7 +61,11 @@ determine_previous_block AS (
LAG(block_number) over (
ORDER BY
block_number
) AS prev_block_number
) AS prev_block_number,
ROW_NUMBER() over(
ORDER BY
block_number
) rn
FROM
check_orphan
)
@ -61,6 +80,6 @@ FROM
determine_previous_block
WHERE
confirmed_parent_hash IS NULL
AND block_number > 1
-- may be some temporarily missing blocks at chainhead, only issue if not filled on subsequent run
AND block_number > 1 -- may be some temporarily missing blocks at chainhead, only issue if not filled on subsequent run
AND rn > 1
AND _inserted_timestamp <= SYSDATE() - INTERVAL '1 hour'