look for new data in current max partition also (#784)

This commit is contained in:
desmond-hui 2025-01-29 07:35:24 -08:00 committed by GitHub
parent 634d9db3e6
commit 759603a6e1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -13,7 +13,7 @@ FROM
{{ ref('bronze__streamline_block_txs_2') }}
WHERE
{% if is_incremental() %}
_partition_id > (SELECT max(_partition_id) FROM {{ this }})
_partition_id >= (SELECT max(_partition_id) FROM {{ this }})
AND _inserted_timestamp >= (SELECT max(_inserted_timestamp) FROM {{ this }})
{% else %}
_partition_id = (SELECT max(_partition_id) FROM {{ source('solana_streamline', 'complete_block_txs_2') }}) -- Reference this once to get a starting point