diff --git a/models/streamline/core/complete/streamline__block_txs_complete.sql b/models/streamline/core/complete/streamline__block_txs_complete.sql index 7557575..9c5143f 100644 --- a/models/streamline/core/complete/streamline__block_txs_complete.sql +++ b/models/streamline/core/complete/streamline__block_txs_complete.sql @@ -1,5 +1,6 @@ -- depends_on: {{ ref('bronze__transactions') }} -- depends_on: {{ ref('bronze__FR_transactions') }} +-- depends_on: {{ ref('streamline__blocks') }} {{ config ( materialized = "incremental", @@ -8,6 +9,27 @@ cluster_by = "ROUND(block_id, -4)", ) }} +{% if execute %} + {% set min_partition_key_query %} + SELECT round(min(block_id),-4)::int AS min_partition_key + FROM ( + SELECT + block_id + FROM + {{ ref("streamline__blocks") }} + WHERE + /* Find the earliest block available from the node provider */ + block_id >= 6572203 + EXCEPT + SELECT + block_id + FROM + {{ this }} + ) + {% endset %} + {% set min_partition_key = run_query(min_partition_key_query)[0][0] %} +{% endif %} + SELECT block_id, error, @@ -20,18 +42,10 @@ FROM {% if is_incremental() %} {{ ref('bronze__transactions') }} WHERE - _inserted_timestamp >= ( + partition_key >= {{ min_partition_key }} + AND _inserted_timestamp >= ( SELECT - COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP - FROM - {{ this }} - ) - AND partition_key >= ( - SELECT - COALESCE( - MAX(partition_key), - 0 - ) + coalesce(max(_inserted_timestamp), '1970-01-01' :: DATE) AS max_inserted_timestamp FROM {{ this }} )