get min partition same as what the request view would do

This commit is contained in:
desmond-hui 2025-03-10 09:28:38 -07:00
parent fb1ebb5d4d
commit c64b4feba2

View File

@ -1,5 +1,6 @@
-- depends_on: {{ ref('bronze__transactions') }}
-- depends_on: {{ ref('bronze__FR_transactions') }}
-- depends_on: {{ ref('streamline__blocks') }}
{{ config (
materialized = "incremental",
@ -8,6 +9,27 @@
cluster_by = "ROUND(block_id, -4)",
) }}
{% if execute %}
{% set min_partition_key_query %}
SELECT round(min(block_id),-4)::int AS min_partition_key
FROM (
SELECT
block_id
FROM
{{ ref("streamline__blocks") }}
WHERE
/* Find the earliest block available from the node provider */
block_id >= 6572203
EXCEPT
SELECT
block_id
FROM
{{ this }}
)
{% endset %}
{% set min_partition_key = run_query(min_partition_key_query)[0][0] %}
{% endif %}
SELECT
block_id,
error,
@ -20,18 +42,10 @@ FROM
{% if is_incremental() %}
{{ ref('bronze__transactions') }}
WHERE
_inserted_timestamp >= (
partition_key >= {{ min_partition_key }}
AND _inserted_timestamp >= (
SELECT
COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
FROM
{{ this }}
)
AND partition_key >= (
SELECT
COALESCE(
MAX(partition_key),
0
)
coalesce(max(_inserted_timestamp), '1970-01-01' :: DATE) AS max_inserted_timestamp
FROM
{{ this }}
)