stop rerequesting

This commit is contained in:
Eric Laurello 2025-05-02 15:46:16 -04:00
parent e52130c90d
commit bcfa0d9765
2 changed files with 34 additions and 2 deletions

View File

@ -1,7 +1,39 @@
{{ config (
materialized = 'view'
) }}
{{ streamline_external_table_query_v2(
{# {{ streamline_external_table_query_v2(
model = "sei_addresses",
partition_function = "to_date(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1), 'YYYYMMDD')"
) }}
#}
WITH meta AS (
SELECT
job_created_time AS _inserted_timestamp,
file_name,
to_date(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1), 'YYYYMMDD') AS partition_key
FROM
TABLE(
information_schema.external_table_file_registration_history(
start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()),
table_name => 'streamline.SEI.sei_addresses')
) A
)
SELECT
s.*,
b.file_name,
_inserted_timestamp
FROM
{{ source(
'bronze_streamline',
'sei_addresses'
) }}
s
JOIN meta b
ON b.file_name = metadata$filename
AND b.partition_key = s.partition_key
WHERE
b.partition_key = s.partition_key

View File

@ -21,7 +21,7 @@ FROM
{{ ref('bronze__streamline_FR_sei_addresses') }}
{% endif %}
WHERE
DATA :result LIKE 'sei%'
1 = 1 --DATA :result LIKE 'sei%'
{% if is_incremental() %}
AND _inserted_timestamp >=(