From be735714d6fc69c8bc633229671b9394cc26f2bd Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Tue, 8 Apr 2025 22:01:32 +0700 Subject: [PATCH] final fixes --- .github/workflows/dbt_run_daily.yml | 2 +- .github/workflows/dbt_run_streamline_daily.yml | 2 +- .../streamline/streamline__artemis_complete.sql | 5 +++-- .../streamline/streamline__artemis_realtime.sql | 10 +++++++++- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index 97bd72b..4a562ff 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -44,4 +44,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m external_models,tag:defillama external_models,tag:deepnftvalue external_models,tag:core external_models,tag:blast external_models,tag:polymarket external_models,tag:bitquery external_models,tag:oklink --exclude models/defillama/bronze/bronze__defillama_stablecoin_supply.sql+ \ No newline at end of file + dbt run -m external_models,tag:defillama external_models,tag:deepnftvalue external_models,tag:core external_models,tag:blast external_models,tag:polymarket external_models,tag:bitquery external_models,tag:oklink external_models,tag:artemis --exclude models/defillama/bronze/bronze__defillama_stablecoin_supply.sql+ \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_daily.yml b/.github/workflows/dbt_run_streamline_daily.yml index 38a1586..4f9b487 100644 --- a/.github/workflows/dbt_run_streamline_daily.yml +++ b/.github/workflows/dbt_run_streamline_daily.yml @@ -44,4 +44,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/bitquery/streamline/streamline__bitquery_realtime.sql 1+models/oklink/streamline/streamline__oklink_realtime.sql \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/bitquery/streamline/streamline__bitquery_realtime.sql 1+models/oklink/streamline/streamline__oklink_realtime.sql 1+models/artemis/streamline/streamline__artemis_realtime.sql \ No newline at end of file diff --git a/models/artemis/streamline/streamline__artemis_complete.sql b/models/artemis/streamline/streamline__artemis_complete.sql index 960107a..4b84905 100644 --- a/models/artemis/streamline/streamline__artemis_complete.sql +++ b/models/artemis/streamline/streamline__artemis_complete.sql @@ -34,7 +34,7 @@ WITH bronze AS ( {% endif %} ), extracted_dates AS ( - SELECT DISTINCT + SELECT request_date_day, TO_DATE(date_vals.value:date::STRING) AS extracted_date, data, @@ -49,7 +49,7 @@ extracted_dates AS ( WHERE data:data:artemis_ids IS NOT NULL ) -SELECT DISTINCT +SELECT extracted_date AS date_day, data, partition_key, @@ -60,6 +60,7 @@ SELECT DISTINCT '{{ invocation_id }}' AS _invocation_id FROM extracted_dates + QUALIFY ROW_NUMBER() OVER ( PARTITION BY extracted_date ORDER BY diff --git a/models/artemis/streamline/streamline__artemis_realtime.sql b/models/artemis/streamline/streamline__artemis_realtime.sql index c1c0b25..40dee23 100644 --- a/models/artemis/streamline/streamline__artemis_realtime.sql +++ b/models/artemis/streamline/streamline__artemis_realtime.sql @@ -16,11 +16,19 @@ WITH complete_data AS ( + {% if is_incremental() %} SELECT - *, + date_day, + _invocation_id, MAX(date_day) OVER () AS max_complete_date FROM {{ ref("streamline__artemis_complete") }} + {% else %} + SELECT + null as date_day, + null as _invocation_id, + '2025-01-01'::DATE AS max_complete_date + {% endif %} ), date_params AS ( SELECT