diff --git a/models/artemis/gold/artemis__active_users.yml b/models/artemis/gold/artemis__active_users.yml index 9123f70..f45f529 100644 --- a/models/artemis/gold/artemis__active_users.yml +++ b/models/artemis/gold/artemis__active_users.yml @@ -19,7 +19,7 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 2 + interval: 3 - name: ACTIVE_USERS description: "The number of active users for the given blockchain and as of date." tests: diff --git a/models/artemis/gold/artemis__tx_count.yml b/models/artemis/gold/artemis__tx_count.yml index 4e91fd4..7cce9c3 100644 --- a/models/artemis/gold/artemis__tx_count.yml +++ b/models/artemis/gold/artemis__tx_count.yml @@ -19,7 +19,7 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 2 + interval: 3 - name: TX_COUNT description: "The number of tx count for the given blockchain and as of date." tests: diff --git a/models/artemis/silver/silver__artemis.sql b/models/artemis/silver/silver__artemis.sql index 07b905a..67be6d5 100644 --- a/models/artemis/silver/silver__artemis.sql +++ b/models/artemis/silver/silver__artemis.sql @@ -4,9 +4,10 @@ materialized = "incremental", unique_key = ['metric_date', 'blockchain', 'metric'], tags = ['silver', 'artemis'] -)}} +) }} WITH source_data AS ( + SELECT raw_data, partition_key, @@ -15,18 +16,19 @@ WITH source_data AS ( FROM {% if is_incremental() %} - {{ ref('bronze__artemis') }} +{{ ref('bronze__artemis') }} {% else %} {{ ref('bronze__artemis_FR') }} {% endif %} {% if is_incremental() %} - -WHERE _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM {{ this }} -) +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) {% endif %} ), parsed_data AS ( @@ -34,17 +36,23 @@ parsed_data AS ( s._inserted_timestamp, s.partition_key, s.file_name, - blockchain_flat.KEY AS blockchain, - metric_flat.KEY AS metric, - metrics.value:date::STRING AS metric_date, - metrics.value:val AS metric_value + blockchain_flat.key AS blockchain, + metric_flat.key AS metric, + metrics.value :date :: STRING AS metric_date, + metrics.value :val AS metric_value FROM source_data s, - LATERAL FLATTEN(INPUT => raw_data:data:artemis_ids) AS blockchain_flat, - LATERAL FLATTEN(INPUT => blockchain_flat.value) AS metric_flat, - LATERAL FLATTEN(INPUT => metric_flat.value) AS metrics + LATERAL FLATTEN( + input => raw_data :data :artemis_ids + ) AS blockchain_flat, + LATERAL FLATTEN( + input => blockchain_flat.value + ) AS metric_flat, + LATERAL FLATTEN( + input => metric_flat.value + ) AS metrics WHERE - raw_data:data:artemis_ids IS NOT NULL + raw_data :data :artemis_ids IS NOT NULL ) SELECT TO_DATE(metric_date) AS metric_date, @@ -59,4 +67,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - parsed_data \ No newline at end of file + parsed_data qualify(ROW_NUMBER() over (PARTITION BY metric_date, blockchain, metric +ORDER BY + _inserted_timestamp DESC) = 1) diff --git a/models/artemis/streamline/streamline__artemis_realtime.sql b/models/artemis/streamline/streamline__artemis_realtime.sql index 40dee23..e7a0154 100644 --- a/models/artemis/streamline/streamline__artemis_realtime.sql +++ b/models/artemis/streamline/streamline__artemis_realtime.sql @@ -15,20 +15,14 @@ ) }} WITH complete_data AS ( - - {% if is_incremental() %} + SELECT date_day, _invocation_id, MAX(date_day) OVER () AS max_complete_date FROM {{ ref("streamline__artemis_complete") }} - {% else %} - SELECT - null as date_day, - null as _invocation_id, - '2025-01-01'::DATE AS max_complete_date - {% endif %} + ), date_params AS ( SELECT