From 11ae8740992c4ce3aac9bd0e410b8d085dbe4472 Mon Sep 17 00:00:00 2001
From: gregoriustanleyy <gstanleytejakusuma@gmail.com>
Date: Fri, 28 Mar 2025 10:53:37 +0700
Subject: [PATCH] bronze + complete + upd source

---
 models/artemis/bronze/bronze__artemis.sql     |  9 ++++
 models/artemis/bronze/bronze__artemis_FR.sql  |  9 ++++
 .../streamline__artemis_complete.sql          | 44 +++++++++++++++++++
 .../streamline__artemis_metrics.sql           |  3 +-
 .../streamline__artemis_realtime.sql          |  8 ++++
 models/sources.yml                            |  1 +
 6 files changed, 73 insertions(+), 1 deletion(-)
 create mode 100644 models/artemis/bronze/bronze__artemis.sql
 create mode 100644 models/artemis/bronze/bronze__artemis_FR.sql
 create mode 100644 models/artemis/streamline/streamline__artemis_complete.sql

diff --git a/models/artemis/bronze/bronze__artemis.sql b/models/artemis/bronze/bronze__artemis.sql
new file mode 100644
index 0000000..729d605
--- /dev/null
+++ b/models/artemis/bronze/bronze__artemis.sql
@@ -0,0 +1,9 @@
+{{ config (
+    materialized = 'view'
+) }}
+{{ streamline_external_table_query_v2(
+    model = 'artemis',
+    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
+    partition_name = "partition_key",
+    other_cols = "to_date(value:DATE_DAY::STRING,'YYYYMMDD') AS DATE_DAY, value:BLOCKCHAIN::STRING AS BLOCKCHAIN, value:METRIC::STRING AS METRIC"
+) }}
diff --git a/models/artemis/bronze/bronze__artemis_FR.sql b/models/artemis/bronze/bronze__artemis_FR.sql
new file mode 100644
index 0000000..ee54f9f
--- /dev/null
+++ b/models/artemis/bronze/bronze__artemis_FR.sql
@@ -0,0 +1,9 @@
+{{ config (
+    materialized = 'view'
+) }}
+{{ streamline_external_table_FR_query_v2(
+    model = 'artemis',
+    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
+    partition_name = "partition_key",
+    other_cols = "to_date(value:DATE_DAY::STRING,'YYYYMMDD') AS DATE_DAY, value:BLOCKCHAIN::STRING AS BLOCKCHAIN, value:METRIC::STRING AS METRIC"
+) }}
diff --git a/models/artemis/streamline/streamline__artemis_complete.sql b/models/artemis/streamline/streamline__artemis_complete.sql
new file mode 100644
index 0000000..4b5719e
--- /dev/null
+++ b/models/artemis/streamline/streamline__artemis_complete.sql
@@ -0,0 +1,44 @@
+-- depends_on: {{ ref("bronze__artemis")}}
+-- depends_on: {{ ref("bronze__artemis_FR")}}
+{{ config (
+    materialized = "incremental",
+    unique_key = [' date_day','blockchain','metric'],
+    merge_exclude_columns = ["inserted_timestamp"],
+    tags = ['streamline_realtime']
+) }}
+
+SELECT
+    date_day,
+    blockchain,
+    metric,
+    patition_key,
+    _inserted_timestamp,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    file_name,
+    '{{ invocation_id }}' AS _invocation_id,
+FROM
+
+{% if is_incremental() %}
+{{ ref('bronze__artemis') }}
+{% else %}
+    {{ ref('bronze__artemis_FR') }}
+{% endif %}
+WHERE
+    DATA :errors IS NULL
+
+{% if is_incremental() %}
+AND _inserted_timestamp >= (
+    SELECT
+        COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
+    FROM
+        {{ this }})
+    {% endif %}
+
+    qualify ROW_NUMBER() over (
+        PARTITION BY date_day,
+        blockchain,
+        metric
+        ORDER BY
+            _inserted_timestamp DESC
+    ) = 1
diff --git a/models/artemis/streamline/streamline__artemis_metrics.sql b/models/artemis/streamline/streamline__artemis_metrics.sql
index 025b41e..c033915 100644
--- a/models/artemis/streamline/streamline__artemis_metrics.sql
+++ b/models/artemis/streamline/streamline__artemis_metrics.sql
@@ -110,4 +110,5 @@ FROM
         'crosschain_core',
         'dim_dates'
     ) }}
-    CROSS JOIN metrics
\ No newline at end of file
+    CROSS JOIN metrics
+WHERE date_day >= '2025-01-01'
\ No newline at end of file
diff --git a/models/artemis/streamline/streamline__artemis_realtime.sql b/models/artemis/streamline/streamline__artemis_realtime.sql
index a390ad3..7b5c513 100644
--- a/models/artemis/streamline/streamline__artemis_realtime.sql
+++ b/models/artemis/streamline/streamline__artemis_realtime.sql
@@ -25,6 +25,14 @@ WITH metrics AS (
         TO_CHAR(date_day, 'YYYY-MM-DD') AS query_date
     FROM
         {{ ref("streamline__artemis_metrics") }}
+    LEFT JOIN {{ ref("streamline__oklink_complete") }}
+        b USING (
+            blockchain,
+            metric,
+            date_day
+        )
+    WHERE
+        b._invocation_id IS NULL
 )
 SELECT
     TO_NUMBER(to_char(date_day, 'YYYYMMDD')) AS date_day,
diff --git a/models/sources.yml b/models/sources.yml
index dee298a..ca8b613 100644
--- a/models/sources.yml
+++ b/models/sources.yml
@@ -7,6 +7,7 @@ sources:
     tables:
       - name: bitquery
       - name: oklink
+      - name: artemis
   - name: tokenflow_starknet_l1_data
     database: tokenflow_starknet
     schema: l1_data