silver, other updates

dbt_project.yml

@@ -52,7 +52,7 @@ vars:
   "dbt_date:time_zone": GMT
   OBSERV_FULL_TEST: False
   START_GHA_TASKS: False
-  STREAMLINE_INVOKE_STREAMS: False
+  STREAMLINE_INVOKE_STREAMS: True
   STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: True
   STREAMLINE_RUN_HISTORY: False
   STREAMLINE_RETRY_UNKNOWN: False

models/bitquery/silver/silver__bitquery_tx_count.sql

@@ -0,0 +1,104 @@
+-- depends_on: {{ ref('bronze__bitquery') }}
+{{ config(
+    materialized = 'incremental',
+    unique_key = ['blockchain', 'metric', 'block_date'],
+    full_refresh = false,
+    tags = ['bitquery']
+) }}
+
+WITH ripple AS (
+
+    SELECT
+        A.blockchain,
+        A.metric,
+        b.value :date :date :: DATE AS block_date,
+        b.value :countBigInt AS tx_count,
+        A._inserted_timestamp
+    FROM
+
+{% if is_incremental() %}
+{{ ref('bronze__bitquery') }}
+{% else %}
+    {{ ref('bronze__bitquery_FR') }}
+{% endif %}
+
+A,
+LATERAL FLATTEN(
+    A.data :data :ripple :transactions
+) b
+WHERE
+    A.metric = 'tx_count'
+    AND A.blockchain = 'ripple'
+
+{% if is_incremental() %}
+AND _inserted_timestamp :: DATE > (
+    SELECT
+        MAX(_inserted_timestamp) :: DATE
+    FROM
+        {{ this }}
+)
+{% endif %}
+),
+hedera AS (
+    SELECT
+        A.blockchain,
+        A.metric,
+        b.value :date :date :: DATE AS block_date,
+        b.value :count AS tx_count,
+        A._inserted_timestamp
+    FROM
+
+{% if is_incremental() %}
+{{ ref('bronze__bitquery_FR') }}
+{% else %}
+    {{ ref('bronze__bitquery_FR') }}
+{% endif %}
+
+A,
+LATERAL FLATTEN(
+    A.data :data :hedera :transactions
+) b
+WHERE
+    A.metric = 'tx_count'
+    AND A.blockchain = 'hedera'
+
+{% if is_incremental() %}
+AND _inserted_timestamp :: DATE > (
+    SELECT
+        MAX(_inserted_timestamp) :: DATE
+    FROM
+        {{ this }}
+)
+{% endif %}
+),
+ua AS (
+    SELECT
+        *
+    FROM
+        ripple
+    UNION ALL
+    SELECT
+        *
+    FROM
+        hedera
+)
+SELECT
+    blockchain,
+    metric,
+    block_date,
+    tx_count,
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['blockchain','metric','block_date']
+    ) }} AS accounts_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    ua qualify ROW_NUMBER() over (
+        PARTITION BY blockchain,
+        metric,
+        block_date
+        ORDER BY
+            _inserted_timestamp DESC
+    ) = 1

models/bitquery/silver/silver__bitquery_tx_count.yml

@@ -0,0 +1,33 @@
+version: 2
+models:
+  - name: silver__bitquery_tx_count
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+            combination_of_columns:
+              - BLOCKCHAIN
+              - METRIC
+              - BLOCK_DATE
+    
+    columns:
+      - name: BLOCKCHAIN
+        tests:
+          - not_null
+      - name: METRIC
+        tests:
+          - not_null
+      - name: BLOCK_DATE
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 2
+      - name: TX_COUNT
+        tests:
+          - not_null
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+          

models/bitquery/streamline/streamline__bitquery_metrics.sql

@@ -19,11 +19,12 @@ WITH metrics AS (
     SELECT
         'hedera' AS blockchain,
         'active_users' AS metric,
-        'query ($network: HederaNetwork!, $dateFormat: String!, $from: ISO8601DateTime, $till: ISO8601DateTime) {hedera(network: $network) {transactions(options: {asc: "date.date"}, date: {since: $from, till: $till}) count: countBigInt(uniq: payer_account)}}}' AS query_text {# UNION ALL
+        'query ($network: HederaNetwork!, $from: ISO8601DateTime, $till: ISO8601DateTime) {hedera(network: $network) {transactions(date: {since: $from, till: $till}) { countBigInt(uniq: payer_account)}}}' AS query_text
+    UNION ALL
     SELECT
         'ripple' AS blockchain,
         'active_users' AS metric,
-        '' AS query_text #}
+        '' AS query_text
 )
 SELECT
     date_day,

models/bitquery/streamline/streamline__bitquery_realtime.sql

@@ -58,7 +58,7 @@ SELECT
             'variables',
             variables
         ),
-        'Vault/prod/bitquery'
+        'Vault/prod/external/bitquery'
     ) AS request
 FROM
     metrics