more perf upgrades

data/metrics/silver__native_fee_token.csv

@@ -2,9 +2,10 @@ blockchain,address,symbol,blockchain_override
 aleo,null,ALEO,null
 aptos,null,APT,null
 arbitrum,0x82af49447d8a07e3bd95bd0d56f35241523fbab1,WETH,null
-avalanche,0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7,AVAX,null
+avalanche,NULL,AVAX,null
 axelar,null,AXL,null
-base,0x4200000000000000000000000000000000000006,WETH,null
+blast,0x4300000000000000000000000000000000000004,WETH,null
+base,0x420000000000000000000000000000000000006,WETH,null
 bob,0x4200000000000000000000000000000000000006,WETH,null
 boba,0xa18bf3994c0cc6e3b63ac420308e5383f53120d7,BOBA,null
 bsc,null,BNB,null

models/docs/stats/crosschain_stats_core.md

@@ -34,12 +34,24 @@ The number of transactions in the hour.
 
 {% enddocs %}
 
+{% docs crosschain_quality_transaction_count %}
+
+The number of transactions in the hour for addresses with a score of 4 or higher
+
+{% enddocs %}
+
 {% docs crosschain_transaction_count_success %}
 
 The number of successful transactions in the hour.
 
 {% enddocs %}
 
+{% docs crosschain_quality_transaction_count_success %}
+
+The number of successful transactions in the hour for addresses with a score of 4 or higher
+
+{% enddocs %}
+
 {% docs crosschain_transaction_count_failed %}
 
 The number of failed transactions in the hour.
@@ -52,6 +64,12 @@ The number of unique initiator or origin from addresses in the hour.
 
 {% enddocs %}
 
+{% docs crosschain_quality_unique_from_count %}
+
+The number of unique initiator or origin from addresses in the hour for addresses with a score of 4 or higher
+
+{% enddocs %}
+
 {% docs crosschain_unique_to_count %}
 
 The number of unique origin to addresses in the hour.
@@ -68,4 +86,16 @@ The sum of all fees in the hour, in the native fee currency.
 
 The sum of all fees in the hour, in USD.
 
+{% enddocs %}
+
+{% docs crosschain_quality_total_fees_native %}
+
+The sum of all fees in the hour, in the native fee currency for addresses with a score of 4 or higher
+
+{% enddocs %}
+
+{% docs crosschain_quality_total_fees_usd %}
+
+The sum of all fees in the hour, in USD for addresses with a score of 4 or higher
+
 {% enddocs %}

models/gold/stats/stats__ez_activity_metrics_daily.sql

@@ -6,6 +6,7 @@
     incremental_strategy = 'merge',
     merge_exclude_columns = ["inserted_timestamp"],
     unique_key = ['blockchain','block_date'],
+    cluster_by = ['blockchain','block_date'],
     tags = ['metrics_daily']
 ) }}
 
@@ -42,6 +43,7 @@ WHERE
 AND modified_timestamp >= '{{ max_mod }}'
 {% else %}
     AND block_timestamp :: DATE >= '2025-01-01'
+    AND block_timestamp :: DATE <= '2025-01-10'
 {% endif %}
 
 {% endset %}
@@ -93,20 +95,38 @@ GROUP BY
     block_timestamp_hour,
     sender {% endset %}
     {% do run_query(inc_query) %}
+    --find distinct score dates
+    {% set score_dates_query %}
+    CREATE
+    OR REPLACE temporary TABLE silver.ez_activity_metrics__score_dates_intermediate_tmp AS
+SELECT
+    DISTINCT A.blockchain,
+    A.score_date
+FROM
+    {{ source(
+        'datascience_onchain_scores',
+        'all_scores'
+    ) }} A {% endset %}
+    {% do run_query(score_dates_query) %}
     --find block dates where we do not have a score for that exact date
     {% set score_asof_query %}
     CREATE
     OR REPLACE temporary TABLE silver.ez_activity_metrics__scores_asof_intermediate_tmp AS
 SELECT
     DISTINCT A.blockchain,
-    A.block_date
+    A.block_date,
+    b.score_date
 FROM
-    silver.ez_activity_metrics__intermediate_tmp A
-    LEFT JOIN datascience.onchain_scores.all_scores b_ex
-    ON A.blockchain = b_ex.blockchain
-    AND A.block_date = b_ex.score_date
-WHERE
-    b_ex.blockchain IS NULL {% endset %}
+    ez_activity_metrics__intermediate_tmp A asof
+    JOIN silver.ez_activity_metrics__score_dates_intermediate_tmp b match_condition (
+        A.block_date >= score_date
+    )
+    ON A.blockchain = b.blockchain qualify ROW_NUMBER() over (
+        PARTITION BY A.blockchain,
+        A.block_Date
+        ORDER BY
+            ABS(DATEDIFF('day', score_date, A.block_date))
+    ) = 1 {% endset %}
     {% do run_query(score_asof_query) %}
     --Get the score for that block date or the closest date we have prior to that date
     {% set scores_query %}
@@ -115,64 +135,23 @@ WHERE
 SELECT
     A.blockchain,
     A.user_address,
-    A.score_date,
     b.block_date,
-    A.total_score,
-    1 AS rn
+    A.total_score
 FROM
     {{ source(
         'datascience_onchain_scores',
         'all_scores'
     ) }} A
-    JOIN silver.ez_activity_metrics__intermediate_tmp b
+    JOIN silver.ez_activity_metrics__scores_asof_intermediate_tmp b
     ON A.blockchain = b.blockchain
-    AND A.score_date = b.block_date
-UNION ALL
-SELECT
-    A.blockchain,
-    A.user_address,
-    A.score_date,
-    b.block_date,
-    A.total_score,
-    ROW_NUMBER() over (
-        PARTITION BY A.blockchain,
-        A.user_address,
-        b.block_Date
-        ORDER BY
-            score_date DESC
-    ) AS rn
-FROM
-    {{ source(
-        'datascience_onchain_scores',
-        'all_scores'
-    ) }} A
-    JOIN (
-        SELECT
-            blockchain,
-            MIN(block_Date) AS block_Date_min,
-            MAX(block_Date) AS block_Date_max
-        FROM
-            silver.ez_activity_metrics__scores_asof_intermediate_tmp
-        GROUP BY
-            blockchain
-    ) inn
-    ON A.blockchain = inn.blockchain
-    AND A.score_Date >= inn.block_Date_min - 7
-    AND A.score_Date <= inn.block_Date_max + 7 asof
-    JOIN silver.ez_activity_metrics__scores_asof_intermediate_tmp b match_condition (
-        score_date <= b.block_date
-    )
-    ON A.blockchain = b.blockchain
-WHERE
-    b.block_date IS NOT NULL {% endset %}
+    AND A.score_date = b.score_date {% endset %}
     {% do run_query(scores_query) %}
     --delete the scores temp with a score less than 4 or the additional rows from the asof join
     {% set scores_del_query %}
 DELETE FROM
     silver.ez_activity_metrics__scores_intermediate_tmp
 WHERE
-    rn > 1
-    OR total_score < 4 {% endset %}
+    total_score < 4 {% endset %}
     {% do run_query(scores_del_query) %}
 {% endif %}
 
@@ -180,11 +159,12 @@ WHERE
 WITH prices AS (
     SELECT
         A.hour,
-        A.blockchain,
+        b.blockchain,
         A.price
     FROM
-        price.ez_prices_hourly A
-        JOIN silver.native_fee_token b
+        {{ ref('price__ez_prices_hourly') }} A
+        JOIN {{ ref('silver__native_fee_token') }}
+        b
         ON A.blockchain = COALESCE(
             b.blockchain_override,
             b.blockchain
@@ -249,8 +229,7 @@ SELECT
     ) AS quality_total_fees_usd,
     {{ dbt_utils.generate_surrogate_key(['a.blockchain',' A.block_timestamp_hour :: DATE']) }} AS ez_activity_metrics_daily_id,
     SYSDATE() AS inserted_timestamp,
-    SYSDATE() AS modified_timestamp,
-    '{{ invocation_id }}' AS _invocation_id
+    SYSDATE() AS modified_timestamp
 FROM
     silver.ez_activity_metrics__tx_intermediate_tmp A asof
     JOIN prices b match_condition (
@@ -261,7 +240,6 @@ FROM
     ON A.blockchain = C.blockchain
     AND A.sender = C.user_address
     AND A.block_timestamp_hour :: DATE = C.block_date
-    AND C.rn = 1
 GROUP BY
     A.blockchain,
     A.block_timestamp_hour :: DATE

models/gold/stats/stats__ez_activity_metrics_daily.yml

@@ -7,7 +7,6 @@ models:
           combination_of_columns:
             - BLOCK_DATE
             - BLOCKCHAIN
-            - IS_QUALITY
 
     columns:
       - name: BLOCKCHAIN 
@@ -18,21 +17,43 @@ models:
           - not_null
           - dbt_expectations.expect_row_values_to_have_recent_data:
               datepart: day
-              interval: 1
-      - name: IS_QUALITY
-        description: '{{ doc("is_quality") }}'
+              interval: 2
       - name: TRANSACTION_COUNT
         description: '{{ doc("crosschain_transaction_count") }}'
-      - name: TRANSACTION_COUNT_SUCCESS
-        description: '{{ doc("crosschain_transaction_count_success") }}'
-      - name: TRANSACTION_COUNT_FAILED
-        description: '{{ doc("crosschain_transaction_count_failed") }}'
+        tests:
+          - not_null
+      - name: QUALITY_TRANSACTION_COUNT
+        description: '{{ doc("crosschain_transaction_count") }}'
+        tests:
+          - not_null
+      - name: QUALITY_TRANSACTION_COUNT_SUCCEEDED
+        description: '{{ doc("crosschain_quality_transaction_count_success") }}'
+        tests:
+          - not_null
       - name: UNIQUE_INITIATOR_COUNT
         description: '{{ doc("crosschain_unique_from_count") }}'
+        tests:
+          - not_null
+      - name: QUALITY_UNIQUE_INITIATOR_COUNT
+        description: '{{ doc("crosschain_quality_unique_from_count") }}'
+        tests:
+          - not_null
       - name: TOTAL_FEES_NATIVE
         description: '{{ doc("crosschain_total_fees_native") }}'
+        tests:
+          - not_null
       - name: TOTAL_FEES_USD
         description: '{{ doc("crosschain_total_fees_usd") }}'
+        tests:
+          - not_null
+      - name: QUALITY_TOTAL_FEES_NATIVE
+        description: '{{ doc("crosschain_quality_total_fees_native") }}'
+        tests:
+          - not_null
+      - name: QUALITY_TOTAL_FEES_USD
+        description: '{{ doc("crosschain_quality_total_fees_usd") }}'
+        tests:
+          - not_null
       - name: EZ_ACTIVITY_METRICS_DAILY_ID
         description: '{{ doc("pk") }}'   
       - name: INSERTED_TIMESTAMP