From 14e11930a9482335ea42e987e75b2f4d2bd04ee6 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Thu, 29 May 2025 11:08:51 -0400 Subject: [PATCH] more perf upgrades --- data/metrics/silver__native_fee_token.csv | 5 +- models/docs/stats/crosschain_stats_core.md | 30 ++++++ .../stats__ez_activity_metrics_daily.sql | 94 +++++++------------ .../stats__ez_activity_metrics_daily.yml | 37 ++++++-- 4 files changed, 98 insertions(+), 68 deletions(-) diff --git a/data/metrics/silver__native_fee_token.csv b/data/metrics/silver__native_fee_token.csv index 2fe0005..06143f3 100644 --- a/data/metrics/silver__native_fee_token.csv +++ b/data/metrics/silver__native_fee_token.csv @@ -2,9 +2,10 @@ blockchain,address,symbol,blockchain_override aleo,null,ALEO,null aptos,null,APT,null arbitrum,0x82af49447d8a07e3bd95bd0d56f35241523fbab1,WETH,null -avalanche,0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7,AVAX,null +avalanche,NULL,AVAX,null axelar,null,AXL,null -base,0x4200000000000000000000000000000000000006,WETH,null +blast,0x4300000000000000000000000000000000000004,WETH,null +base,0x420000000000000000000000000000000000006,WETH,null bob,0x4200000000000000000000000000000000000006,WETH,null boba,0xa18bf3994c0cc6e3b63ac420308e5383f53120d7,BOBA,null bsc,null,BNB,null diff --git a/models/docs/stats/crosschain_stats_core.md b/models/docs/stats/crosschain_stats_core.md index 8b7439f..e1ff27b 100644 --- a/models/docs/stats/crosschain_stats_core.md +++ b/models/docs/stats/crosschain_stats_core.md @@ -34,12 +34,24 @@ The number of transactions in the hour. {% enddocs %} +{% docs crosschain_quality_transaction_count %} + +The number of transactions in the hour for addresses with a score of 4 or higher + +{% enddocs %} + {% docs crosschain_transaction_count_success %} The number of successful transactions in the hour. {% enddocs %} +{% docs crosschain_quality_transaction_count_success %} + +The number of successful transactions in the hour for addresses with a score of 4 or higher + +{% enddocs %} + {% docs crosschain_transaction_count_failed %} The number of failed transactions in the hour. @@ -52,6 +64,12 @@ The number of unique initiator or origin from addresses in the hour. {% enddocs %} +{% docs crosschain_quality_unique_from_count %} + +The number of unique initiator or origin from addresses in the hour for addresses with a score of 4 or higher + +{% enddocs %} + {% docs crosschain_unique_to_count %} The number of unique origin to addresses in the hour. @@ -68,4 +86,16 @@ The sum of all fees in the hour, in the native fee currency. The sum of all fees in the hour, in USD. +{% enddocs %} + +{% docs crosschain_quality_total_fees_native %} + +The sum of all fees in the hour, in the native fee currency for addresses with a score of 4 or higher + +{% enddocs %} + +{% docs crosschain_quality_total_fees_usd %} + +The sum of all fees in the hour, in USD for addresses with a score of 4 or higher + {% enddocs %} \ No newline at end of file diff --git a/models/gold/stats/stats__ez_activity_metrics_daily.sql b/models/gold/stats/stats__ez_activity_metrics_daily.sql index e14f2aa..283222d 100644 --- a/models/gold/stats/stats__ez_activity_metrics_daily.sql +++ b/models/gold/stats/stats__ez_activity_metrics_daily.sql @@ -6,6 +6,7 @@ incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], unique_key = ['blockchain','block_date'], + cluster_by = ['blockchain','block_date'], tags = ['metrics_daily'] ) }} @@ -42,6 +43,7 @@ WHERE AND modified_timestamp >= '{{ max_mod }}' {% else %} AND block_timestamp :: DATE >= '2025-01-01' + AND block_timestamp :: DATE <= '2025-01-10' {% endif %} {% endset %} @@ -93,20 +95,38 @@ GROUP BY block_timestamp_hour, sender {% endset %} {% do run_query(inc_query) %} + --find distinct score dates + {% set score_dates_query %} + CREATE + OR REPLACE temporary TABLE silver.ez_activity_metrics__score_dates_intermediate_tmp AS +SELECT + DISTINCT A.blockchain, + A.score_date +FROM + {{ source( + 'datascience_onchain_scores', + 'all_scores' + ) }} A {% endset %} + {% do run_query(score_dates_query) %} --find block dates where we do not have a score for that exact date {% set score_asof_query %} CREATE OR REPLACE temporary TABLE silver.ez_activity_metrics__scores_asof_intermediate_tmp AS SELECT DISTINCT A.blockchain, - A.block_date + A.block_date, + b.score_date FROM - silver.ez_activity_metrics__intermediate_tmp A - LEFT JOIN datascience.onchain_scores.all_scores b_ex - ON A.blockchain = b_ex.blockchain - AND A.block_date = b_ex.score_date -WHERE - b_ex.blockchain IS NULL {% endset %} + ez_activity_metrics__intermediate_tmp A asof + JOIN silver.ez_activity_metrics__score_dates_intermediate_tmp b match_condition ( + A.block_date >= score_date + ) + ON A.blockchain = b.blockchain qualify ROW_NUMBER() over ( + PARTITION BY A.blockchain, + A.block_Date + ORDER BY + ABS(DATEDIFF('day', score_date, A.block_date)) + ) = 1 {% endset %} {% do run_query(score_asof_query) %} --Get the score for that block date or the closest date we have prior to that date {% set scores_query %} @@ -115,64 +135,23 @@ WHERE SELECT A.blockchain, A.user_address, - A.score_date, b.block_date, - A.total_score, - 1 AS rn + A.total_score FROM {{ source( 'datascience_onchain_scores', 'all_scores' ) }} A - JOIN silver.ez_activity_metrics__intermediate_tmp b + JOIN silver.ez_activity_metrics__scores_asof_intermediate_tmp b ON A.blockchain = b.blockchain - AND A.score_date = b.block_date -UNION ALL -SELECT - A.blockchain, - A.user_address, - A.score_date, - b.block_date, - A.total_score, - ROW_NUMBER() over ( - PARTITION BY A.blockchain, - A.user_address, - b.block_Date - ORDER BY - score_date DESC - ) AS rn -FROM - {{ source( - 'datascience_onchain_scores', - 'all_scores' - ) }} A - JOIN ( - SELECT - blockchain, - MIN(block_Date) AS block_Date_min, - MAX(block_Date) AS block_Date_max - FROM - silver.ez_activity_metrics__scores_asof_intermediate_tmp - GROUP BY - blockchain - ) inn - ON A.blockchain = inn.blockchain - AND A.score_Date >= inn.block_Date_min - 7 - AND A.score_Date <= inn.block_Date_max + 7 asof - JOIN silver.ez_activity_metrics__scores_asof_intermediate_tmp b match_condition ( - score_date <= b.block_date - ) - ON A.blockchain = b.blockchain -WHERE - b.block_date IS NOT NULL {% endset %} + AND A.score_date = b.score_date {% endset %} {% do run_query(scores_query) %} --delete the scores temp with a score less than 4 or the additional rows from the asof join {% set scores_del_query %} DELETE FROM silver.ez_activity_metrics__scores_intermediate_tmp WHERE - rn > 1 - OR total_score < 4 {% endset %} + total_score < 4 {% endset %} {% do run_query(scores_del_query) %} {% endif %} @@ -180,11 +159,12 @@ WHERE WITH prices AS ( SELECT A.hour, - A.blockchain, + b.blockchain, A.price FROM - price.ez_prices_hourly A - JOIN silver.native_fee_token b + {{ ref('price__ez_prices_hourly') }} A + JOIN {{ ref('silver__native_fee_token') }} + b ON A.blockchain = COALESCE( b.blockchain_override, b.blockchain @@ -249,8 +229,7 @@ SELECT ) AS quality_total_fees_usd, {{ dbt_utils.generate_surrogate_key(['a.blockchain',' A.block_timestamp_hour :: DATE']) }} AS ez_activity_metrics_daily_id, SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id + SYSDATE() AS modified_timestamp FROM silver.ez_activity_metrics__tx_intermediate_tmp A asof JOIN prices b match_condition ( @@ -261,7 +240,6 @@ FROM ON A.blockchain = C.blockchain AND A.sender = C.user_address AND A.block_timestamp_hour :: DATE = C.block_date - AND C.rn = 1 GROUP BY A.blockchain, A.block_timestamp_hour :: DATE diff --git a/models/gold/stats/stats__ez_activity_metrics_daily.yml b/models/gold/stats/stats__ez_activity_metrics_daily.yml index 86a2125..eb921c9 100644 --- a/models/gold/stats/stats__ez_activity_metrics_daily.yml +++ b/models/gold/stats/stats__ez_activity_metrics_daily.yml @@ -7,7 +7,6 @@ models: combination_of_columns: - BLOCK_DATE - BLOCKCHAIN - - IS_QUALITY columns: - name: BLOCKCHAIN @@ -18,21 +17,43 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 1 - - name: IS_QUALITY - description: '{{ doc("is_quality") }}' + interval: 2 - name: TRANSACTION_COUNT description: '{{ doc("crosschain_transaction_count") }}' - - name: TRANSACTION_COUNT_SUCCESS - description: '{{ doc("crosschain_transaction_count_success") }}' - - name: TRANSACTION_COUNT_FAILED - description: '{{ doc("crosschain_transaction_count_failed") }}' + tests: + - not_null + - name: QUALITY_TRANSACTION_COUNT + description: '{{ doc("crosschain_transaction_count") }}' + tests: + - not_null + - name: QUALITY_TRANSACTION_COUNT_SUCCEEDED + description: '{{ doc("crosschain_quality_transaction_count_success") }}' + tests: + - not_null - name: UNIQUE_INITIATOR_COUNT description: '{{ doc("crosschain_unique_from_count") }}' + tests: + - not_null + - name: QUALITY_UNIQUE_INITIATOR_COUNT + description: '{{ doc("crosschain_quality_unique_from_count") }}' + tests: + - not_null - name: TOTAL_FEES_NATIVE description: '{{ doc("crosschain_total_fees_native") }}' + tests: + - not_null - name: TOTAL_FEES_USD description: '{{ doc("crosschain_total_fees_usd") }}' + tests: + - not_null + - name: QUALITY_TOTAL_FEES_NATIVE + description: '{{ doc("crosschain_quality_total_fees_native") }}' + tests: + - not_null + - name: QUALITY_TOTAL_FEES_USD + description: '{{ doc("crosschain_quality_total_fees_usd") }}' + tests: + - not_null - name: EZ_ACTIVITY_METRICS_DAILY_ID description: '{{ doc("pk") }}' - name: INSERTED_TIMESTAMP