From f3ddada41fc6d43765704604aa44ccc2135ced14 Mon Sep 17 00:00:00 2001 From: drethereum <71602799+drethereum@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:47:43 -0700 Subject: [PATCH] AN-4454/core-stats (#239) * stats and docs * native * alias * test * incremental logic * round * docs * impute --- .../doc_descriptions/general/__overview__.md | 3 + .../doc_descriptions/stats/evm_stats_core.md | 71 ++++++++++++++++ .../stats/stats__ez_core_metrics_hourly.sql | 37 ++++++++ .../stats/stats__ez_core_metrics_hourly.yml | 34 ++++++++ .../silver_stats__core_metrics_hourly.sql | 66 +++++++++++++++ .../silver_stats__core_metrics_hourly.yml | 84 +++++++++++++++++++ 6 files changed, 295 insertions(+) create mode 100644 models/doc_descriptions/stats/evm_stats_core.md create mode 100644 models/gold/stats/stats__ez_core_metrics_hourly.sql create mode 100644 models/gold/stats/stats__ez_core_metrics_hourly.yml create mode 100644 models/silver/stats/silver_stats__core_metrics_hourly.sql create mode 100644 models/silver/stats/silver_stats__core_metrics_hourly.yml diff --git a/models/doc_descriptions/general/__overview__.md b/models/doc_descriptions/general/__overview__.md index 9cf1a5c0..63f99ea3 100644 --- a/models/doc_descriptions/general/__overview__.md +++ b/models/doc_descriptions/general/__overview__.md @@ -57,6 +57,9 @@ There is more information on how to use dbt docs in the last section of this doc - [ez_lending_repayments](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.defi__ez_lending_repayments) - [ez_lending_withdraws](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.defi__ez_lending_withdraws) +### Stats Tables (ethereum.stats) +- [ez_core_metrics_hourly](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.stats__ez_core_metrics_hourly) + **Velodrome** - [ez_claimed_rewards](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.velodrome__ez_claimed_rewards) - [ez_lp_actions](https://flipsidecrypto.github.io/optimism-models/#!/model/model.optimism_models.velodrome__ez_lp_actions) diff --git a/models/doc_descriptions/stats/evm_stats_core.md b/models/doc_descriptions/stats/evm_stats_core.md new file mode 100644 index 00000000..e8a450a9 --- /dev/null +++ b/models/doc_descriptions/stats/evm_stats_core.md @@ -0,0 +1,71 @@ +{% docs evm_ez_core_metrics_hourly_table_doc %} + +A convenience table that aggregates block and transaction related metrics using various aggregate functions such as SUM, COUNT, MIN and MAX from the fact_transactions table, on an hourly basis. + +{% enddocs %} + +{% docs evm_block_timestamp_hour %} + +The hour of the timestamp of the block. + +{% enddocs %} + +{% docs evm_block_number_min %} + +The minimum block number in the hour. + +{% enddocs %} + +{% docs evm_block_number_max %} + +The maximum block number in the hour. + +{% enddocs %} + +{% docs evm_block_count %} + +The number of blocks in the hour. + +{% enddocs %} + +{% docs evm_transaction_count %} + +The number of transactions in the hour. + +{% enddocs %} + +{% docs evm_transaction_count_success %} + +The number of successful transactions in the hour. + +{% enddocs %} + +{% docs evm_transaction_count_failed %} + +The number of failed transactions in the hour. + +{% enddocs %} + +{% docs evm_unique_from_count %} + +The number of unique origin from addresses in the hour. + +{% enddocs %} + +{% docs evm_unique_to_count %} + +The number of unique origin to addresses in the hour. + +{% enddocs %} + +{% docs evm_total_fees_native %} + +The sum of all fees in the hour, in the native fee currency. Note, Snowflake's native SUM() function may limit decimal precision. To view precise transaction fee values, please use the `tx_fee_precise` column in `core.fact_transactions`. + +{% enddocs %} + +{% docs evm_total_fees_usd %} + +The sum of all fees in the hour, in USD, rounded to 2 decimal places. + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql new file mode 100644 index 00000000..29f8672a --- /dev/null +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STATS, METRICS, CORE, HOURLY', + } } } +) }} + +SELECT + block_timestamp_hour, + block_number_min, + block_number_max, + block_count, + transaction_count, + transaction_count_success, + transaction_count_failed, + unique_from_count, + unique_to_count, + total_fees AS total_fees_native, + ROUND( + total_fees * LAST_VALUE( + p.price ignore nulls + ) over ( + ORDER BY + block_timestamp_hour rows unbounded preceding + ), + 2 + ) AS total_fees_usd, + core_metrics_hourly_id AS ez_core_metrics_hourly_id, + s.inserted_timestamp AS inserted_timestamp, + s.modified_timestamp AS modified_timestamp +FROM + {{ ref('silver_stats__core_metrics_hourly') }} + s + LEFT JOIN {{ ref('silver__hourly_prices_priority_eth') }} + p + ON s.block_timestamp_hour = p.hour \ No newline at end of file diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.yml b/models/gold/stats/stats__ez_core_metrics_hourly.yml new file mode 100644 index 00000000..8a61528c --- /dev/null +++ b/models/gold/stats/stats__ez_core_metrics_hourly.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: stats__ez_core_metrics_hourly + description: '{{ doc("evm_ez_core_metrics_hourly_table_doc") }}' + + columns: + - name: BLOCK_TIMESTAMP_HOUR + description: '{{ doc("evm_block_timestamp_hour") }}' + - name: BLOCK_NUMBER_MIN + description: '{{ doc("evm_block_number_min") }}' + - name: BLOCK_NUMBER_MAX + description: '{{ doc("evm_block_number_max") }}' + - name: BLOCK_COUNT + description: '{{ doc("evm_block_count") }}' + - name: TRANSACTION_COUNT + description: '{{ doc("evm_transaction_count") }}' + - name: TRANSACTION_COUNT_SUCCESS + description: '{{ doc("evm_transaction_count_success") }}' + - name: TRANSACTION_COUNT_FAILED + description: '{{ doc("evm_transaction_count_failed") }}' + - name: UNIQUE_FROM_COUNT + description: '{{ doc("evm_unique_from_count") }}' + - name: UNIQUE_TO_COUNT + description: '{{ doc("evm_unique_to_count") }}' + - name: TOTAL_FEES_NATIVE + description: '{{ doc("evm_total_fees_native") }}' + - name: TOTAL_FEES_USD + description: '{{ doc("evm_total_fees_usd") }}' + - name: EZ_CORE_METRICS_HOURLY_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql new file mode 100644 index 00000000..c5bd1b86 --- /dev/null +++ b/models/silver/stats/silver_stats__core_metrics_hourly.sql @@ -0,0 +1,66 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_timestamp_hour", + cluster_by = ['block_timestamp_hour::DATE'], + tags = ['curated'] +) }} + +SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_timestamp_hour, + MIN(block_number) AS block_number_min, + MAX(block_number) AS block_number_max, + COUNT( + DISTINCT block_number + ) AS block_count, + COUNT( + DISTINCT tx_hash + ) AS transaction_count, + COUNT( + DISTINCT CASE + WHEN tx_success THEN tx_hash + END + ) AS transaction_count_success, + COUNT( + DISTINCT CASE + WHEN NOT tx_success THEN tx_hash + END + ) AS transaction_count_failed, + COUNT( + DISTINCT from_address + ) AS unique_from_count, + COUNT( + DISTINCT to_address + ) AS unique_to_count, + SUM(tx_fee_precise) AS total_fees, + MAX(_inserted_timestamp) AS _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_timestamp_hour'] + ) }} AS core_metrics_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('silver__transactions') }} +WHERE + block_timestamp_hour < DATE_TRUNC( + 'hour', + CURRENT_TIMESTAMP + ) + +{% if is_incremental() %} +AND DATE_TRUNC( + 'hour', + _inserted_timestamp +) >= ( + SELECT + MAX(DATE_TRUNC('hour', _inserted_timestamp)) - INTERVAL '12 hours' + FROM + {{ this }} +) +{% endif %} +GROUP BY + 1 diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.yml b/models/silver/stats/silver_stats__core_metrics_hourly.yml new file mode 100644 index 00000000..df231f39 --- /dev/null +++ b/models/silver/stats/silver_stats__core_metrics_hourly.yml @@ -0,0 +1,84 @@ +version: 2 +models: + - name: silver_stats__core_metrics_hourly + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP_HOUR + columns: + - name: BLOCK_TIMESTAMP_HOUR + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: BLOCK_NUMBER_MIN + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_NUMBER_MAX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRANSACTION_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRANSACTION_COUNT_SUCCESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRANSACTION_COUNT_FAILED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: UNIQUE_FROM_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: UNIQUE_TO_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TOTAL_FEES + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - DECIMAL + - FLOAT + - NUMBER + - name: _INSERTED_TIMESTAMP + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1