From 6aded6d497eaf9805f7eea6dfb37d1ae210a819b Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 17 Mar 2025 16:30:20 -0600 Subject: [PATCH 01/16] + silver_stats --- models/silver/stats/silver_stats.yml | 73 +++++++++++++++++ .../silver_stats__core_metrics_hourly.sql | 78 +++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 models/silver/stats/silver_stats.yml create mode 100644 models/silver/stats/silver_stats__core_metrics_hourly.sql diff --git a/models/silver/stats/silver_stats.yml b/models/silver/stats/silver_stats.yml new file mode 100644 index 0000000..12763c0 --- /dev/null +++ b/models/silver/stats/silver_stats.yml @@ -0,0 +1,73 @@ +version: 2 +models: + - name: silver_stats__core_metrics_hourly + config: + contract: + enforced: true + tests: + - dbt_utils.sequential_values: + column_name: BLOCK_TIMESTAMP_HOUR + interval: 1 + config: + severity: error + error_if: ">0" + tags: ['test_recency'] + columns: + - name: BLOCK_TIMESTAMP_HOUR + data_type: TIMESTAMP_NTZ + tests: + - not_null: + tags: ['test_quality'] + - name: BLOCK_NUMBER_MIN + data_type: FLOAT + tests: + - not_null: + tags: ['test_quality'] + - name: BLOCK_NUMBER_MAX + data_type: FLOAT + tests: + - not_null: + tags: ['test_quality'] + - name: BLOCK_COUNT + data_type: NUMBER + tests: + - not_null: + tags: ['test_quality'] + - name: TRANSACTION_COUNT + data_type: NUMBER + tests: + - not_null: + tags: ['test_quality'] + - name: TRANSACTION_COUNT_SUCCESS + data_type: NUMBER + tests: + - not_null: + tags: ['test_quality'] + - name: TRANSACTION_COUNT_FAILED + data_type: NUMBER + tests: + - not_null: + tags: ['test_quality'] + - name: UNIQUE_SENDER_COUNT + data_type: NUMBER + tests: + - not_null: + tags: ['test_quality'] + - name: UNIQUE_PAYLOAD_FUNCTION_COUNT + data_type: NUMBER + tests: + - not_null: + tags: ['test_quality'] + - name: TOTAL_FEES + data_type: NUMBER + tests: + - not_null: + tags: ['test_quality'] + - name: CORE_METRICS_HOURLY_ID + data_type: VARCHAR + - name: INSERTED_TIMESTAMP + data_type: TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + data_type: TIMESTAMP_NTZ + - name: _INVOCATION_ID + data_type: VARCHAR diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql new file mode 100644 index 0000000..58b87cd --- /dev/null +++ b/models/silver/stats/silver_stats__core_metrics_hourly.sql @@ -0,0 +1,78 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_timestamp_hour", + cluster_by = ['block_timestamp_hour::DATE'], + tags = ['noncore'] +) }} + +{% if is_incremental() %} + {% set min_block_timestamp_hour_query %} + SELECT + MIN(DATE_TRUNC('hour', block_timestamp)) as block_timestamp_hour + FROM + {{ ref('silver__transactions') }} + WHERE + inserted_timestamp >= ( + SELECT + MAX(inserted_timestamp) + FROM + {{ this }} + ) + {% endset %} + + {% set min_block_timestamp_hour = run_query(min_block_timestamp_hour_query).columns[0].values()[0] %} +{% endif %} + +SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_timestamp_hour, + MIN(block_number) :: FLOAT AS block_number_min, + MAX(block_number) :: FLOAT AS block_number_max, + COUNT( + DISTINCT block_number + ) AS block_count, + COUNT( + DISTINCT tx_hash + ) AS transaction_count, + COUNT( + DISTINCT CASE + WHEN success THEN tx_hash + END + ) AS transaction_count_success, + COUNT( + DISTINCT CASE + WHEN NOT success THEN tx_hash + END + ) AS transaction_count_failed, + COUNT( + DISTINCT sender + ) AS unique_sender_count, + COUNT( + DISTINCT payload_function + ) AS unique_payload_function_count, + SUM(COALESCE(gas_unit_price,0) * gas_used) AS total_fees, + {{ dbt_utils.generate_surrogate_key( + ['block_timestamp_hour'] + ) }} AS core_metrics_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('silver__transactions') }} +WHERE + block_timestamp_hour < DATE_TRUNC( + 'hour', + CURRENT_TIMESTAMP + ) + +{% if is_incremental() %} +AND DATE_TRUNC( + 'hour', + block_timestamp +) >= '{{ min_block_timestamp_hour }}' +{% endif %} +GROUP BY + 1 \ No newline at end of file From a58df58f947045c494afe5b2a22ad9f4690bd11a Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 18 Mar 2025 15:13:14 -0600 Subject: [PATCH 02/16] added stats gold, still need incrementality --- models/gold/stats/gold_stats.yml | 37 +++++++++++++++++ .../stats/stats__ez_core_metrics_hourly.sql | 40 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 models/gold/stats/gold_stats.yml create mode 100644 models/gold/stats/stats__ez_core_metrics_hourly.sql diff --git a/models/gold/stats/gold_stats.yml b/models/gold/stats/gold_stats.yml new file mode 100644 index 0000000..779f6e7 --- /dev/null +++ b/models/gold/stats/gold_stats.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: stats__ez_core_metrics_hourly + description: '{{ doc("ez_core_metrics_hourly_table_doc") }}' + + columns: + - name: BLOCK_TIMESTAMP_HOUR + description: '{{ doc("block_timestamp_hour") }}' + - name: BLOCK_NUMBER_MIN + description: '{{ doc("block_number_min") }}' + - name: BLOCK_NUMBER_MAX + description: '{{ doc("block_number_max") }}' + - name: BLOCK_COUNT + description: '{{ doc("block_count") }}' + - name: TRANSACTION_COUNT + description: '{{ doc("transaction_count") }}' + - name: TRANSACTION_COUNT_SUCCESS + description: '{{ doc("transaction_count_success") }}' + - name: TRANSACTION_COUNT_FAILED + description: '{{ doc("transaction_count_failed") }}' + - name: UNIQUE_SENDER_COUNT + description: '{{ doc("unique_sender_count") }}' + - name: UNIQUE_PAYLOAD_FUNCTION_COUNT + description: '{{ doc("unique_payload_function_count") }}' + - name: TOTAL_FEES_NATIVE + description: '{{ doc("total_fees_native") }}' + - name: TOTAL_FEES_USD + description: '{{ doc("total_fees_usd") }}' + tests: + - not_null: + tags: ['test_quality'] + - name: EZ_CORE_METRICS_HOURLY_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql new file mode 100644 index 0000000..5ee7a47 --- /dev/null +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -0,0 +1,40 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_timestamp_hour", + cluster_by = ['block_timestamp_hour::DATE'], + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STATS, METRICS, CORE, HOURLY', + } } }, + tags = ['noncore'] +) }} + +SELECT + block_timestamp_hour, + block_number_min, + block_number_max, + block_count, + transaction_count, + transaction_count_success, + transaction_count_failed, + unique_sender_count, + unique_payload_function_count, + total_fees AS total_fees_native, + ROUND( + (total_fees / pow( + 10, + 8 + )) * p.price, + 2 + ) AS total_fees_usd, + core_metrics_hourly_id AS ez_core_metrics_hourly_id, + s.inserted_timestamp AS inserted_timestamp, + s.modified_timestamp AS modified_timestamp +FROM + {{ ref('silver_stats__core_metrics_hourly') }} + s + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON s.block_timestamp_hour = p.hour +WHERE + p.is_native + AND p.price IS NOT NULL \ No newline at end of file From a748a0917ed424034b88ec6ca243c82d7cd24da4 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 18 Mar 2025 15:37:22 -0600 Subject: [PATCH 03/16] added incremental logic, switched silver to view --- .../stats/stats__ez_core_metrics_hourly.sql | 7 +++- .../silver_stats__core_metrics_hourly.sql | 34 +++---------------- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql index 5ee7a47..539ea44 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -37,4 +37,9 @@ FROM ON s.block_timestamp_hour = p.hour WHERE p.is_native - AND p.price IS NOT NULL \ No newline at end of file + {% if is_incremental() %} + AND s.block_timestamp_hour >= ( + SELECT + MAX(block_timestamp_hour) + FROM {{ this }}) + {% endif %} \ No newline at end of file diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql index 58b87cd..0578f31 100644 --- a/models/silver/stats/silver_stats__core_metrics_hourly.sql +++ b/models/silver/stats/silver_stats__core_metrics_hourly.sql @@ -1,29 +1,12 @@ {{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = "block_timestamp_hour", - cluster_by = ['block_timestamp_hour::DATE'], + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STATS, METRICS, CORE, HOURLY', + } } }, tags = ['noncore'] ) }} -{% if is_incremental() %} - {% set min_block_timestamp_hour_query %} - SELECT - MIN(DATE_TRUNC('hour', block_timestamp)) as block_timestamp_hour - FROM - {{ ref('silver__transactions') }} - WHERE - inserted_timestamp >= ( - SELECT - MAX(inserted_timestamp) - FROM - {{ this }} - ) - {% endset %} - - {% set min_block_timestamp_hour = run_query(min_block_timestamp_hour_query).columns[0].values()[0] %} -{% endif %} - SELECT DATE_TRUNC( 'hour', @@ -67,12 +50,5 @@ WHERE 'hour', CURRENT_TIMESTAMP ) - -{% if is_incremental() %} -AND DATE_TRUNC( - 'hour', - block_timestamp -) >= '{{ min_block_timestamp_hour }}' -{% endif %} GROUP BY 1 \ No newline at end of file From b0db1e5c83ee45e4017f57fab394b084fb59668f Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Wed, 19 Mar 2025 10:08:07 -0600 Subject: [PATCH 04/16] rm files mistakenly committed from price --- models/gold/price/gold_price.yml | 115 --------- .../gold/price/price__dim_asset_metadata.sql | 19 -- .../gold/price/price__ez_asset_metadata.sql | 35 --- models/gold/price/price__ez_prices_hourly.sql | 39 ---- .../price/price__fact_prices_ohlc_hourly.sql | 19 -- ...silver__complete_native_asset_metadata.sql | 38 --- .../price/silver__complete_native_prices.sql | 42 ---- ...lver__complete_provider_asset_metadata.sql | 38 --- .../silver__complete_provider_prices.sql | 47 ---- .../silver__complete_token_asset_metadata.sql | 43 ---- .../price/silver__complete_token_prices.sql | 48 ---- models/silver/price/silver_price.yml | 218 ------------------ 12 files changed, 701 deletions(-) delete mode 100644 models/gold/price/gold_price.yml delete mode 100644 models/gold/price/price__dim_asset_metadata.sql delete mode 100644 models/gold/price/price__ez_asset_metadata.sql delete mode 100644 models/gold/price/price__ez_prices_hourly.sql delete mode 100644 models/gold/price/price__fact_prices_ohlc_hourly.sql delete mode 100644 models/silver/price/silver__complete_native_asset_metadata.sql delete mode 100644 models/silver/price/silver__complete_native_prices.sql delete mode 100644 models/silver/price/silver__complete_provider_asset_metadata.sql delete mode 100644 models/silver/price/silver__complete_provider_prices.sql delete mode 100644 models/silver/price/silver__complete_token_asset_metadata.sql delete mode 100644 models/silver/price/silver__complete_token_prices.sql delete mode 100644 models/silver/price/silver_price.yml diff --git a/models/gold/price/gold_price.yml b/models/gold/price/gold_price.yml deleted file mode 100644 index 465de68..0000000 --- a/models/gold/price/gold_price.yml +++ /dev/null @@ -1,115 +0,0 @@ -version: 2 -models: - - name: price__dim_asset_metadata - description: '{{ doc("prices_dim_asset_metadata_table_doc") }}' - columns: - - name: PROVIDER - description: '{{ doc("prices_provider")}}' - - name: ASSET_ID - description: '{{ doc("prices_asset_id") }}' - - name: NAME - description: '{{ doc("prices_name") }}' - - name: SYMBOL - description: '{{ doc("prices_symbol") }}' - - name: TOKEN_ADDRESS - description: '{{ doc("prices_token_address_evm") }}' - - name: BLOCKCHAIN - description: '{{ doc("prices_blockchain") }}' - - name: BLOCKCHAIN_ID - description: '{{ doc("prices_blockchain_id") }}' - - name: DIM_ASSET_METADATA_ID - description: '{{ doc("pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' - - - name: price__ez_asset_metadata - description: '{{ doc("prices_ez_asset_metadata_table_doc") }}' - columns: - - name: ASSET_ID - description: '{{ doc("prices_asset_id") }}' - - name: NAME - description: '{{ doc("prices_name") }}' - - name: SYMBOL - description: '{{ doc("prices_symbol") }}' - - name: TOKEN_ADDRESS - description: '{{ doc("prices_token_address_evm") }}' - - name: BLOCKCHAIN - description: '{{ doc("prices_blockchain") }}' - - name: DECIMALS - description: '{{ doc("prices_decimals") }}' - - name: IS_NATIVE - description: '{{ doc("prices_is_native") }}' - - name: IS_DEPRECATED - description: '{{ doc("prices_is_deprecated") }}' - - name: EZ_ASSET_METADATA_ID - description: '{{ doc("pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' - - - name: price__ez_prices_hourly - description: '{{ doc("prices_ez_prices_hourly_table_doc") }}' - tests: - - dbt_utils.recency: - datepart: hour - field: MODIFIED_TIMESTAMP - interval: 6 - severity: error - tags: ['test_recency'] - columns: - - name: HOUR - description: '{{ doc("prices_hour")}}' - - name: TOKEN_ADDRESS - description: '{{ doc("prices_token_address_evm") }}' - - name: SYMBOL - description: '{{ doc("prices_symbol") }}' - - name: BLOCKCHAIN - description: '{{ doc("prices_blockchain") }}' - - name: DECIMALS - description: '{{ doc("prices_decimals") }}' - - name: PRICE - description: '{{ doc("prices_price") }}' - - name: IS_NATIVE - description: '{{ doc("prices_is_native") }}' - - name: IS_IMPUTED - description: '{{ doc("prices_is_imputed") }}' - - name: IS_DEPRECATED - description: '{{ doc("prices_is_deprecated") }}' - - name: EZ_PRICES_HOURLY_ID - description: '{{ doc("pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' - - - name: price__fact_prices_ohlc_hourly - description: '{{ doc("prices_fact_prices_ohlc_hourly_table_doc") }}' - tests: - - dbt_utils.recency: - datepart: hour - field: MODIFIED_TIMESTAMP - interval: 6 - severity: error - tags: ['test_recency'] - columns: - - name: HOUR - description: '{{ doc("prices_hour")}}' - - name: ASSET_ID - description: '{{ doc("prices_asset_id") }}' - - name: OPEN - description: '{{ doc("prices_open") }}' - - name: HIGH - description: '{{ doc("prices_high") }}' - - name: LOW - description: '{{ doc("prices_low") }}' - - name: CLOSE - description: '{{ doc("prices_close") }}' - - name: FACT_PRICES_OHLC_HOURLY_ID - description: '{{ doc("pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/price/price__dim_asset_metadata.sql b/models/gold/price/price__dim_asset_metadata.sql deleted file mode 100644 index 71cf674..0000000 --- a/models/gold/price/price__dim_asset_metadata.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, "columns": true }, - tags = ['noncore'] -) }} - -SELECT - token_address, - asset_id, - symbol, - NAME, - platform AS blockchain, - platform_id AS blockchain_id, - provider, - inserted_timestamp, - modified_timestamp, - complete_provider_asset_metadata_id AS dim_asset_metadata_id -FROM - {{ ref('silver__complete_provider_asset_metadata') }} diff --git a/models/gold/price/price__ez_asset_metadata.sql b/models/gold/price/price__ez_asset_metadata.sql deleted file mode 100644 index eacf895..0000000 --- a/models/gold/price/price__ez_asset_metadata.sql +++ /dev/null @@ -1,35 +0,0 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, "columns": true }, - tags = ['noncore'] -) }} - -SELECT - token_address, - asset_id, - symbol, - NAME, - decimals, - blockchain, - TRUE AS is_native, - is_deprecated, - inserted_timestamp, - modified_timestamp, - complete_token_asset_metadata_id AS ez_asset_metadata_id -FROM - {{ ref('silver__complete_token_asset_metadata') }} -UNION ALL -SELECT - NULL AS token_address, - asset_id, - symbol, - NAME, - decimals, - blockchain, - TRUE AS is_native, - is_deprecated, - inserted_timestamp, - modified_timestamp, - complete_native_asset_metadata_id AS ez_asset_metadata_id -FROM - {{ ref('silver__complete_native_asset_metadata') }} diff --git a/models/gold/price/price__ez_prices_hourly.sql b/models/gold/price/price__ez_prices_hourly.sql deleted file mode 100644 index d816d7d..0000000 --- a/models/gold/price/price__ez_prices_hourly.sql +++ /dev/null @@ -1,39 +0,0 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, "columns": true }, - tags = ['noncore'] -) }} - -SELECT - HOUR, - token_address, - symbol, - NAME, - decimals, - price, - blockchain, - FALSE AS is_native, - is_imputed, - is_deprecated, - inserted_timestamp, - modified_timestamp, - complete_token_prices_id AS ez_prices_hourly_id -FROM - {{ ref('silver__complete_token_prices') }} -UNION ALL -SELECT - HOUR, - NULL AS token_address, - symbol, - NAME, - decimals, - price, - blockchain, - TRUE AS is_native, - is_imputed, - is_deprecated, - inserted_timestamp, - modified_timestamp, - complete_native_prices_id AS ez_prices_hourly_id -FROM - {{ ref('silver__complete_native_prices') }} diff --git a/models/gold/price/price__fact_prices_ohlc_hourly.sql b/models/gold/price/price__fact_prices_ohlc_hourly.sql deleted file mode 100644 index 552efa9..0000000 --- a/models/gold/price/price__fact_prices_ohlc_hourly.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, "columns": true }, - tags = ['noncore'] -) }} - -SELECT - asset_id, - recorded_hour AS HOUR, - OPEN, - high, - low, - CLOSE, - provider, - inserted_timestamp, - modified_timestamp, - complete_provider_prices_id AS fact_prices_ohlc_hourly_id -FROM - {{ ref('silver__complete_provider_prices') }} diff --git a/models/silver/price/silver__complete_native_asset_metadata.sql b/models/silver/price/silver__complete_native_asset_metadata.sql deleted file mode 100644 index be2e8f6..0000000 --- a/models/silver/price/silver__complete_native_asset_metadata.sql +++ /dev/null @@ -1,38 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_native_asset_metadata_id', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, symbol, name),SUBSTRING(asset_id, symbol, name)", - tags = ['noncore'] -) }} - -SELECT - asset_id, - symbol, - NAME, - decimals, - blockchain, - is_deprecated, - provider, - source, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp, - complete_native_asset_metadata_id, - _invocation_id -FROM - {{ ref( - 'bronze__complete_native_asset_metadata' - ) }} - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} diff --git a/models/silver/price/silver__complete_native_prices.sql b/models/silver/price/silver__complete_native_prices.sql deleted file mode 100644 index 3c792ac..0000000 --- a/models/silver/price/silver__complete_native_prices.sql +++ /dev/null @@ -1,42 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_native_prices_id', - cluster_by = ['hour::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, symbol, name),SUBSTRING(asset_id, symbol, name)", - tags = ['noncore'] -) }} - -SELECT - HOUR, - asset_id, - symbol, - NAME, - decimals, - price, - blockchain, - is_imputed, - is_deprecated, - provider, - source, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp, - complete_native_prices_id, - _invocation_id -FROM - {{ ref( - 'bronze__complete_native_prices' - ) }} - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} diff --git a/models/silver/price/silver__complete_provider_asset_metadata.sql b/models/silver/price/silver__complete_provider_asset_metadata.sql deleted file mode 100644 index de1ef9d..0000000 --- a/models/silver/price/silver__complete_provider_asset_metadata.sql +++ /dev/null @@ -1,38 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_provider_asset_metadata_id', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, token_address, symbol, name),SUBSTRING(asset_id, token_address, symbol, name)", - tags = ['noncore'] -) }} - -SELECT - asset_id, - token_address, - NAME, - symbol, - platform, - platform_id, - provider, - source, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp, - complete_provider_asset_metadata_id, - _invocation_id -FROM - {{ ref( - 'bronze__complete_provider_asset_metadata' - ) }} - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} diff --git a/models/silver/price/silver__complete_provider_prices.sql b/models/silver/price/silver__complete_provider_prices.sql deleted file mode 100644 index bc53e07..0000000 --- a/models/silver/price/silver__complete_provider_prices.sql +++ /dev/null @@ -1,47 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_provider_prices_id', - cluster_by = ['recorded_hour::DATE','provider'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id),SUBSTRING(asset_id)", - tags = ['noncore'] -) }} - -SELECT - p.asset_id, - recorded_hour, - OPEN, - high, - low, - CLOSE, - p.provider, - p.source, - p._inserted_timestamp, - p.inserted_timestamp, - p.modified_timestamp, - p.complete_provider_prices_id, - p._invocation_id -FROM - {{ ref( - 'bronze__complete_provider_prices' - ) }} - p - INNER JOIN {{ ref('bronze__complete_provider_asset_metadata') }} - m - ON p.asset_id = m.asset_id - -{% if is_incremental() %} -WHERE - p.modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY p.asset_id, recorded_hour, p.provider -ORDER BY - p.modified_timestamp DESC)) = 1 diff --git a/models/silver/price/silver__complete_token_asset_metadata.sql b/models/silver/price/silver__complete_token_asset_metadata.sql deleted file mode 100644 index 7e525b0..0000000 --- a/models/silver/price/silver__complete_token_asset_metadata.sql +++ /dev/null @@ -1,43 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_token_asset_metadata_id', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, token_address, symbol, name),SUBSTRING(asset_id, token_address, symbol, name)", - tags = ['noncore'] -) }} - -SELECT - LOWER( - A.token_address - ) AS token_address, - asset_id, - symbol, - NAME, - decimals, - blockchain, - blockchain_name, - blockchain_id, - is_deprecated, - provider, - source, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp, - complete_token_asset_metadata_id, - _invocation_id -FROM - {{ ref( - 'bronze__complete_token_asset_metadata' - ) }} A - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} diff --git a/models/silver/price/silver__complete_token_prices.sql b/models/silver/price/silver__complete_token_prices.sql deleted file mode 100644 index 77d3630..0000000 --- a/models/silver/price/silver__complete_token_prices.sql +++ /dev/null @@ -1,48 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_token_prices_id', - cluster_by = ['hour::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, token_address, symbol, name),SUBSTRING(asset_id, token_address, symbol, name)", - tags = ['noncore'] -) }} - -SELECT - HOUR, - LOWER( - p.token_address - ) AS token_address, - asset_id, - symbol, - NAME, - decimals, - price, - blockchain, - blockchain_name, - blockchain_id, - is_imputed, - is_deprecated, - provider, - source, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp, - complete_token_prices_id, - _invocation_id -FROM - {{ ref( - 'bronze__complete_token_prices' - ) }} - p - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} diff --git a/models/silver/price/silver_price.yml b/models/silver/price/silver_price.yml deleted file mode 100644 index f66f90f..0000000 --- a/models/silver/price/silver_price.yml +++ /dev/null @@ -1,218 +0,0 @@ -version: 2 -models: - - name: silver__complete_native_asset_metadata - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - SYMBOL - tags: ['test_quality'] - columns: - - name: PROVIDER - tests: - - not_null: - tags: ['test_quality'] - - name: SYMBOL - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCKCHAIN - tests: - - not_null: - tags: ['test_quality'] - - name: MODIFIED_TIMESTAMP - tests: - - not_null: - tags: ['test_quality'] - - name: COMPLETE_NATIVE_ASSET_METADATA_ID - tests: - - unique: - tags: ['test_quality'] - - - name: silver__complete_native_prices - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - HOUR - - SYMBOL - tags: ['test_quality'] - - columns: - - name: HOUR - tests: - - not_null: - tags: ['test_quality'] - - name: SYMBOL - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCKCHAIN - tests: - - not_null: - tags: ['test_quality'] - - name: PROVIDER - tests: - - not_null: - tags: ['test_quality'] - - name: PRICE - tests: - - not_null: - tags: ['test_quality'] - - name: IS_IMPUTED - tests: - - not_null: - tags: ['test_quality'] - - name: _INSERTED_TIMESTAMP - tests: - - not_null: - tags: ['test_quality'] - - name: MODIFIED_TIMESTAMP - tests: - - not_null: - tags: ['test_quality'] - - name: COMPLETE_NATIVE_PRICES_ID - tests: - - unique: - tags: ['test_quality'] - - - name: silver__complete_provider_asset_metadata - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - ASSET_ID - - TOKEN_ADDRESS - - NAME - - SYMBOL - - PLATFORM - - PLATFORM_ID - - PROVIDER - tags: ['test_quality'] - columns: - - name: PROVIDER - tests: - - not_null: - tags: ['test_quality'] - - name: ASSET_ID - tests: - - not_null: - tags: ['test_quality'] - - name: MODIFIED_TIMESTAMP - tests: - - not_null: - tags: ['test_quality'] - - name: COMPLETE_PROVIDER_ASSET_METADATA_ID - tests: - - unique: - tags: ['test_quality'] - - - name: silver__complete_provider_prices - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - ASSET_ID - - RECORDED_HOUR - - PROVIDER - tags: ['test_quality'] - columns: - - name: PROVIDER - tests: - - not_null: - tags: ['test_quality'] - - name: ASSET_ID - tests: - - not_null: - tags: ['test_quality'] - - name: RECORDED_HOUR - tests: - - not_null: - tags: ['test_quality'] - - name: MODIFIED_TIMESTAMP - tests: - - not_null: - tags: ['test_quality'] - - name: COMPLETE_PROVIDER_PRICES_ID - tests: - - unique: - tags: ['test_quality'] - - - name: silver__complete_token_asset_metadata - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - TOKEN_ADDRESS - - BLOCKCHAIN - tags: ['test_quality'] - columns: - - name: PROVIDER - tests: - - not_null: - tags: ['test_quality'] - - name: TOKEN_ADDRESS - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCKCHAIN - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCKCHAIN_ID - tests: - - not_null: - tags: ['test_quality'] - - name: MODIFIED_TIMESTAMP - tests: - - not_null: - tags: ['test_quality'] - - name: COMPLETE_TOKEN_ASSET_METADATA_ID - tests: - - unique: - tags: ['test_quality'] - - - name: silver__complete_token_prices - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - HOUR - - TOKEN_ADDRESS - - BLOCKCHAIN - tags: ['test_quality'] - columns: - - name: HOUR - tests: - - not_null: - tags: ['test_quality'] - - name: TOKEN_ADDRESS - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCKCHAIN - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCKCHAIN_ID - tests: - - not_null: - tags: ['test_quality'] - - name: PROVIDER - tests: - - not_null: - tags: ['test_quality'] - - name: PRICE - tests: - - not_null: - tags: ['test_quality'] - - name: IS_IMPUTED - tests: - - not_null: - tags: ['test_quality'] - - name: _INSERTED_TIMESTAMP - tests: - - not_null: - tags: ['test_quality'] - - name: MODIFIED_TIMESTAMP - tests: - - not_null: - tags: ['test_quality'] - - name: COMPLETE_TOKEN_PRICES_ID - tests: - - unique: - tags: ['test_quality'] \ No newline at end of file From cb16d1289d95bc0ff83163d6e90ab2d2587642a1 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Wed, 19 Mar 2025 10:10:26 -0600 Subject: [PATCH 05/16] removed more files --- ...bronze__complete_native_asset_metadata.sql | 21 ---------- .../price/bronze__complete_native_prices.sql | 28 ------------- ...onze__complete_provider_asset_metadata.sql | 26 ------------ .../bronze__complete_provider_prices.sql | 24 ----------- .../bronze__complete_token_asset_metadata.sql | 30 -------------- .../price/bronze__complete_token_prices.sql | 33 --------------- models/sources.yml | 40 ------------------- 7 files changed, 202 deletions(-) delete mode 100644 models/bronze/price/bronze__complete_native_asset_metadata.sql delete mode 100644 models/bronze/price/bronze__complete_native_prices.sql delete mode 100644 models/bronze/price/bronze__complete_provider_asset_metadata.sql delete mode 100644 models/bronze/price/bronze__complete_provider_prices.sql delete mode 100644 models/bronze/price/bronze__complete_token_asset_metadata.sql delete mode 100644 models/bronze/price/bronze__complete_token_prices.sql delete mode 100644 models/sources.yml diff --git a/models/bronze/price/bronze__complete_native_asset_metadata.sql b/models/bronze/price/bronze__complete_native_asset_metadata.sql deleted file mode 100644 index 1eac19d..0000000 --- a/models/bronze/price/bronze__complete_native_asset_metadata.sql +++ /dev/null @@ -1,21 +0,0 @@ -SELECT - asset_id, - symbol, - NAME, - decimals, - blockchain, - is_deprecated, - provider, - source, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp, - complete_native_asset_metadata_id, - _invocation_id -FROM - {{ source( - 'crosschain_silver', - 'complete_native_asset_metadata' - ) }} -WHERE - blockchain = 'movement' \ No newline at end of file diff --git a/models/bronze/price/bronze__complete_native_prices.sql b/models/bronze/price/bronze__complete_native_prices.sql deleted file mode 100644 index bc27af6..0000000 --- a/models/bronze/price/bronze__complete_native_prices.sql +++ /dev/null @@ -1,28 +0,0 @@ -{{ config ( - materialized = 'view' -) }} - -SELECT - HOUR, - asset_id, - symbol, - NAME, - decimals, - price, - blockchain, - is_imputed, - is_deprecated, - provider, - source, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp, - complete_native_prices_id, - _invocation_id -FROM - {{ source( - 'crosschain_silver', - 'complete_native_prices' - ) }} -WHERE - blockchain = 'movement' diff --git a/models/bronze/price/bronze__complete_provider_asset_metadata.sql b/models/bronze/price/bronze__complete_provider_asset_metadata.sql deleted file mode 100644 index 15bd2f2..0000000 --- a/models/bronze/price/bronze__complete_provider_asset_metadata.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config ( - materialized = 'view' -) }} - -SELECT - asset_id, - 'movement' AS token_address, - NAME, - symbol, - 'movement' AS platform, - platform_id, - provider, - source, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp, - complete_provider_asset_metadata_id, - _invocation_id -FROM - {{ source( - 'crosschain_silver', - 'complete_provider_asset_metadata' - ) }} -WHERE - asset_id = 'movement' - AND token_address IS NULL diff --git a/models/bronze/price/bronze__complete_provider_prices.sql b/models/bronze/price/bronze__complete_provider_prices.sql deleted file mode 100644 index c726661..0000000 --- a/models/bronze/price/bronze__complete_provider_prices.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config ( - materialized = 'view' -) }} - -SELECT - asset_id, - recorded_hour, - OPEN, - high, - low, - CLOSE, - provider, - source, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp, - complete_provider_prices_id, - _invocation_id -FROM - {{ source( - 'crosschain_silver', - 'complete_provider_prices' - ) }} - -- prices for all ids diff --git a/models/bronze/price/bronze__complete_token_asset_metadata.sql b/models/bronze/price/bronze__complete_token_asset_metadata.sql deleted file mode 100644 index 2ad08d9..0000000 --- a/models/bronze/price/bronze__complete_token_asset_metadata.sql +++ /dev/null @@ -1,30 +0,0 @@ -{{ config ( - materialized = 'view' -) }} - -SELECT - 'movement' AS token_address, - asset_id, - symbol, - 'Movement' AS NAME, - decimals, - 'movement' AS blockchain, - 'movement' AS blockchain_name, - blockchain_id, - is_deprecated, - provider, - source, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp, - complete_token_asset_metadata_id, - _invocation_id -FROM - {{ source( - 'crosschain_silver', - 'complete_token_asset_metadata' - ) }} -WHERE - asset_id = 'movement' qualify(ROW_NUMBER() over(PARTITION BY asset_id -ORDER BY - is_deprecated, blockchain_id) = 1) diff --git a/models/bronze/price/bronze__complete_token_prices.sql b/models/bronze/price/bronze__complete_token_prices.sql deleted file mode 100644 index d929e87..0000000 --- a/models/bronze/price/bronze__complete_token_prices.sql +++ /dev/null @@ -1,33 +0,0 @@ -{{ config ( - materialized = 'view' -) }} - -SELECT - HOUR, - 'movement' AS token_address, - asset_id, - symbol, - 'Movement' AS NAME, - decimals, - price, - 'movement' AS blockchain, - 'movement' AS blockchain_name, - blockchain_id, - is_imputed, - is_deprecated, - provider, - source, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp, - complete_token_prices_id, - _invocation_id -FROM - {{ source( - 'crosschain_silver', - 'complete_token_prices' - ) }} -WHERE - asset_id = 'movement' qualify(ROW_NUMBER() over(PARTITION BY asset_id -ORDER BY - is_deprecated, blockchain_id) = 1) diff --git a/models/sources.yml b/models/sources.yml deleted file mode 100644 index a5da39e..0000000 --- a/models/sources.yml +++ /dev/null @@ -1,40 +0,0 @@ -version: 2 - -sources: - - name: crosschain - database: "{{ 'crosschain' if target.database == 'MOVEMENT' else 'crosschain_dev' }}" - schema: core - tables: - - name: dim_date_hours - - name: address_tags - - name: dim_dates - - name: crosschain_silver - database: "{{ 'crosschain' if target.database == 'MOVEMENT' else 'crosschain_dev' }}" - schema: silver - tables: - - name: number_sequence - - name: labels_combined - - name: complete_token_asset_metadata - - name: complete_token_prices - - name: complete_provider_asset_metadata - - name: complete_provider_prices - - name: complete_native_asset_metadata - - name: complete_native_prices - - name: bronze_streamline - database: streamline - schema: | - {{ "MOVEMENT_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "MOVEMENT" }} - tables: - - name: blocks_tx - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 4, period: hour} - - name: transactions - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 4, period: hour} - - name: github_actions - database: movement - schema: github_actions - tables: - - name: workflows From 22ea481028d46f5b03def986228801ccecd5fea2 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Wed, 19 Mar 2025 10:12:11 -0600 Subject: [PATCH 06/16] more changes --- .github/workflows/dbt_run_noncore.yml | 44 --------------------------- models/sources.yml | 40 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 44 deletions(-) delete mode 100644 .github/workflows/dbt_run_noncore.yml create mode 100644 models/sources.yml diff --git a/.github/workflows/dbt_run_noncore.yml b/.github/workflows/dbt_run_noncore.yml deleted file mode 100644 index 77b234a..0000000 --- a/.github/workflows/dbt_run_noncore.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_incremental_core -run-name: dbt_run_incremental_core - -on: - workflow_dispatch: - schedule: - - cron: "25,55 0/4 * * *" - -env: - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "${{ vars.PYTHON_VERSION }}" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -m movement_models,tag:core \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml new file mode 100644 index 0000000..a5da39e --- /dev/null +++ b/models/sources.yml @@ -0,0 +1,40 @@ +version: 2 + +sources: + - name: crosschain + database: "{{ 'crosschain' if target.database == 'MOVEMENT' else 'crosschain_dev' }}" + schema: core + tables: + - name: dim_date_hours + - name: address_tags + - name: dim_dates + - name: crosschain_silver + database: "{{ 'crosschain' if target.database == 'MOVEMENT' else 'crosschain_dev' }}" + schema: silver + tables: + - name: number_sequence + - name: labels_combined + - name: complete_token_asset_metadata + - name: complete_token_prices + - name: complete_provider_asset_metadata + - name: complete_provider_prices + - name: complete_native_asset_metadata + - name: complete_native_prices + - name: bronze_streamline + database: streamline + schema: | + {{ "MOVEMENT_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "MOVEMENT" }} + tables: + - name: blocks_tx + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 4, period: hour} + - name: transactions + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 4, period: hour} + - name: github_actions + database: movement + schema: github_actions + tables: + - name: workflows From 21a2b804022831cf326f32d1767ccd5ccdb18ce2 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Wed, 19 Mar 2025 10:14:09 -0600 Subject: [PATCH 07/16] undid changes in sources as they're covered by another pr --- models/sources.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/models/sources.yml b/models/sources.yml index a5da39e..f8f61f9 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -13,13 +13,6 @@ sources: schema: silver tables: - name: number_sequence - - name: labels_combined - - name: complete_token_asset_metadata - - name: complete_token_prices - - name: complete_provider_asset_metadata - - name: complete_provider_prices - - name: complete_native_asset_metadata - - name: complete_native_prices - name: bronze_streamline database: streamline schema: | From 8c7a989553af824d790a2f48b14eb8b8668cdb87 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Wed, 19 Mar 2025 10:14:49 -0600 Subject: [PATCH 08/16] one last md file --- models/descriptions/prices.md | 143 ---------------------------------- 1 file changed, 143 deletions(-) delete mode 100644 models/descriptions/prices.md diff --git a/models/descriptions/prices.md b/models/descriptions/prices.md deleted file mode 100644 index 60d4e96..0000000 --- a/models/descriptions/prices.md +++ /dev/null @@ -1,143 +0,0 @@ -{% docs prices_dim_asset_metadata_table_doc %} - -A comprehensive dimensional table holding asset metadata and other relevant details pertaining to each id, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this dimensional model. If you are interested in using a curated data set instead, please utilize ez_asset_metadata. - -{% enddocs %} - -{% docs prices_ez_asset_metadata_table_doc %} - -A convenience table holding prioritized asset metadata and other relevant details pertaining to each token_address and native asset. This data set is highly curated and contains metadata for one unique asset per blockchain. - -{% enddocs %} - -{% docs prices_fact_prices_ohlc_hourly_table_doc %} - -A comprehensive fact table holding id and provider specific open, high, low, close hourly prices, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this fact based model. If you are interested in using a curated data set instead, please utilize ez_prices_hourly. - -{% enddocs %} - -{% docs prices_ez_prices_hourly_table_doc %} - -A convenience table for determining token prices by address and blockchain, and native asset prices by symbol and blockchain. This data set is highly curated and contains metadata for one price per hour per unique asset and blockchain. - -{% enddocs %} - -{% docs prices_provider %} - -The provider or source of the data. - -{% enddocs %} - -{% docs prices_asset_id %} - -The unique identifier representing the asset. - -{% enddocs %} - -{% docs prices_name %} - -The name of asset. - -{% enddocs %} - -{% docs prices_symbol %} - -The symbol of asset. - -{% enddocs %} - -{% docs prices_token_address %} - -The specific address representing the asset on a specific platform. This will be NULL if referring to a native asset. - -{% enddocs %} - -{% docs prices_token_address_evm %} - -The specific address representing the asset on a specific platform. This will be NULL if referring to a native asset. The case (upper / lower) may or may not be specified within the `dim_asset_metadata` table, as this column is raw and not transformed, coming directly from the provider APIs. However, in the `ez_` views, it will be lowercase by default for all EVMs. - -{% enddocs %} - -{% docs prices_blockchain %} - -The Blockchain, Network, or Platform for this asset. - -{% enddocs %} - -{% docs prices_blockchain_id %} - -The unique identifier of the Blockchain, Network, or Platform for this asset. - -{% enddocs %} - -{% docs prices_decimals %} - -The number of decimals for the asset. May be NULL. - -{% enddocs %} - -{% docs prices_is_native %} - -A flag indicating assets native to the respective blockchain. - -{% enddocs %} - -{% docs prices_is_deprecated %} - -A flag indicating if the asset is deprecated or no longer supported by the provider. - -{% enddocs %} - -{% docs prices_id_deprecation %} - -Deprecating soon! Please use the `asset_id` column instead. - -{% enddocs %} - -{% docs prices_decimals_deprecation %} - -Deprecating soon! Please use the decimals column in `ez_asset_metadata` or join in `dim_contracts` instead. - -{% enddocs %} - -{% docs prices_hour %} - -Hour that the price was recorded at. - -{% enddocs %} - -{% docs prices_price %} - -Closing price of the recorded hour in USD. - -{% enddocs %} - -{% docs prices_is_imputed %} - -A flag indicating if the price was imputed, or derived, from the last arriving record. This is generally used for tokens with low-liquidity or inconsistent reporting. - -{% enddocs %} - -{% docs prices_open %} - -Opening price of the recorded hour in USD. - -{% enddocs %} - -{% docs prices_high %} - -Highest price of the recorded hour in USD - -{% enddocs %} - -{% docs prices_low %} - -Lowest price of the recorded hour in USD - -{% enddocs %} - -{% docs prices_close %} - -Closing price of the recorded hour in USD - -{% enddocs %} \ No newline at end of file From 66875b4b6057dba6a84204d1278b1fecc194504e Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Thu, 20 Mar 2025 13:14:37 -0600 Subject: [PATCH 09/16] added core models similar to aptos --- .../_observability/silver_observability.yml | 123 +++++++++++++ ...ver_observability__blocks_completeness.sql | 168 ++++++++++++++++++ ...servability__transactions_completeness.sql | 137 ++++++++++++++ 3 files changed, 428 insertions(+) create mode 100644 models/silver/_observability/silver_observability.yml create mode 100644 models/silver/_observability/silver_observability__blocks_completeness.sql create mode 100644 models/silver/_observability/silver_observability__transactions_completeness.sql diff --git a/models/silver/_observability/silver_observability.yml b/models/silver/_observability/silver_observability.yml new file mode 100644 index 0000000..fdc0628 --- /dev/null +++ b/models/silver/_observability/silver_observability.yml @@ -0,0 +1,123 @@ +version: 2 +models: + - name: silver_observability__blocks_completeness + description: Records of all blocks block gaps (missing blocks) with a timestamp the test was run + config: + contract: + enforced: true + tests: + - unique: + - TEST_TIMESTAMP + - dbt_utils.recency: + datepart: day + field: TEST_TIMESTAMP + interval: 2 + severity: error + tags: ['test_recency'] + columns: + - name: MIN_BLOCK + data_type: NUMBER + description: The lowest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK + data_type: NUMBER + description: The highest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MIN_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The lowest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The highest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_TESTED + data_type: NUMBER + description: Count of blocks in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_COUNT + data_type: NUMBER + description: Count of block gaps in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_ARRAY + data_type: ARRAY + description: Array of affected blocks + - name: TEST_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: When the test was run + tests: + - not_null: + tag: ['test_quality'] + + - name: silver_observability__transactions_completeness + description: Records of all blocks with missing transactions with a timestamp the test was run + tests: + - unique: + - TEST_TIMESTAMP + - dbt_utils.recency: + datepart: day + field: TEST_TIMESTAMP + interval: 2 + severity: error + tags: ['test_recency'] + columns: + - name: TEST_NAME + data_type: VARCHAR + description: Name for the test + - name: MIN_BLOCK + data_type: NUMBER + description: The lowest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK + data_type: NUMBER + description: The highest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MIN_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The lowest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The highest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_TESTED + data_type: NUMBER + description: Count of blocks in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_COUNT + data_type: NUMBER + description: Count of block gaps in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_ARRAY + data_type: ARRAY + description: Array of affected blocks + - name: TEST_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: When the test was run + tests: + - not_null: + tag: ['test_quality'] diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql new file mode 100644 index 0000000..5c09f85 --- /dev/null +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -0,0 +1,168 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability'], + enabled = true +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +blocks AS ( + SELECT + l.block_number, + block_timestamp, + LAG( + l.block_number, + 1 + ) over ( + ORDER BY + l.block_number ASC + ) AS prev_BLOCK_NUMBER + FROM + {{ ref("silver__blocks") }} + l + INNER JOIN block_range b + ON l.block_number = b.block_number + AND l.block_number >= ( + SELECT + MIN(block_number) + FROM + block_range + ) +), +block_gen AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + MIN(block_number) + FROM + blocks + ) + AND ( + SELECT + MAX(block_number) + FROM + blocks + ) +) +SELECT + 'blocks' AS test_name, + MIN( + b.block_number + ) AS min_block, + MAX( + b.block_number + ) AS max_block, + MIN( + b.block_timestamp + ) AS min_block_timestamp, + MAX( + b.block_timestamp + ) AS max_block_timestamp, + COUNT(1) AS blocks_tested, + COUNT( + CASE + WHEN C.block_number IS NOT NULL THEN A.block_number + END + ) AS blocks_impacted_count, + ARRAY_AGG( + CASE + WHEN C.block_number IS NOT NULL THEN A.block_number + END + ) within GROUP ( + ORDER BY + A.block_number + ) AS blocks_impacted_array, + SYSDATE() AS test_timestamp +FROM + block_gen A + LEFT JOIN blocks b + ON A.block_number = b.block_number + LEFT JOIN blocks C + ON A.block_number > C.prev_block_number + AND A.block_number < C.block_number + AND C.block_number - C.prev_block_number <> 1 +WHERE + COALESCE( + b.block_number, + C.block_number + ) IS NOT NULL diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql new file mode 100644 index 0000000..441e789 --- /dev/null +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -0,0 +1,137 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability'] +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +base_blocks AS ( + SELECT + block_number, + tx_count_from_versions AS transaction_count + FROM + {{ ref('silver__blocks') }} + WHERE + block_number BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +actual_tx_counts AS ( + SELECT + block_number, + COUNT(1) AS transaction_count + FROM + {{ ref('silver__transactions') }} + WHERE + block_number BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) + GROUP BY + block_number +), +potential_missing_txs AS ( + SELECT + e.block_number + FROM + base_blocks e + LEFT OUTER JOIN actual_tx_counts A + ON e.block_number = A.block_number + WHERE + COALESCE( + A.transaction_count, + 0 + ) <> e.transaction_count +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + potential_missing_txs +) +SELECT + 'transactions' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + SYSDATE() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 From c10fe8901693f786831ec60c378b23279579bb42 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Thu, 20 Mar 2025 13:20:57 -0600 Subject: [PATCH 10/16] added blocks_ and tx_ completeness --- .../_observability/silver_observability.yml | 123 +++++++++--------- 1 file changed, 63 insertions(+), 60 deletions(-) diff --git a/models/silver/_observability/silver_observability.yml b/models/silver/_observability/silver_observability.yml index fdc0628..4d29807 100644 --- a/models/silver/_observability/silver_observability.yml +++ b/models/silver/_observability/silver_observability.yml @@ -6,66 +6,6 @@ models: contract: enforced: true tests: - - unique: - - TEST_TIMESTAMP - - dbt_utils.recency: - datepart: day - field: TEST_TIMESTAMP - interval: 2 - severity: error - tags: ['test_recency'] - columns: - - name: MIN_BLOCK - data_type: NUMBER - description: The lowest block id in the test - tests: - - not_null: - tag: ['test_quality'] - - name: MAX_BLOCK - data_type: NUMBER - description: The highest block id in the test - tests: - - not_null: - tag: ['test_quality'] - - name: MIN_BLOCK_TIMESTAMP - data_type: TIMESTAMP_NTZ - description: The lowest block timestamp in the test - tests: - - not_null: - tag: ['test_quality'] - - name: MAX_BLOCK_TIMESTAMP - data_type: TIMESTAMP_NTZ - description: The highest block timestamp in the test - tests: - - not_null: - tag: ['test_quality'] - - name: BLOCKS_TESTED - data_type: NUMBER - description: Count of blocks in the test - tests: - - not_null: - tag: ['test_quality'] - - name: BLOCKS_IMPACTED_COUNT - data_type: NUMBER - description: Count of block gaps in the test - tests: - - not_null: - tag: ['test_quality'] - - name: BLOCKS_IMPACTED_ARRAY - data_type: ARRAY - description: Array of affected blocks - - name: TEST_TIMESTAMP - data_type: TIMESTAMP_NTZ - description: When the test was run - tests: - - not_null: - tag: ['test_quality'] - - - name: silver_observability__transactions_completeness - description: Records of all blocks with missing transactions with a timestamp the test was run - tests: - - unique: - - TEST_TIMESTAMP - dbt_utils.recency: datepart: day field: TEST_TIMESTAMP @@ -121,3 +61,66 @@ models: tests: - not_null: tag: ['test_quality'] + - unique: + tag: ['test_quality'] + + - name: silver_observability__transactions_completeness + description: Records of all blocks with missing transactions with a timestamp the test was run + tests: + - dbt_utils.recency: + datepart: day + field: TEST_TIMESTAMP + interval: 2 + severity: error + tags: ['test_recency'] + columns: + - name: TEST_NAME + data_type: VARCHAR + description: Name for the test + - name: MIN_BLOCK + data_type: NUMBER + description: The lowest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK + data_type: NUMBER + description: The highest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MIN_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The lowest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The highest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_TESTED + data_type: NUMBER + description: Count of blocks in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_COUNT + data_type: NUMBER + description: Count of block gaps in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_ARRAY + data_type: ARRAY + description: Array of affected blocks + - name: TEST_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: When the test was run + tests: + - not_null: + tag: ['test_quality'] + - unique: + tag: ['test_quality'] From 84cde10239c975d129c78ab53f40925a9794e5e2 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Sun, 23 Mar 2025 18:03:59 -0600 Subject: [PATCH 11/16] cutting over to FS node --- .../silver/realtime/streamline__blocks_tx_realtime.sql | 2 +- .../silver/realtime/streamline__transactions_realtime.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql b/models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql index eb59fee..a7ec06d 100644 --- a/models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql +++ b/models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql @@ -41,7 +41,7 @@ SELECT 'Flipside_Crypto/0.1' ), PARSE_JSON('{}'), - 'Vault/prod/movement/mainnet' + 'Vault/prod/movement/mainnet_fsc' ) AS request FROM blocks diff --git a/models/streamline/silver/realtime/streamline__transactions_realtime.sql b/models/streamline/silver/realtime/streamline__transactions_realtime.sql index 5fc62a5..42213bf 100644 --- a/models/streamline/silver/realtime/streamline__transactions_realtime.sql +++ b/models/streamline/silver/realtime/streamline__transactions_realtime.sql @@ -105,7 +105,7 @@ numbers AS ( 'Flipside_Crypto/0.1' ), PARSE_JSON('{}'), - 'Vault/prod/movement/mainnet' + 'Vault/prod/movement/mainnet_fsc' ) AS request FROM WORK From bb3fa5e97c2a7a41d789ff924e957635386cfb12 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 24 Mar 2025 08:08:06 -0600 Subject: [PATCH 12/16] cutting SL chainhead to internal node --- models/streamline/silver/streamline__chainhead.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/streamline/silver/streamline__chainhead.sql b/models/streamline/silver/streamline__chainhead.sql index 86ea4ac..0d21d97 100644 --- a/models/streamline/silver/streamline__chainhead.sql +++ b/models/streamline/silver/streamline__chainhead.sql @@ -16,5 +16,5 @@ SELECT 'Flipside_Crypto/0.1' ), OBJECT_CONSTRUCT(), - 'Vault/prod/movement/mainnet' + 'Vault/prod/movement/mainnet_fsc' ) :data :block_height :: INT AS block_number From 0dafbecc040b77056055eb86460b06869a3d3999 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Thu, 3 Apr 2025 11:34:15 -0600 Subject: [PATCH 13/16] removed stats from this PR --- models/gold/stats/gold_stats.yml | 37 ---------- .../stats/stats__ez_core_metrics_hourly.sql | 45 ------------ models/silver/stats/silver_stats.yml | 73 ------------------- .../silver_stats__core_metrics_hourly.sql | 54 -------------- 4 files changed, 209 deletions(-) delete mode 100644 models/gold/stats/gold_stats.yml delete mode 100644 models/gold/stats/stats__ez_core_metrics_hourly.sql delete mode 100644 models/silver/stats/silver_stats.yml delete mode 100644 models/silver/stats/silver_stats__core_metrics_hourly.sql diff --git a/models/gold/stats/gold_stats.yml b/models/gold/stats/gold_stats.yml deleted file mode 100644 index 779f6e7..0000000 --- a/models/gold/stats/gold_stats.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 2 -models: - - name: stats__ez_core_metrics_hourly - description: '{{ doc("ez_core_metrics_hourly_table_doc") }}' - - columns: - - name: BLOCK_TIMESTAMP_HOUR - description: '{{ doc("block_timestamp_hour") }}' - - name: BLOCK_NUMBER_MIN - description: '{{ doc("block_number_min") }}' - - name: BLOCK_NUMBER_MAX - description: '{{ doc("block_number_max") }}' - - name: BLOCK_COUNT - description: '{{ doc("block_count") }}' - - name: TRANSACTION_COUNT - description: '{{ doc("transaction_count") }}' - - name: TRANSACTION_COUNT_SUCCESS - description: '{{ doc("transaction_count_success") }}' - - name: TRANSACTION_COUNT_FAILED - description: '{{ doc("transaction_count_failed") }}' - - name: UNIQUE_SENDER_COUNT - description: '{{ doc("unique_sender_count") }}' - - name: UNIQUE_PAYLOAD_FUNCTION_COUNT - description: '{{ doc("unique_payload_function_count") }}' - - name: TOTAL_FEES_NATIVE - description: '{{ doc("total_fees_native") }}' - - name: TOTAL_FEES_USD - description: '{{ doc("total_fees_usd") }}' - tests: - - not_null: - tags: ['test_quality'] - - name: EZ_CORE_METRICS_HOURLY_ID - description: '{{ doc("pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql deleted file mode 100644 index 539ea44..0000000 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ /dev/null @@ -1,45 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = "block_timestamp_hour", - cluster_by = ['block_timestamp_hour::DATE'], - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STATS, METRICS, CORE, HOURLY', - } } }, - tags = ['noncore'] -) }} - -SELECT - block_timestamp_hour, - block_number_min, - block_number_max, - block_count, - transaction_count, - transaction_count_success, - transaction_count_failed, - unique_sender_count, - unique_payload_function_count, - total_fees AS total_fees_native, - ROUND( - (total_fees / pow( - 10, - 8 - )) * p.price, - 2 - ) AS total_fees_usd, - core_metrics_hourly_id AS ez_core_metrics_hourly_id, - s.inserted_timestamp AS inserted_timestamp, - s.modified_timestamp AS modified_timestamp -FROM - {{ ref('silver_stats__core_metrics_hourly') }} - s - LEFT JOIN {{ ref('price__ez_prices_hourly') }} - p - ON s.block_timestamp_hour = p.hour -WHERE - p.is_native - {% if is_incremental() %} - AND s.block_timestamp_hour >= ( - SELECT - MAX(block_timestamp_hour) - FROM {{ this }}) - {% endif %} \ No newline at end of file diff --git a/models/silver/stats/silver_stats.yml b/models/silver/stats/silver_stats.yml deleted file mode 100644 index 12763c0..0000000 --- a/models/silver/stats/silver_stats.yml +++ /dev/null @@ -1,73 +0,0 @@ -version: 2 -models: - - name: silver_stats__core_metrics_hourly - config: - contract: - enforced: true - tests: - - dbt_utils.sequential_values: - column_name: BLOCK_TIMESTAMP_HOUR - interval: 1 - config: - severity: error - error_if: ">0" - tags: ['test_recency'] - columns: - - name: BLOCK_TIMESTAMP_HOUR - data_type: TIMESTAMP_NTZ - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCK_NUMBER_MIN - data_type: FLOAT - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCK_NUMBER_MAX - data_type: FLOAT - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCK_COUNT - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: TRANSACTION_COUNT - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: TRANSACTION_COUNT_SUCCESS - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: TRANSACTION_COUNT_FAILED - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: UNIQUE_SENDER_COUNT - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: UNIQUE_PAYLOAD_FUNCTION_COUNT - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: TOTAL_FEES - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: CORE_METRICS_HOURLY_ID - data_type: VARCHAR - - name: INSERTED_TIMESTAMP - data_type: TIMESTAMP_NTZ - - name: MODIFIED_TIMESTAMP - data_type: TIMESTAMP_NTZ - - name: _INVOCATION_ID - data_type: VARCHAR diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql deleted file mode 100644 index 0578f31..0000000 --- a/models/silver/stats/silver_stats__core_metrics_hourly.sql +++ /dev/null @@ -1,54 +0,0 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, - "columns": true }, - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STATS, METRICS, CORE, HOURLY', - } } }, - tags = ['noncore'] -) }} - -SELECT - DATE_TRUNC( - 'hour', - block_timestamp - ) AS block_timestamp_hour, - MIN(block_number) :: FLOAT AS block_number_min, - MAX(block_number) :: FLOAT AS block_number_max, - COUNT( - DISTINCT block_number - ) AS block_count, - COUNT( - DISTINCT tx_hash - ) AS transaction_count, - COUNT( - DISTINCT CASE - WHEN success THEN tx_hash - END - ) AS transaction_count_success, - COUNT( - DISTINCT CASE - WHEN NOT success THEN tx_hash - END - ) AS transaction_count_failed, - COUNT( - DISTINCT sender - ) AS unique_sender_count, - COUNT( - DISTINCT payload_function - ) AS unique_payload_function_count, - SUM(COALESCE(gas_unit_price,0) * gas_used) AS total_fees, - {{ dbt_utils.generate_surrogate_key( - ['block_timestamp_hour'] - ) }} AS core_metrics_hourly_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - {{ ref('silver__transactions') }} -WHERE - block_timestamp_hour < DATE_TRUNC( - 'hour', - CURRENT_TIMESTAMP - ) -GROUP BY - 1 \ No newline at end of file From 3c0e18b477d5db360ccfd458aaa9a2fbbb0c5865 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 11:55:40 -0600 Subject: [PATCH 14/16] cutover to gold --- .github/workflows/dbt_run_observability.yml | 45 +++++++++++++++++++ ...ver_observability__blocks_completeness.sql | 6 +-- ...servability__transactions_completeness.sql | 8 ++-- 3 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/dbt_run_observability.yml diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml new file mode 100644 index 0000000..90c6509 --- /dev/null +++ b/.github/workflows/dbt_run_observability.yml @@ -0,0 +1,45 @@ +name: dbt_run_observability +run-name: dbt_run_observability + +on: + workflow_dispatch: + schedule: + # Runs “At minute 0 past every 8th hour.” (see https://crontab.guru) + - cron: '0 */8 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt build -s "movement_models,tag:observability" \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index 5c09f85..f3d669b 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -15,7 +15,7 @@ WITH summary_stats AS ( MAX(block_timestamp) AS max_block_timestamp, COUNT(1) AS blocks_tested FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) @@ -29,7 +29,7 @@ AND ( SELECT MIN(block_number) AS block_number FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) @@ -91,7 +91,7 @@ blocks AS ( l.block_number ASC ) AS prev_BLOCK_NUMBER FROM - {{ ref("silver__blocks") }} + {{ ref("core__fact_blocks") }} l INNER JOIN block_range b ON l.block_number = b.block_number diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql index 441e789..3e5d533 100644 --- a/models/silver/_observability/silver_observability__transactions_completeness.sql +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -14,7 +14,7 @@ WITH summary_stats AS ( MAX(block_timestamp) AS max_block_timestamp, COUNT(1) AS blocks_tested FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) @@ -28,7 +28,7 @@ AND ( SELECT MIN(block_number) AS block_number FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) @@ -61,7 +61,7 @@ base_blocks AS ( block_number, tx_count_from_versions AS transaction_count FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_number BETWEEN ( SELECT @@ -81,7 +81,7 @@ actual_tx_counts AS ( block_number, COUNT(1) AS transaction_count FROM - {{ ref('silver__transactions') }} + {{ ref('core__fact_transactions') }} WHERE block_number BETWEEN ( SELECT From 301dcf560b755d542ecddcf8b35ae928d3f02441 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 12:29:31 -0600 Subject: [PATCH 15/16] added block exclusions from mainnet beta as they didn't resolve after upstream FR. versioning related --- .github/workflows/dbt_run_observability.yml | 2 +- .../silver_observability__transactions_completeness.sql | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml index 90c6509..2ee2fd5 100644 --- a/.github/workflows/dbt_run_observability.yml +++ b/.github/workflows/dbt_run_observability.yml @@ -42,4 +42,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt build -s "movement_models,tag:observability" \ No newline at end of file + dbt run -s tag:observability \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql index 3e5d533..5e206ef 100644 --- a/models/silver/_observability/silver_observability__transactions_completeness.sql +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -59,7 +59,7 @@ AND ( base_blocks AS ( SELECT block_number, - tx_count_from_versions AS transaction_count + tx_count AS transaction_count FROM {{ ref('core__fact_blocks') }} WHERE @@ -75,6 +75,8 @@ base_blocks AS ( FROM summary_stats ) + AND + block_number NOT IN (0, 1758, 1760, 1761, 1762, 1763, 1764, 1766) ), actual_tx_counts AS ( SELECT @@ -95,6 +97,8 @@ actual_tx_counts AS ( FROM summary_stats ) + AND + block_number NOT IN (0, 1758, 1760, 1761, 1762, 1763, 1764, 1766) GROUP BY block_number ), From 3f7eb439678d154f4c6b55fcb81405b06869a2d1 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 12:57:42 -0600 Subject: [PATCH 16/16] added modified_timestamp as a dummy to enable recency tests. updated yml tags --- .../_observability/silver_observability.yml | 36 ++++++++++--------- ...ver_observability__blocks_completeness.sql | 3 +- ...servability__transactions_completeness.sql | 3 +- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/models/silver/_observability/silver_observability.yml b/models/silver/_observability/silver_observability.yml index 4d29807..9386e32 100644 --- a/models/silver/_observability/silver_observability.yml +++ b/models/silver/_observability/silver_observability.yml @@ -21,37 +21,37 @@ models: description: The lowest block id in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MAX_BLOCK data_type: NUMBER description: The highest block id in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MIN_BLOCK_TIMESTAMP data_type: TIMESTAMP_NTZ description: The lowest block timestamp in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MAX_BLOCK_TIMESTAMP data_type: TIMESTAMP_NTZ description: The highest block timestamp in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_TESTED data_type: NUMBER description: Count of blocks in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_IMPACTED_COUNT data_type: NUMBER description: Count of block gaps in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_IMPACTED_ARRAY data_type: ARRAY description: Array of affected blocks @@ -60,9 +60,11 @@ models: description: When the test was run tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - unique: - tag: ['test_quality'] + tags: ['test_quality'] + - name: MODIFIED_TIMESTAMP + data_type: TIMESTAMP_NTZ - name: silver_observability__transactions_completeness description: Records of all blocks with missing transactions with a timestamp the test was run @@ -82,37 +84,37 @@ models: description: The lowest block id in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MAX_BLOCK data_type: NUMBER description: The highest block id in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MIN_BLOCK_TIMESTAMP data_type: TIMESTAMP_NTZ description: The lowest block timestamp in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MAX_BLOCK_TIMESTAMP data_type: TIMESTAMP_NTZ description: The highest block timestamp in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_TESTED data_type: NUMBER description: Count of blocks in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_IMPACTED_COUNT data_type: NUMBER description: Count of block gaps in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_IMPACTED_ARRAY data_type: ARRAY description: Array of affected blocks @@ -121,6 +123,8 @@ models: description: When the test was run tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - unique: - tag: ['test_quality'] + tags: ['test_quality'] + - name: MODIFIED_TIMESTAMP + data_type: TIMESTAMP_NTZ diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index f3d669b..c3ae205 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -152,7 +152,8 @@ SELECT ORDER BY A.block_number ) AS blocks_impacted_array, - SYSDATE() AS test_timestamp + SYSDATE() AS test_timestamp, + SYSDATE() AS modified_timestamp FROM block_gen A LEFT JOIN blocks b diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql index 5e206ef..bd19d2f 100644 --- a/models/silver/_observability/silver_observability__transactions_completeness.sql +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -134,7 +134,8 @@ SELECT blocks_tested, blocks_impacted_count, blocks_impacted_array, - SYSDATE() AS test_timestamp + SYSDATE() AS test_timestamp, + SYSDATE() AS modified_timestamp FROM summary_stats JOIN impacted_blocks