From 40e16ab609cc7a89f093eb5e26d46e084c66f0f2 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 18 Mar 2025 14:54:58 -0600 Subject: [PATCH 01/14] swapped views with models, added clustering and SO where appropriate --- models/gold/price/price__dim_asset_metadata.sql | 6 ++++-- models/gold/price/price__ez_asset_metadata.sql | 7 +++++-- models/gold/price/price__ez_prices_hourly.sql | 7 +++++-- models/gold/price/price__fact_prices_ohlc_hourly.sql | 7 +++++-- .../price/silver__complete_native_asset_metadata.sql | 6 ++---- models/silver/price/silver__complete_native_prices.sql | 7 ++----- .../price/silver__complete_provider_asset_metadata.sql | 6 ++---- models/silver/price/silver__complete_provider_prices.sql | 7 ++----- .../silver/price/silver__complete_token_asset_metadata.sql | 6 ++---- models/silver/price/silver__complete_token_prices.sql | 7 ++----- 10 files changed, 31 insertions(+), 35 deletions(-) diff --git a/models/gold/price/price__dim_asset_metadata.sql b/models/gold/price/price__dim_asset_metadata.sql index 71cf674..6588fe9 100644 --- a/models/gold/price/price__dim_asset_metadata.sql +++ b/models/gold/price/price__dim_asset_metadata.sql @@ -1,6 +1,8 @@ {{ config( - materialized = 'view', - persist_docs ={ "relation": true, "columns": true }, + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'dim_asset_metadata_id', + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(token_address, asset_id, symbol, name)", tags = ['noncore'] ) }} diff --git a/models/gold/price/price__ez_asset_metadata.sql b/models/gold/price/price__ez_asset_metadata.sql index eacf895..e8ed30a 100644 --- a/models/gold/price/price__ez_asset_metadata.sql +++ b/models/gold/price/price__ez_asset_metadata.sql @@ -1,9 +1,12 @@ {{ config( - materialized = 'view', - persist_docs ={ "relation": true, "columns": true }, + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'ez_asset_metadata_id', + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, symbol, name)", tags = ['noncore'] ) }} + SELECT token_address, asset_id, diff --git a/models/gold/price/price__ez_prices_hourly.sql b/models/gold/price/price__ez_prices_hourly.sql index d816d7d..c6e5882 100644 --- a/models/gold/price/price__ez_prices_hourly.sql +++ b/models/gold/price/price__ez_prices_hourly.sql @@ -1,6 +1,9 @@ {{ config( - materialized = 'view', - persist_docs ={ "relation": true, "columns": true }, + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'ez_prices_hourly_id', + cluster_by = ['HOUR::DATE','is_native'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(token_address, symbol, NAME)", tags = ['noncore'] ) }} diff --git a/models/gold/price/price__fact_prices_ohlc_hourly.sql b/models/gold/price/price__fact_prices_ohlc_hourly.sql index 552efa9..98ff1b0 100644 --- a/models/gold/price/price__fact_prices_ohlc_hourly.sql +++ b/models/gold/price/price__fact_prices_ohlc_hourly.sql @@ -1,6 +1,9 @@ {{ config( - materialized = 'view', - persist_docs ={ "relation": true, "columns": true }, + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'fact_prices_ohlc_hourly_id', + cluster_by = ['HOUR::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, provider)", tags = ['noncore'] ) }} diff --git a/models/silver/price/silver__complete_native_asset_metadata.sql b/models/silver/price/silver__complete_native_asset_metadata.sql index be2e8f6..3cbf6ac 100644 --- a/models/silver/price/silver__complete_native_asset_metadata.sql +++ b/models/silver/price/silver__complete_native_asset_metadata.sql @@ -1,8 +1,6 @@ {{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_native_asset_metadata_id', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, symbol, name),SUBSTRING(asset_id, symbol, name)", + materialized = 'view', + persist_docs ={ "relation": true, "columns": true }, tags = ['noncore'] ) }} diff --git a/models/silver/price/silver__complete_native_prices.sql b/models/silver/price/silver__complete_native_prices.sql index 3c792ac..fb79380 100644 --- a/models/silver/price/silver__complete_native_prices.sql +++ b/models/silver/price/silver__complete_native_prices.sql @@ -1,9 +1,6 @@ {{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_native_prices_id', - cluster_by = ['hour::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, symbol, name),SUBSTRING(asset_id, symbol, name)", + materialized = 'view', + persist_docs ={ "relation": true, "columns": true }, tags = ['noncore'] ) }} diff --git a/models/silver/price/silver__complete_provider_asset_metadata.sql b/models/silver/price/silver__complete_provider_asset_metadata.sql index de1ef9d..8bb653c 100644 --- a/models/silver/price/silver__complete_provider_asset_metadata.sql +++ b/models/silver/price/silver__complete_provider_asset_metadata.sql @@ -1,8 +1,6 @@ {{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_provider_asset_metadata_id', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, token_address, symbol, name),SUBSTRING(asset_id, token_address, symbol, name)", + materialized = 'view', + persist_docs ={ "relation": true, "columns": true }, tags = ['noncore'] ) }} diff --git a/models/silver/price/silver__complete_provider_prices.sql b/models/silver/price/silver__complete_provider_prices.sql index bc53e07..93ee69b 100644 --- a/models/silver/price/silver__complete_provider_prices.sql +++ b/models/silver/price/silver__complete_provider_prices.sql @@ -1,9 +1,6 @@ {{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_provider_prices_id', - cluster_by = ['recorded_hour::DATE','provider'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id),SUBSTRING(asset_id)", + materialized = 'view', + persist_docs ={ "relation": true, "columns": true }, tags = ['noncore'] ) }} diff --git a/models/silver/price/silver__complete_token_asset_metadata.sql b/models/silver/price/silver__complete_token_asset_metadata.sql index 7e525b0..32a6961 100644 --- a/models/silver/price/silver__complete_token_asset_metadata.sql +++ b/models/silver/price/silver__complete_token_asset_metadata.sql @@ -1,8 +1,6 @@ {{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_token_asset_metadata_id', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, token_address, symbol, name),SUBSTRING(asset_id, token_address, symbol, name)", + materialized = 'view', + persist_docs ={ "relation": true, "columns": true }, tags = ['noncore'] ) }} diff --git a/models/silver/price/silver__complete_token_prices.sql b/models/silver/price/silver__complete_token_prices.sql index 77d3630..3a80041 100644 --- a/models/silver/price/silver__complete_token_prices.sql +++ b/models/silver/price/silver__complete_token_prices.sql @@ -1,9 +1,6 @@ {{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_token_prices_id', - cluster_by = ['hour::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_id, token_address, symbol, name),SUBSTRING(asset_id, token_address, symbol, name)", + materialized = 'view', + persist_docs ={ "relation": true, "columns": true }, tags = ['noncore'] ) }} From 462c4a0d8f9655972e7a94a8d7b631bdafc6b1c3 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 18 Mar 2025 15:09:38 -0600 Subject: [PATCH 02/14] updated views with tags --- .../bronze/price/bronze__complete_native_asset_metadata.sql | 5 +++++ models/bronze/price/bronze__complete_native_prices.sql | 3 ++- .../price/bronze__complete_provider_asset_metadata.sql | 3 ++- models/bronze/price/bronze__complete_provider_prices.sql | 3 ++- .../bronze/price/bronze__complete_token_asset_metadata.sql | 3 ++- models/bronze/price/bronze__complete_token_prices.sql | 3 ++- 6 files changed, 15 insertions(+), 5 deletions(-) diff --git a/models/bronze/price/bronze__complete_native_asset_metadata.sql b/models/bronze/price/bronze__complete_native_asset_metadata.sql index 1eac19d..c27f4f3 100644 --- a/models/bronze/price/bronze__complete_native_asset_metadata.sql +++ b/models/bronze/price/bronze__complete_native_asset_metadata.sql @@ -1,3 +1,8 @@ +{{ config ( + materialized = 'view', + tags = ['noncore'] +) }} + SELECT asset_id, symbol, diff --git a/models/bronze/price/bronze__complete_native_prices.sql b/models/bronze/price/bronze__complete_native_prices.sql index bc27af6..22be63f 100644 --- a/models/bronze/price/bronze__complete_native_prices.sql +++ b/models/bronze/price/bronze__complete_native_prices.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['noncore'] ) }} SELECT diff --git a/models/bronze/price/bronze__complete_provider_asset_metadata.sql b/models/bronze/price/bronze__complete_provider_asset_metadata.sql index 15bd2f2..72c1168 100644 --- a/models/bronze/price/bronze__complete_provider_asset_metadata.sql +++ b/models/bronze/price/bronze__complete_provider_asset_metadata.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['noncore'] ) }} SELECT diff --git a/models/bronze/price/bronze__complete_provider_prices.sql b/models/bronze/price/bronze__complete_provider_prices.sql index c726661..740a042 100644 --- a/models/bronze/price/bronze__complete_provider_prices.sql +++ b/models/bronze/price/bronze__complete_provider_prices.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['noncore'] ) }} SELECT diff --git a/models/bronze/price/bronze__complete_token_asset_metadata.sql b/models/bronze/price/bronze__complete_token_asset_metadata.sql index 2ad08d9..141bdfe 100644 --- a/models/bronze/price/bronze__complete_token_asset_metadata.sql +++ b/models/bronze/price/bronze__complete_token_asset_metadata.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['noncore'] ) }} SELECT diff --git a/models/bronze/price/bronze__complete_token_prices.sql b/models/bronze/price/bronze__complete_token_prices.sql index d929e87..9ec838a 100644 --- a/models/bronze/price/bronze__complete_token_prices.sql +++ b/models/bronze/price/bronze__complete_token_prices.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['noncore'] ) }} SELECT From 94382d96c6fc68421bb5436309e4bcd17aa9391b Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 18 Mar 2025 15:16:19 -0600 Subject: [PATCH 03/14] forgot to fix incremental jinja --- .../gold/price/price__dim_asset_metadata.sql | 12 ++++++++++ .../gold/price/price__ez_asset_metadata.sql | 23 +++++++++++++++++++ models/gold/price/price__ez_prices_hourly.sql | 22 ++++++++++++++++++ .../price/price__fact_prices_ohlc_hourly.sql | 11 +++++++++ ...silver__complete_native_asset_metadata.sql | 14 +---------- .../price/silver__complete_native_prices.sql | 12 ---------- ...lver__complete_provider_asset_metadata.sql | 12 ---------- .../silver__complete_provider_prices.sql | 12 ---------- .../silver__complete_token_asset_metadata.sql | 12 ---------- .../price/silver__complete_token_prices.sql | 15 +----------- 10 files changed, 70 insertions(+), 75 deletions(-) diff --git a/models/gold/price/price__dim_asset_metadata.sql b/models/gold/price/price__dim_asset_metadata.sql index 6588fe9..4e02854 100644 --- a/models/gold/price/price__dim_asset_metadata.sql +++ b/models/gold/price/price__dim_asset_metadata.sql @@ -19,3 +19,15 @@ SELECT complete_provider_asset_metadata_id AS dim_asset_metadata_id FROM {{ ref('silver__complete_provider_asset_metadata') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} \ No newline at end of file diff --git a/models/gold/price/price__ez_asset_metadata.sql b/models/gold/price/price__ez_asset_metadata.sql index e8ed30a..ed00d2f 100644 --- a/models/gold/price/price__ez_asset_metadata.sql +++ b/models/gold/price/price__ez_asset_metadata.sql @@ -21,6 +21,18 @@ SELECT complete_token_asset_metadata_id AS ez_asset_metadata_id FROM {{ ref('silver__complete_token_asset_metadata') }} +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + UNION ALL SELECT NULL AS token_address, @@ -36,3 +48,14 @@ SELECT complete_native_asset_metadata_id AS ez_asset_metadata_id FROM {{ ref('silver__complete_native_asset_metadata') }} +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/price/price__ez_prices_hourly.sql b/models/gold/price/price__ez_prices_hourly.sql index c6e5882..400ec4e 100644 --- a/models/gold/price/price__ez_prices_hourly.sql +++ b/models/gold/price/price__ez_prices_hourly.sql @@ -23,6 +23,17 @@ SELECT complete_token_prices_id AS ez_prices_hourly_id FROM {{ ref('silver__complete_token_prices') }} +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} UNION ALL SELECT HOUR, @@ -40,3 +51,14 @@ SELECT complete_native_prices_id AS ez_prices_hourly_id FROM {{ ref('silver__complete_native_prices') }} +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} \ No newline at end of file diff --git a/models/gold/price/price__fact_prices_ohlc_hourly.sql b/models/gold/price/price__fact_prices_ohlc_hourly.sql index 98ff1b0..364a7af 100644 --- a/models/gold/price/price__fact_prices_ohlc_hourly.sql +++ b/models/gold/price/price__fact_prices_ohlc_hourly.sql @@ -20,3 +20,14 @@ SELECT complete_provider_prices_id AS fact_prices_ohlc_hourly_id FROM {{ ref('silver__complete_provider_prices') }} +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} \ No newline at end of file diff --git a/models/silver/price/silver__complete_native_asset_metadata.sql b/models/silver/price/silver__complete_native_asset_metadata.sql index 3cbf6ac..9e9d2eb 100644 --- a/models/silver/price/silver__complete_native_asset_metadata.sql +++ b/models/silver/price/silver__complete_native_asset_metadata.sql @@ -21,16 +21,4 @@ SELECT FROM {{ ref( 'bronze__complete_native_asset_metadata' - ) }} - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} + ) }} \ No newline at end of file diff --git a/models/silver/price/silver__complete_native_prices.sql b/models/silver/price/silver__complete_native_prices.sql index fb79380..8086387 100644 --- a/models/silver/price/silver__complete_native_prices.sql +++ b/models/silver/price/silver__complete_native_prices.sql @@ -25,15 +25,3 @@ FROM {{ ref( 'bronze__complete_native_prices' ) }} - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} diff --git a/models/silver/price/silver__complete_provider_asset_metadata.sql b/models/silver/price/silver__complete_provider_asset_metadata.sql index 8bb653c..819679b 100644 --- a/models/silver/price/silver__complete_provider_asset_metadata.sql +++ b/models/silver/price/silver__complete_provider_asset_metadata.sql @@ -22,15 +22,3 @@ FROM {{ ref( 'bronze__complete_provider_asset_metadata' ) }} - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} diff --git a/models/silver/price/silver__complete_provider_prices.sql b/models/silver/price/silver__complete_provider_prices.sql index 93ee69b..42229c8 100644 --- a/models/silver/price/silver__complete_provider_prices.sql +++ b/models/silver/price/silver__complete_provider_prices.sql @@ -27,18 +27,6 @@ FROM m ON p.asset_id = m.asset_id -{% if is_incremental() %} -WHERE - p.modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} - qualify(ROW_NUMBER() over (PARTITION BY p.asset_id, recorded_hour, p.provider ORDER BY p.modified_timestamp DESC)) = 1 diff --git a/models/silver/price/silver__complete_token_asset_metadata.sql b/models/silver/price/silver__complete_token_asset_metadata.sql index 32a6961..8d67acf 100644 --- a/models/silver/price/silver__complete_token_asset_metadata.sql +++ b/models/silver/price/silver__complete_token_asset_metadata.sql @@ -27,15 +27,3 @@ FROM {{ ref( 'bronze__complete_token_asset_metadata' ) }} A - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} diff --git a/models/silver/price/silver__complete_token_prices.sql b/models/silver/price/silver__complete_token_prices.sql index 3a80041..df1a73a 100644 --- a/models/silver/price/silver__complete_token_prices.sql +++ b/models/silver/price/silver__complete_token_prices.sql @@ -7,7 +7,7 @@ SELECT HOUR, LOWER( - p.token_address + token_address ) AS token_address, asset_id, symbol, @@ -30,16 +30,3 @@ FROM {{ ref( 'bronze__complete_token_prices' ) }} - p - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX( - modified_timestamp - ) - FROM - {{ this }} - ) -{% endif %} From 66875b4b6057dba6a84204d1278b1fecc194504e Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Thu, 20 Mar 2025 13:14:37 -0600 Subject: [PATCH 04/14] added core models similar to aptos --- .../_observability/silver_observability.yml | 123 +++++++++++++ ...ver_observability__blocks_completeness.sql | 168 ++++++++++++++++++ ...servability__transactions_completeness.sql | 137 ++++++++++++++ 3 files changed, 428 insertions(+) create mode 100644 models/silver/_observability/silver_observability.yml create mode 100644 models/silver/_observability/silver_observability__blocks_completeness.sql create mode 100644 models/silver/_observability/silver_observability__transactions_completeness.sql diff --git a/models/silver/_observability/silver_observability.yml b/models/silver/_observability/silver_observability.yml new file mode 100644 index 0000000..fdc0628 --- /dev/null +++ b/models/silver/_observability/silver_observability.yml @@ -0,0 +1,123 @@ +version: 2 +models: + - name: silver_observability__blocks_completeness + description: Records of all blocks block gaps (missing blocks) with a timestamp the test was run + config: + contract: + enforced: true + tests: + - unique: + - TEST_TIMESTAMP + - dbt_utils.recency: + datepart: day + field: TEST_TIMESTAMP + interval: 2 + severity: error + tags: ['test_recency'] + columns: + - name: MIN_BLOCK + data_type: NUMBER + description: The lowest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK + data_type: NUMBER + description: The highest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MIN_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The lowest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The highest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_TESTED + data_type: NUMBER + description: Count of blocks in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_COUNT + data_type: NUMBER + description: Count of block gaps in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_ARRAY + data_type: ARRAY + description: Array of affected blocks + - name: TEST_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: When the test was run + tests: + - not_null: + tag: ['test_quality'] + + - name: silver_observability__transactions_completeness + description: Records of all blocks with missing transactions with a timestamp the test was run + tests: + - unique: + - TEST_TIMESTAMP + - dbt_utils.recency: + datepart: day + field: TEST_TIMESTAMP + interval: 2 + severity: error + tags: ['test_recency'] + columns: + - name: TEST_NAME + data_type: VARCHAR + description: Name for the test + - name: MIN_BLOCK + data_type: NUMBER + description: The lowest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK + data_type: NUMBER + description: The highest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MIN_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The lowest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The highest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_TESTED + data_type: NUMBER + description: Count of blocks in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_COUNT + data_type: NUMBER + description: Count of block gaps in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_ARRAY + data_type: ARRAY + description: Array of affected blocks + - name: TEST_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: When the test was run + tests: + - not_null: + tag: ['test_quality'] diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql new file mode 100644 index 0000000..5c09f85 --- /dev/null +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -0,0 +1,168 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability'], + enabled = true +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +blocks AS ( + SELECT + l.block_number, + block_timestamp, + LAG( + l.block_number, + 1 + ) over ( + ORDER BY + l.block_number ASC + ) AS prev_BLOCK_NUMBER + FROM + {{ ref("silver__blocks") }} + l + INNER JOIN block_range b + ON l.block_number = b.block_number + AND l.block_number >= ( + SELECT + MIN(block_number) + FROM + block_range + ) +), +block_gen AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + MIN(block_number) + FROM + blocks + ) + AND ( + SELECT + MAX(block_number) + FROM + blocks + ) +) +SELECT + 'blocks' AS test_name, + MIN( + b.block_number + ) AS min_block, + MAX( + b.block_number + ) AS max_block, + MIN( + b.block_timestamp + ) AS min_block_timestamp, + MAX( + b.block_timestamp + ) AS max_block_timestamp, + COUNT(1) AS blocks_tested, + COUNT( + CASE + WHEN C.block_number IS NOT NULL THEN A.block_number + END + ) AS blocks_impacted_count, + ARRAY_AGG( + CASE + WHEN C.block_number IS NOT NULL THEN A.block_number + END + ) within GROUP ( + ORDER BY + A.block_number + ) AS blocks_impacted_array, + SYSDATE() AS test_timestamp +FROM + block_gen A + LEFT JOIN blocks b + ON A.block_number = b.block_number + LEFT JOIN blocks C + ON A.block_number > C.prev_block_number + AND A.block_number < C.block_number + AND C.block_number - C.prev_block_number <> 1 +WHERE + COALESCE( + b.block_number, + C.block_number + ) IS NOT NULL diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql new file mode 100644 index 0000000..441e789 --- /dev/null +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -0,0 +1,137 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability'] +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +base_blocks AS ( + SELECT + block_number, + tx_count_from_versions AS transaction_count + FROM + {{ ref('silver__blocks') }} + WHERE + block_number BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +actual_tx_counts AS ( + SELECT + block_number, + COUNT(1) AS transaction_count + FROM + {{ ref('silver__transactions') }} + WHERE + block_number BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) + GROUP BY + block_number +), +potential_missing_txs AS ( + SELECT + e.block_number + FROM + base_blocks e + LEFT OUTER JOIN actual_tx_counts A + ON e.block_number = A.block_number + WHERE + COALESCE( + A.transaction_count, + 0 + ) <> e.transaction_count +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + potential_missing_txs +) +SELECT + 'transactions' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + SYSDATE() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 From c10fe8901693f786831ec60c378b23279579bb42 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Thu, 20 Mar 2025 13:20:57 -0600 Subject: [PATCH 05/14] added blocks_ and tx_ completeness --- .../_observability/silver_observability.yml | 123 +++++++++--------- 1 file changed, 63 insertions(+), 60 deletions(-) diff --git a/models/silver/_observability/silver_observability.yml b/models/silver/_observability/silver_observability.yml index fdc0628..4d29807 100644 --- a/models/silver/_observability/silver_observability.yml +++ b/models/silver/_observability/silver_observability.yml @@ -6,66 +6,6 @@ models: contract: enforced: true tests: - - unique: - - TEST_TIMESTAMP - - dbt_utils.recency: - datepart: day - field: TEST_TIMESTAMP - interval: 2 - severity: error - tags: ['test_recency'] - columns: - - name: MIN_BLOCK - data_type: NUMBER - description: The lowest block id in the test - tests: - - not_null: - tag: ['test_quality'] - - name: MAX_BLOCK - data_type: NUMBER - description: The highest block id in the test - tests: - - not_null: - tag: ['test_quality'] - - name: MIN_BLOCK_TIMESTAMP - data_type: TIMESTAMP_NTZ - description: The lowest block timestamp in the test - tests: - - not_null: - tag: ['test_quality'] - - name: MAX_BLOCK_TIMESTAMP - data_type: TIMESTAMP_NTZ - description: The highest block timestamp in the test - tests: - - not_null: - tag: ['test_quality'] - - name: BLOCKS_TESTED - data_type: NUMBER - description: Count of blocks in the test - tests: - - not_null: - tag: ['test_quality'] - - name: BLOCKS_IMPACTED_COUNT - data_type: NUMBER - description: Count of block gaps in the test - tests: - - not_null: - tag: ['test_quality'] - - name: BLOCKS_IMPACTED_ARRAY - data_type: ARRAY - description: Array of affected blocks - - name: TEST_TIMESTAMP - data_type: TIMESTAMP_NTZ - description: When the test was run - tests: - - not_null: - tag: ['test_quality'] - - - name: silver_observability__transactions_completeness - description: Records of all blocks with missing transactions with a timestamp the test was run - tests: - - unique: - - TEST_TIMESTAMP - dbt_utils.recency: datepart: day field: TEST_TIMESTAMP @@ -121,3 +61,66 @@ models: tests: - not_null: tag: ['test_quality'] + - unique: + tag: ['test_quality'] + + - name: silver_observability__transactions_completeness + description: Records of all blocks with missing transactions with a timestamp the test was run + tests: + - dbt_utils.recency: + datepart: day + field: TEST_TIMESTAMP + interval: 2 + severity: error + tags: ['test_recency'] + columns: + - name: TEST_NAME + data_type: VARCHAR + description: Name for the test + - name: MIN_BLOCK + data_type: NUMBER + description: The lowest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK + data_type: NUMBER + description: The highest block id in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MIN_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The lowest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: MAX_BLOCK_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: The highest block timestamp in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_TESTED + data_type: NUMBER + description: Count of blocks in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_COUNT + data_type: NUMBER + description: Count of block gaps in the test + tests: + - not_null: + tag: ['test_quality'] + - name: BLOCKS_IMPACTED_ARRAY + data_type: ARRAY + description: Array of affected blocks + - name: TEST_TIMESTAMP + data_type: TIMESTAMP_NTZ + description: When the test was run + tests: + - not_null: + tag: ['test_quality'] + - unique: + tag: ['test_quality'] From 84cde10239c975d129c78ab53f40925a9794e5e2 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Sun, 23 Mar 2025 18:03:59 -0600 Subject: [PATCH 06/14] cutting over to FS node --- .../silver/realtime/streamline__blocks_tx_realtime.sql | 2 +- .../silver/realtime/streamline__transactions_realtime.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql b/models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql index eb59fee..a7ec06d 100644 --- a/models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql +++ b/models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql @@ -41,7 +41,7 @@ SELECT 'Flipside_Crypto/0.1' ), PARSE_JSON('{}'), - 'Vault/prod/movement/mainnet' + 'Vault/prod/movement/mainnet_fsc' ) AS request FROM blocks diff --git a/models/streamline/silver/realtime/streamline__transactions_realtime.sql b/models/streamline/silver/realtime/streamline__transactions_realtime.sql index 5fc62a5..42213bf 100644 --- a/models/streamline/silver/realtime/streamline__transactions_realtime.sql +++ b/models/streamline/silver/realtime/streamline__transactions_realtime.sql @@ -105,7 +105,7 @@ numbers AS ( 'Flipside_Crypto/0.1' ), PARSE_JSON('{}'), - 'Vault/prod/movement/mainnet' + 'Vault/prod/movement/mainnet_fsc' ) AS request FROM WORK From bb3fa5e97c2a7a41d789ff924e957635386cfb12 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 24 Mar 2025 08:08:06 -0600 Subject: [PATCH 07/14] cutting SL chainhead to internal node --- models/streamline/silver/streamline__chainhead.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/streamline/silver/streamline__chainhead.sql b/models/streamline/silver/streamline__chainhead.sql index 86ea4ac..0d21d97 100644 --- a/models/streamline/silver/streamline__chainhead.sql +++ b/models/streamline/silver/streamline__chainhead.sql @@ -16,5 +16,5 @@ SELECT 'Flipside_Crypto/0.1' ), OBJECT_CONSTRUCT(), - 'Vault/prod/movement/mainnet' + 'Vault/prod/movement/mainnet_fsc' ) :data :block_height :: INT AS block_number From 0dafbecc040b77056055eb86460b06869a3d3999 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Thu, 3 Apr 2025 11:34:15 -0600 Subject: [PATCH 08/14] removed stats from this PR --- models/gold/stats/gold_stats.yml | 37 ---------- .../stats/stats__ez_core_metrics_hourly.sql | 45 ------------ models/silver/stats/silver_stats.yml | 73 ------------------- .../silver_stats__core_metrics_hourly.sql | 54 -------------- 4 files changed, 209 deletions(-) delete mode 100644 models/gold/stats/gold_stats.yml delete mode 100644 models/gold/stats/stats__ez_core_metrics_hourly.sql delete mode 100644 models/silver/stats/silver_stats.yml delete mode 100644 models/silver/stats/silver_stats__core_metrics_hourly.sql diff --git a/models/gold/stats/gold_stats.yml b/models/gold/stats/gold_stats.yml deleted file mode 100644 index 779f6e7..0000000 --- a/models/gold/stats/gold_stats.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 2 -models: - - name: stats__ez_core_metrics_hourly - description: '{{ doc("ez_core_metrics_hourly_table_doc") }}' - - columns: - - name: BLOCK_TIMESTAMP_HOUR - description: '{{ doc("block_timestamp_hour") }}' - - name: BLOCK_NUMBER_MIN - description: '{{ doc("block_number_min") }}' - - name: BLOCK_NUMBER_MAX - description: '{{ doc("block_number_max") }}' - - name: BLOCK_COUNT - description: '{{ doc("block_count") }}' - - name: TRANSACTION_COUNT - description: '{{ doc("transaction_count") }}' - - name: TRANSACTION_COUNT_SUCCESS - description: '{{ doc("transaction_count_success") }}' - - name: TRANSACTION_COUNT_FAILED - description: '{{ doc("transaction_count_failed") }}' - - name: UNIQUE_SENDER_COUNT - description: '{{ doc("unique_sender_count") }}' - - name: UNIQUE_PAYLOAD_FUNCTION_COUNT - description: '{{ doc("unique_payload_function_count") }}' - - name: TOTAL_FEES_NATIVE - description: '{{ doc("total_fees_native") }}' - - name: TOTAL_FEES_USD - description: '{{ doc("total_fees_usd") }}' - tests: - - not_null: - tags: ['test_quality'] - - name: EZ_CORE_METRICS_HOURLY_ID - description: '{{ doc("pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql deleted file mode 100644 index 539ea44..0000000 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ /dev/null @@ -1,45 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = "block_timestamp_hour", - cluster_by = ['block_timestamp_hour::DATE'], - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STATS, METRICS, CORE, HOURLY', - } } }, - tags = ['noncore'] -) }} - -SELECT - block_timestamp_hour, - block_number_min, - block_number_max, - block_count, - transaction_count, - transaction_count_success, - transaction_count_failed, - unique_sender_count, - unique_payload_function_count, - total_fees AS total_fees_native, - ROUND( - (total_fees / pow( - 10, - 8 - )) * p.price, - 2 - ) AS total_fees_usd, - core_metrics_hourly_id AS ez_core_metrics_hourly_id, - s.inserted_timestamp AS inserted_timestamp, - s.modified_timestamp AS modified_timestamp -FROM - {{ ref('silver_stats__core_metrics_hourly') }} - s - LEFT JOIN {{ ref('price__ez_prices_hourly') }} - p - ON s.block_timestamp_hour = p.hour -WHERE - p.is_native - {% if is_incremental() %} - AND s.block_timestamp_hour >= ( - SELECT - MAX(block_timestamp_hour) - FROM {{ this }}) - {% endif %} \ No newline at end of file diff --git a/models/silver/stats/silver_stats.yml b/models/silver/stats/silver_stats.yml deleted file mode 100644 index 12763c0..0000000 --- a/models/silver/stats/silver_stats.yml +++ /dev/null @@ -1,73 +0,0 @@ -version: 2 -models: - - name: silver_stats__core_metrics_hourly - config: - contract: - enforced: true - tests: - - dbt_utils.sequential_values: - column_name: BLOCK_TIMESTAMP_HOUR - interval: 1 - config: - severity: error - error_if: ">0" - tags: ['test_recency'] - columns: - - name: BLOCK_TIMESTAMP_HOUR - data_type: TIMESTAMP_NTZ - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCK_NUMBER_MIN - data_type: FLOAT - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCK_NUMBER_MAX - data_type: FLOAT - tests: - - not_null: - tags: ['test_quality'] - - name: BLOCK_COUNT - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: TRANSACTION_COUNT - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: TRANSACTION_COUNT_SUCCESS - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: TRANSACTION_COUNT_FAILED - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: UNIQUE_SENDER_COUNT - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: UNIQUE_PAYLOAD_FUNCTION_COUNT - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: TOTAL_FEES - data_type: NUMBER - tests: - - not_null: - tags: ['test_quality'] - - name: CORE_METRICS_HOURLY_ID - data_type: VARCHAR - - name: INSERTED_TIMESTAMP - data_type: TIMESTAMP_NTZ - - name: MODIFIED_TIMESTAMP - data_type: TIMESTAMP_NTZ - - name: _INVOCATION_ID - data_type: VARCHAR diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql deleted file mode 100644 index 0578f31..0000000 --- a/models/silver/stats/silver_stats__core_metrics_hourly.sql +++ /dev/null @@ -1,54 +0,0 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, - "columns": true }, - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STATS, METRICS, CORE, HOURLY', - } } }, - tags = ['noncore'] -) }} - -SELECT - DATE_TRUNC( - 'hour', - block_timestamp - ) AS block_timestamp_hour, - MIN(block_number) :: FLOAT AS block_number_min, - MAX(block_number) :: FLOAT AS block_number_max, - COUNT( - DISTINCT block_number - ) AS block_count, - COUNT( - DISTINCT tx_hash - ) AS transaction_count, - COUNT( - DISTINCT CASE - WHEN success THEN tx_hash - END - ) AS transaction_count_success, - COUNT( - DISTINCT CASE - WHEN NOT success THEN tx_hash - END - ) AS transaction_count_failed, - COUNT( - DISTINCT sender - ) AS unique_sender_count, - COUNT( - DISTINCT payload_function - ) AS unique_payload_function_count, - SUM(COALESCE(gas_unit_price,0) * gas_used) AS total_fees, - {{ dbt_utils.generate_surrogate_key( - ['block_timestamp_hour'] - ) }} AS core_metrics_hourly_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - {{ ref('silver__transactions') }} -WHERE - block_timestamp_hour < DATE_TRUNC( - 'hour', - CURRENT_TIMESTAMP - ) -GROUP BY - 1 \ No newline at end of file From db0d212d63b024a86492e043f356005bf57f1f2f Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 11:34:29 -0600 Subject: [PATCH 09/14] updated noncore workflow and double checked silver model tags: noncore --- .github/workflows/dbt_run_noncore.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dbt_run_noncore.yml b/.github/workflows/dbt_run_noncore.yml index 77b234a..6b36bce 100644 --- a/.github/workflows/dbt_run_noncore.yml +++ b/.github/workflows/dbt_run_noncore.yml @@ -1,5 +1,5 @@ -name: dbt_run_incremental_core -run-name: dbt_run_incremental_core +name: dbt_run_noncore +run-name: dbt_run_noncore on: workflow_dispatch: @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m movement_models,tag:core \ No newline at end of file + dbt run -m movement_models,tag:noncore \ No newline at end of file From 3c0e18b477d5db360ccfd458aaa9a2fbbb0c5865 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 11:55:40 -0600 Subject: [PATCH 10/14] cutover to gold --- .github/workflows/dbt_run_observability.yml | 45 +++++++++++++++++++ ...ver_observability__blocks_completeness.sql | 6 +-- ...servability__transactions_completeness.sql | 8 ++-- 3 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/dbt_run_observability.yml diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml new file mode 100644 index 0000000..90c6509 --- /dev/null +++ b/.github/workflows/dbt_run_observability.yml @@ -0,0 +1,45 @@ +name: dbt_run_observability +run-name: dbt_run_observability + +on: + workflow_dispatch: + schedule: + # Runs “At minute 0 past every 8th hour.” (see https://crontab.guru) + - cron: '0 */8 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt build -s "movement_models,tag:observability" \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index 5c09f85..f3d669b 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -15,7 +15,7 @@ WITH summary_stats AS ( MAX(block_timestamp) AS max_block_timestamp, COUNT(1) AS blocks_tested FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) @@ -29,7 +29,7 @@ AND ( SELECT MIN(block_number) AS block_number FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) @@ -91,7 +91,7 @@ blocks AS ( l.block_number ASC ) AS prev_BLOCK_NUMBER FROM - {{ ref("silver__blocks") }} + {{ ref("core__fact_blocks") }} l INNER JOIN block_range b ON l.block_number = b.block_number diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql index 441e789..3e5d533 100644 --- a/models/silver/_observability/silver_observability__transactions_completeness.sql +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -14,7 +14,7 @@ WITH summary_stats AS ( MAX(block_timestamp) AS max_block_timestamp, COUNT(1) AS blocks_tested FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) @@ -28,7 +28,7 @@ AND ( SELECT MIN(block_number) AS block_number FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) @@ -61,7 +61,7 @@ base_blocks AS ( block_number, tx_count_from_versions AS transaction_count FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_number BETWEEN ( SELECT @@ -81,7 +81,7 @@ actual_tx_counts AS ( block_number, COUNT(1) AS transaction_count FROM - {{ ref('silver__transactions') }} + {{ ref('core__fact_transactions') }} WHERE block_number BETWEEN ( SELECT From 301dcf560b755d542ecddcf8b35ae928d3f02441 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 12:29:31 -0600 Subject: [PATCH 11/14] added block exclusions from mainnet beta as they didn't resolve after upstream FR. versioning related --- .github/workflows/dbt_run_observability.yml | 2 +- .../silver_observability__transactions_completeness.sql | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml index 90c6509..2ee2fd5 100644 --- a/.github/workflows/dbt_run_observability.yml +++ b/.github/workflows/dbt_run_observability.yml @@ -42,4 +42,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt build -s "movement_models,tag:observability" \ No newline at end of file + dbt run -s tag:observability \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql index 3e5d533..5e206ef 100644 --- a/models/silver/_observability/silver_observability__transactions_completeness.sql +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -59,7 +59,7 @@ AND ( base_blocks AS ( SELECT block_number, - tx_count_from_versions AS transaction_count + tx_count AS transaction_count FROM {{ ref('core__fact_blocks') }} WHERE @@ -75,6 +75,8 @@ base_blocks AS ( FROM summary_stats ) + AND + block_number NOT IN (0, 1758, 1760, 1761, 1762, 1763, 1764, 1766) ), actual_tx_counts AS ( SELECT @@ -95,6 +97,8 @@ actual_tx_counts AS ( FROM summary_stats ) + AND + block_number NOT IN (0, 1758, 1760, 1761, 1762, 1763, 1764, 1766) GROUP BY block_number ), From 3f7eb439678d154f4c6b55fcb81405b06869a2d1 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 12:57:42 -0600 Subject: [PATCH 12/14] added modified_timestamp as a dummy to enable recency tests. updated yml tags --- .../_observability/silver_observability.yml | 36 ++++++++++--------- ...ver_observability__blocks_completeness.sql | 3 +- ...servability__transactions_completeness.sql | 3 +- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/models/silver/_observability/silver_observability.yml b/models/silver/_observability/silver_observability.yml index 4d29807..9386e32 100644 --- a/models/silver/_observability/silver_observability.yml +++ b/models/silver/_observability/silver_observability.yml @@ -21,37 +21,37 @@ models: description: The lowest block id in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MAX_BLOCK data_type: NUMBER description: The highest block id in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MIN_BLOCK_TIMESTAMP data_type: TIMESTAMP_NTZ description: The lowest block timestamp in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MAX_BLOCK_TIMESTAMP data_type: TIMESTAMP_NTZ description: The highest block timestamp in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_TESTED data_type: NUMBER description: Count of blocks in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_IMPACTED_COUNT data_type: NUMBER description: Count of block gaps in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_IMPACTED_ARRAY data_type: ARRAY description: Array of affected blocks @@ -60,9 +60,11 @@ models: description: When the test was run tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - unique: - tag: ['test_quality'] + tags: ['test_quality'] + - name: MODIFIED_TIMESTAMP + data_type: TIMESTAMP_NTZ - name: silver_observability__transactions_completeness description: Records of all blocks with missing transactions with a timestamp the test was run @@ -82,37 +84,37 @@ models: description: The lowest block id in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MAX_BLOCK data_type: NUMBER description: The highest block id in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MIN_BLOCK_TIMESTAMP data_type: TIMESTAMP_NTZ description: The lowest block timestamp in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: MAX_BLOCK_TIMESTAMP data_type: TIMESTAMP_NTZ description: The highest block timestamp in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_TESTED data_type: NUMBER description: Count of blocks in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_IMPACTED_COUNT data_type: NUMBER description: Count of block gaps in the test tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - name: BLOCKS_IMPACTED_ARRAY data_type: ARRAY description: Array of affected blocks @@ -121,6 +123,8 @@ models: description: When the test was run tests: - not_null: - tag: ['test_quality'] + tags: ['test_quality'] - unique: - tag: ['test_quality'] + tags: ['test_quality'] + - name: MODIFIED_TIMESTAMP + data_type: TIMESTAMP_NTZ diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index f3d669b..c3ae205 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -152,7 +152,8 @@ SELECT ORDER BY A.block_number ) AS blocks_impacted_array, - SYSDATE() AS test_timestamp + SYSDATE() AS test_timestamp, + SYSDATE() AS modified_timestamp FROM block_gen A LEFT JOIN blocks b diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql index 5e206ef..bd19d2f 100644 --- a/models/silver/_observability/silver_observability__transactions_completeness.sql +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -134,7 +134,8 @@ SELECT blocks_tested, blocks_impacted_count, blocks_impacted_array, - SYSDATE() AS test_timestamp + SYSDATE() AS test_timestamp, + SYSDATE() AS modified_timestamp FROM summary_stats JOIN impacted_blocks From c26c9d1ab2474df990ef3cc42140e2f5370a2c39 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 14:01:32 -0600 Subject: [PATCH 13/14] added noncore WF --- .github/workflows/dbt_run_noncore.yml | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/dbt_run_noncore.yml diff --git a/.github/workflows/dbt_run_noncore.yml b/.github/workflows/dbt_run_noncore.yml new file mode 100644 index 0000000..6b36bce --- /dev/null +++ b/.github/workflows/dbt_run_noncore.yml @@ -0,0 +1,44 @@ +name: dbt_run_noncore +run-name: dbt_run_noncore + +on: + workflow_dispatch: + schedule: + - cron: "25,55 0/4 * * *" + +env: + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m movement_models,tag:noncore \ No newline at end of file From 006b77c471946fecd09cde21adbc7a8cfc3b5cff Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 14:09:32 -0600 Subject: [PATCH 14/14] sources.yml dropped through a merge --- models/sources.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/models/sources.yml b/models/sources.yml index f8f61f9..8e809e3 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -1,5 +1,4 @@ version: 2 - sources: - name: crosschain database: "{{ 'crosschain' if target.database == 'MOVEMENT' else 'crosschain_dev' }}" @@ -13,6 +12,13 @@ sources: schema: silver tables: - name: number_sequence + - name: labels_combined + - name: complete_token_asset_metadata + - name: complete_token_prices + - name: complete_provider_asset_metadata + - name: complete_provider_prices + - name: complete_native_asset_metadata + - name: complete_native_prices - name: bronze_streamline database: streamline schema: | @@ -30,4 +36,4 @@ sources: database: movement schema: github_actions tables: - - name: workflows + - name: workflows \ No newline at end of file