From 16e5851a3b0ca48268aafb09716ab95328be321e Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 1 Apr 2025 08:41:14 -0700 Subject: [PATCH 01/14] update heal filters, update workflows --- .github/workflows/dbt_run_scheduled_non_realtime.yml | 2 +- .github/workflows/dbt_run_trace_backfill.yml | 4 +--- models/silver/core/silver__fact_traces2.sql | 7 ++++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dbt_run_scheduled_non_realtime.yml b/.github/workflows/dbt_run_scheduled_non_realtime.yml index 8c310f5..e08ea41 100644 --- a/.github/workflows/dbt_run_scheduled_non_realtime.yml +++ b/.github/workflows/dbt_run_scheduled_non_realtime.yml @@ -43,4 +43,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "kaia_models,tag:non_realtime" \ No newline at end of file + dbt run -m "kaia_models,tag:non_realtime" --exclude "silver__fact_traces2" \ No newline at end of file diff --git a/.github/workflows/dbt_run_trace_backfill.yml b/.github/workflows/dbt_run_trace_backfill.yml index 3b298d4..3ffd112 100644 --- a/.github/workflows/dbt_run_trace_backfill.yml +++ b/.github/workflows/dbt_run_trace_backfill.yml @@ -44,7 +44,5 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --threads 1 --vars '{"full_reload_mode": true}' -m silver__traces2 silver__fact_traces2 - - + dbt run --threads 1 silver__fact_traces2 diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql index c196b32..d0e9a37 100644 --- a/models/silver/core/silver__fact_traces2.sql +++ b/models/silver/core/silver__fact_traces2.sql @@ -110,7 +110,7 @@ trace_index_sub_traces AS ( b.tx_position ORDER BY number_array ASC - ) - 1 AS trace_index, + ) - 1 AS trace_index, b.trace_json, b.traces_id, b.source @@ -336,12 +336,13 @@ heal_missing_data AS ( JOIN {{ ref('silver__transactions') }} txs ON t.tx_position = txs.position AND t.block_number = txs.block_number - and txs.block_number > 160000000 + WHERE (t.tx_hash IS NULL OR t.block_timestamp IS NULL OR t.tx_status IS NULL) - and t.block_number > 160000000 + LIMIT 1000000000 + ) {% endif %}, From f782d8f63a7bd9d6fe70b93f089f9a5b8972a7d4 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 1 Apr 2025 08:44:02 -0700 Subject: [PATCH 02/14] remove space --- models/silver/core/silver__fact_traces2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql index d0e9a37..c7ded8b 100644 --- a/models/silver/core/silver__fact_traces2.sql +++ b/models/silver/core/silver__fact_traces2.sql @@ -110,7 +110,7 @@ trace_index_sub_traces AS ( b.tx_position ORDER BY number_array ASC - ) - 1 AS trace_index, + ) - 1 AS trace_index, b.trace_json, b.traces_id, b.source From e92371e747133824ca1251097119c73b369bf540 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 1 Apr 2025 09:36:55 -0700 Subject: [PATCH 03/14] fix command --- .github/workflows/dbt_run_trace_backfill.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_trace_backfill.yml b/.github/workflows/dbt_run_trace_backfill.yml index 3ffd112..6bfdbfa 100644 --- a/.github/workflows/dbt_run_trace_backfill.yml +++ b/.github/workflows/dbt_run_trace_backfill.yml @@ -44,5 +44,5 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --threads 1 silver__fact_traces2 + dbt run --threads 1 -m silver__fact_traces2 From bcabefb5d6b2bbfebd94e2308b2d4ae66b7f58da Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 1 Apr 2025 09:44:01 -0700 Subject: [PATCH 04/14] update to var limit --- .github/workflows/dbt_run_trace_backfill.yml | 2 +- models/silver/core/silver__fact_traces2.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dbt_run_trace_backfill.yml b/.github/workflows/dbt_run_trace_backfill.yml index 6bfdbfa..077578b 100644 --- a/.github/workflows/dbt_run_trace_backfill.yml +++ b/.github/workflows/dbt_run_trace_backfill.yml @@ -44,5 +44,5 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --threads 1 -m silver__fact_traces2 + dbt run --select silver__fact_traces2 diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql index c7ded8b..299faa5 100644 --- a/models/silver/core/silver__fact_traces2.sql +++ b/models/silver/core/silver__fact_traces2.sql @@ -341,7 +341,7 @@ heal_missing_data AS ( (t.tx_hash IS NULL OR t.block_timestamp IS NULL OR t.tx_status IS NULL) - LIMIT 1000000000 + LIMIT {{ var('HEAL_LIMIT', 1000000000) }} ) {% endif %}, From 398b45c18abe8346a438ae6c3e83a83730a83146 Mon Sep 17 00:00:00 2001 From: mattromano Date: Wed, 2 Apr 2025 11:51:05 -0700 Subject: [PATCH 05/14] update fact_traces2 --- models/silver/core/silver__fact_traces2.sql | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql index 299faa5..41eeba6 100644 --- a/models/silver/core/silver__fact_traces2.sql +++ b/models/silver/core/silver__fact_traces2.sql @@ -25,7 +25,7 @@ WITH silver_traces AS ( 1 = 1 {% if is_incremental() and not var('full_reload_mode', false) %} - AND block_number > 160000000 + AND block_number >= 160000000 AND modified_timestamp > ( SELECT MAX(modified_timestamp) @@ -35,18 +35,7 @@ WITH silver_traces AS ( block_number > 160000000 ) {% elif is_incremental() and var('full_reload_mode', false) %} - AND block_number < 80000000 - AND modified_timestamp > COALESCE( - ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - WHERE - block_number < 80000000 - ), - '2024-01-01' - ) + AND block_number < {{ var('RELOAD_BLOCK', 10000000) }} {% else %} AND block_number <= 149500000 {% endif %} @@ -280,7 +269,7 @@ incremental_traces AS ( ON f.tx_position = t.position AND f.block_number = t.block_number - {% if is_incremental() and not full_reload_mode %} + {% if is_incremental() and not var('full_reload_mode', false)%} AND t.modified_timestamp >= ( SELECT DATEADD('hour', -24, MAX(modified_timestamp)) From 79e3abb61e9d09478aaff3eceb31d38a833d78d7 Mon Sep 17 00:00:00 2001 From: mattromano Date: Wed, 2 Apr 2025 12:30:03 -0700 Subject: [PATCH 06/14] add min --- models/silver/core/silver__fact_traces2.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql index 41eeba6..d791860 100644 --- a/models/silver/core/silver__fact_traces2.sql +++ b/models/silver/core/silver__fact_traces2.sql @@ -35,7 +35,8 @@ WITH silver_traces AS ( block_number > 160000000 ) {% elif is_incremental() and var('full_reload_mode', false) %} - AND block_number < {{ var('RELOAD_BLOCK', 10000000) }} + AND block_number < {{ var('RELOAD_BLOCK_MAX', 10000000) }} + AND block_number >= {{ var('RELOAD_BLOCK_MIN', 10000000) }} {% else %} AND block_number <= 149500000 {% endif %} From 22dd7754d09b3a7b1ce1c63c5e7ac3f1d4295039 Mon Sep 17 00:00:00 2001 From: mattromano Date: Wed, 9 Apr 2025 13:14:51 -0700 Subject: [PATCH 07/14] remove block incremental lookbacks --- models/silver/core/silver__fact_traces2.sql | 3 --- models/silver/core/silver__traces2.sql | 2 -- 2 files changed, 5 deletions(-) diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql index d791860..73ea10d 100644 --- a/models/silver/core/silver__fact_traces2.sql +++ b/models/silver/core/silver__fact_traces2.sql @@ -25,14 +25,11 @@ WITH silver_traces AS ( 1 = 1 {% if is_incremental() and not var('full_reload_mode', false) %} - AND block_number >= 160000000 AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM {{ this }} - WHERE - block_number > 160000000 ) {% elif is_incremental() and var('full_reload_mode', false) %} AND block_number < {{ var('RELOAD_BLOCK_MAX', 10000000) }} diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index eb32be2..8db85eb 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -27,8 +27,6 @@ WITH bronze_traces AS ( {{ this }} ) AND DATA :result IS NOT NULL - and block_number > 160000000 - and partition_key > 160000000 {% elif is_incremental() and var('full_reload_mode', false) and not var('initial_load', false) %} {{ ref('bronze__streamline_fr_traces') }} From 9dc46c6c3b76bb9c0d0689a9e77c1b024457f41f Mon Sep 17 00:00:00 2001 From: mattromano Date: Thu, 10 Apr 2025 10:42:09 -0700 Subject: [PATCH 08/14] add observability traces2 --- models/gold/core/core__fact_traces.sql | 1 + .../silver_observability__fact_traces2.sql | 115 +++++++++++++++++ ..._observability__traces_completeness_sf.sql | 116 ++++++++++++++++++ .../tests/traces/test_silver__traces_full.sql | 9 ++ .../tests/traces/test_silver__traces_full.yml | 52 ++++++++ .../traces/test_silver__traces_recent.sql | 16 +++ .../traces/test_silver__traces_recent.yml | 52 ++++++++ 7 files changed, 361 insertions(+) create mode 100644 models/silver/_observability/silver_observability__fact_traces2.sql create mode 100644 models/silver/_observability/silver_observability__traces_completeness_sf.sql create mode 100644 models/silver/core/tests/traces/test_silver__traces_full.sql create mode 100644 models/silver/core/tests/traces/test_silver__traces_full.yml create mode 100644 models/silver/core/tests/traces/test_silver__traces_recent.sql create mode 100644 models/silver/core/tests/traces/test_silver__traces_recent.yml diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql index 65f8a61..598119e 100644 --- a/models/gold/core/core__fact_traces.sql +++ b/models/gold/core/core__fact_traces.sql @@ -51,3 +51,4 @@ WHERE {{ this }} ) {% endif %} +dbt run -m models/silver/core/silver__fact_traces2.sql --vars '{"full_reload_mode": true, "RELOAD_BLOCK_MAX": 20000000, "RELOAD_BLOCK_MIN": 10000000}' \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__fact_traces2.sql b/models/silver/_observability/silver_observability__fact_traces2.sql new file mode 100644 index 0000000..989b45d --- /dev/null +++ b/models/silver/_observability/silver_observability__fact_traces2.sql @@ -0,0 +1,115 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = true, + tags = ['observability'] +) }} + +WITH summary_stats AS ( + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ ref('silver__number_sequence') }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +broken_blocks AS ( + SELECT + DISTINCT block_number + FROM + {{ ref("silver__transactions") }} + tx + LEFT JOIN {{ ref("silver__fact_traces2") }} + tr USING ( + block_number, + tx_hash + ) + JOIN block_range USING (block_number) + WHERE + tr.tx_hash IS NULL +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + broken_blocks +) +SELECT + 'traces' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + CURRENT_TIMESTAMP() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 diff --git a/models/silver/_observability/silver_observability__traces_completeness_sf.sql b/models/silver/_observability/silver_observability__traces_completeness_sf.sql new file mode 100644 index 0000000..a042d96 --- /dev/null +++ b/models/silver/_observability/silver_observability__traces_completeness_sf.sql @@ -0,0 +1,116 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability'] +) }} + +WITH summary_stats AS ( + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + AND block_timestamp >= '2024-02-23 00:00:00.000' --earliest trace data backfilled + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ ref('silver__number_sequence') }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +broken_blocks AS ( + SELECT + DISTINCT block_number + FROM + {{ ref("silver__transactions") }} + tx + LEFT JOIN {{ ref("silver__traces") }} + tr USING ( + block_number, + tx_hash + ) + JOIN block_range USING (block_number) + WHERE + tr.tx_hash IS NULL +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + broken_blocks +) +SELECT + 'traces' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + CURRENT_TIMESTAMP() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 diff --git a/models/silver/core/tests/traces/test_silver__traces_full.sql b/models/silver/core/tests/traces/test_silver__traces_full.sql new file mode 100644 index 0000000..8dd4ca8 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__traces2') }} diff --git a/models/silver/core/tests/traces/test_silver__traces_full.yml b/models/silver/core/tests/traces/test_silver__traces_full.yml new file mode 100644 index 0000000..0f3ff2b --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_full.yml @@ -0,0 +1,52 @@ +version: 2 +models: + - name: test_silver__traces_full + description: "This is a view used to test all of the silver traces model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TRACES_ID + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_POSITION + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: TRACE_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.sql b/models/silver/core/tests/traces/test_silver__traces_recent.sql new file mode 100644 index 0000000..6068370 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__traces2') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_block_lookback') }} + ) diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.yml b/models/silver/core/tests/traces/test_silver__traces_recent.yml new file mode 100644 index 0000000..eb8e689 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_recent.yml @@ -0,0 +1,52 @@ +version: 2 +models: + - name: test_silver__traces_recent + description: "This is a view used to test the last three days of traces." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TRACES_ID + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_POSITION + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: TRACE_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file From 9f3ec45e0bc2c80d4234584590a40fa058ebf090 Mon Sep 17 00:00:00 2001 From: mattromano Date: Thu, 10 Apr 2025 11:25:24 -0700 Subject: [PATCH 09/14] pause bronze abis --- .github/workflows/dbt_run_scheduled_curated.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_scheduled_curated.yml b/.github/workflows/dbt_run_scheduled_curated.yml index 4a51989..36a1f11 100644 --- a/.github/workflows/dbt_run_scheduled_curated.yml +++ b/.github/workflows/dbt_run_scheduled_curated.yml @@ -43,4 +43,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "kaia_models,tag:curated" \ No newline at end of file + dbt run -m "kaia_models,tag:curated" --exclude "models/bronze/api_udf/bronze_api__contract_abis.sql" \ No newline at end of file From 10b0ce838479ee232f023afc92603b15dbc0804a Mon Sep 17 00:00:00 2001 From: mattromano Date: Mon, 14 Apr 2025 08:09:40 -0700 Subject: [PATCH 10/14] remove lookback on fact_traces2 --- models/silver/core/silver__fact_traces2.sql | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql index d791860..713a257 100644 --- a/models/silver/core/silver__fact_traces2.sql +++ b/models/silver/core/silver__fact_traces2.sql @@ -25,14 +25,11 @@ WITH silver_traces AS ( 1 = 1 {% if is_incremental() and not var('full_reload_mode', false) %} - AND block_number >= 160000000 AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM {{ this }} - WHERE - block_number > 160000000 ) {% elif is_incremental() and var('full_reload_mode', false) %} AND block_number < {{ var('RELOAD_BLOCK_MAX', 10000000) }} @@ -271,12 +268,7 @@ incremental_traces AS ( AND f.block_number = t.block_number {% if is_incremental() and not var('full_reload_mode', false)%} - AND t.modified_timestamp >= ( - SELECT - DATEADD('hour', -24, MAX(modified_timestamp)) - FROM - {{ this }} - ) + WHERE t.block_number in (select distinct block_number from silver_traces) {% endif %} ) From ccb949559aa269591eb75331a09d19db774c443f Mon Sep 17 00:00:00 2001 From: mattromano Date: Mon, 14 Apr 2025 08:10:59 -0700 Subject: [PATCH 11/14] remove line --- models/gold/core/core__fact_traces.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql index 598119e..c3e0a60 100644 --- a/models/gold/core/core__fact_traces.sql +++ b/models/gold/core/core__fact_traces.sql @@ -50,5 +50,4 @@ WHERE FROM {{ this }} ) -{% endif %} -dbt run -m models/silver/core/silver__fact_traces2.sql --vars '{"full_reload_mode": true, "RELOAD_BLOCK_MAX": 20000000, "RELOAD_BLOCK_MIN": 10000000}' \ No newline at end of file +{% endif %} \ No newline at end of file From 8c31e4f347a6e36b2c593d6f5f640f0b20741948 Mon Sep 17 00:00:00 2001 From: mattromano Date: Mon, 14 Apr 2025 08:11:42 -0700 Subject: [PATCH 12/14] rename file --- ...s2.sql => silver_observability__fact_traces2_completeness.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/silver/_observability/{silver_observability__fact_traces2.sql => silver_observability__fact_traces2_completeness.sql} (100%) diff --git a/models/silver/_observability/silver_observability__fact_traces2.sql b/models/silver/_observability/silver_observability__fact_traces2_completeness.sql similarity index 100% rename from models/silver/_observability/silver_observability__fact_traces2.sql rename to models/silver/_observability/silver_observability__fact_traces2_completeness.sql From 36cae96b021234fb2a001cda45c780024171bc0a Mon Sep 17 00:00:00 2001 From: mattromano Date: Mon, 14 Apr 2025 08:14:52 -0700 Subject: [PATCH 13/14] remove files --- models/gold/core/core__fact_traces.sql | 2 +- ..._observability__traces_completeness_sf.sql | 116 ------------------ .../tests/traces/test_silver__traces_full.sql | 9 -- .../tests/traces/test_silver__traces_full.yml | 52 -------- .../traces/test_silver__traces_recent.sql | 16 --- .../traces/test_silver__traces_recent.yml | 52 -------- 6 files changed, 1 insertion(+), 246 deletions(-) delete mode 100644 models/silver/_observability/silver_observability__traces_completeness_sf.sql delete mode 100644 models/silver/core/tests/traces/test_silver__traces_full.sql delete mode 100644 models/silver/core/tests/traces/test_silver__traces_full.yml delete mode 100644 models/silver/core/tests/traces/test_silver__traces_recent.sql delete mode 100644 models/silver/core/tests/traces/test_silver__traces_recent.yml diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql index c3e0a60..65f8a61 100644 --- a/models/gold/core/core__fact_traces.sql +++ b/models/gold/core/core__fact_traces.sql @@ -50,4 +50,4 @@ WHERE FROM {{ this }} ) -{% endif %} \ No newline at end of file +{% endif %} diff --git a/models/silver/_observability/silver_observability__traces_completeness_sf.sql b/models/silver/_observability/silver_observability__traces_completeness_sf.sql deleted file mode 100644 index a042d96..0000000 --- a/models/silver/_observability/silver_observability__traces_completeness_sf.sql +++ /dev/null @@ -1,116 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = 'test_timestamp', - full_refresh = false, - tags = ['observability'] -) }} - -WITH summary_stats AS ( - SELECT - MIN(block_number) AS min_block, - MAX(block_number) AS max_block, - MIN(block_timestamp) AS min_block_timestamp, - MAX(block_timestamp) AS max_block_timestamp, - COUNT(1) AS blocks_tested - FROM - {{ ref('silver__blocks') }} - WHERE - block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) - AND block_timestamp >= '2024-02-23 00:00:00.000' --earliest trace data backfilled - -{% if is_incremental() %} -AND ( - block_number >= ( - SELECT - MIN(block_number) - FROM - ( - SELECT - MIN(block_number) AS block_number - FROM - {{ ref('silver__blocks') }} - WHERE - block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) - AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) - UNION - SELECT - MIN(VALUE) - 1 AS block_number - FROM - ( - SELECT - blocks_impacted_array - FROM - {{ this }} - qualify ROW_NUMBER() over ( - ORDER BY - test_timestamp DESC - ) = 1 - ), - LATERAL FLATTEN( - input => blocks_impacted_array - ) - ) - ) {% if var('OBSERV_FULL_TEST') %} - OR block_number >= 0 - {% endif %} -) -{% endif %} -), -block_range AS ( - SELECT - _id AS block_number - FROM - {{ ref('silver__number_sequence') }} - WHERE - _id BETWEEN ( - SELECT - min_block - FROM - summary_stats - ) - AND ( - SELECT - max_block - FROM - summary_stats - ) -), -broken_blocks AS ( - SELECT - DISTINCT block_number - FROM - {{ ref("silver__transactions") }} - tx - LEFT JOIN {{ ref("silver__traces") }} - tr USING ( - block_number, - tx_hash - ) - JOIN block_range USING (block_number) - WHERE - tr.tx_hash IS NULL -), -impacted_blocks AS ( - SELECT - COUNT(1) AS blocks_impacted_count, - ARRAY_AGG(block_number) within GROUP ( - ORDER BY - block_number - ) AS blocks_impacted_array - FROM - broken_blocks -) -SELECT - 'traces' AS test_name, - min_block, - max_block, - min_block_timestamp, - max_block_timestamp, - blocks_tested, - blocks_impacted_count, - blocks_impacted_array, - CURRENT_TIMESTAMP() AS test_timestamp -FROM - summary_stats - JOIN impacted_blocks - ON 1 = 1 diff --git a/models/silver/core/tests/traces/test_silver__traces_full.sql b/models/silver/core/tests/traces/test_silver__traces_full.sql deleted file mode 100644 index 8dd4ca8..0000000 --- a/models/silver/core/tests/traces/test_silver__traces_full.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ config ( - materialized = "view", - tags = ['full_test'] -) }} - -SELECT - * -FROM - {{ ref('silver__traces2') }} diff --git a/models/silver/core/tests/traces/test_silver__traces_full.yml b/models/silver/core/tests/traces/test_silver__traces_full.yml deleted file mode 100644 index 0f3ff2b..0000000 --- a/models/silver/core/tests/traces/test_silver__traces_full.yml +++ /dev/null @@ -1,52 +0,0 @@ -version: 2 -models: - - name: test_silver__traces_full - description: "This is a view used to test all of the silver traces model." - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - TRACES_ID - - columns: - - name: BLOCK_NUMBER - tests: - - not_null - - name: TX_POSITION - tests: - - not_null - - name: TRACE_ADDRESS - tests: - - not_null - - name: TRACE_JSON - tests: - - not_null - - name: _INSERTED_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: hour - interval: 2 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_LTZ - - TIMESTAMP_NTZ - - name: INSERTED_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: hour - interval: 2 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_LTZ - - TIMESTAMP_NTZ - - name: MODIFIED_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: hour - interval: 2 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_LTZ - - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.sql b/models/silver/core/tests/traces/test_silver__traces_recent.sql deleted file mode 100644 index 6068370..0000000 --- a/models/silver/core/tests/traces/test_silver__traces_recent.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config ( - materialized = "view", - tags = ['recent_test'] -) }} - -SELECT - * -FROM - {{ ref('silver__traces2') }} -WHERE - block_number > ( - SELECT - block_number - FROM - {{ ref('_block_lookback') }} - ) diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.yml b/models/silver/core/tests/traces/test_silver__traces_recent.yml deleted file mode 100644 index eb8e689..0000000 --- a/models/silver/core/tests/traces/test_silver__traces_recent.yml +++ /dev/null @@ -1,52 +0,0 @@ -version: 2 -models: - - name: test_silver__traces_recent - description: "This is a view used to test the last three days of traces." - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - TRACES_ID - - columns: - - name: BLOCK_NUMBER - tests: - - not_null - - name: TX_POSITION - tests: - - not_null - - name: TRACE_ADDRESS - tests: - - not_null - - name: TRACE_JSON - tests: - - not_null - - name: _INSERTED_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: hour - interval: 2 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_LTZ - - TIMESTAMP_NTZ - - name: INSERTED_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: hour - interval: 2 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_LTZ - - TIMESTAMP_NTZ - - name: MODIFIED_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: hour - interval: 2 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_LTZ - - TIMESTAMP_NTZ \ No newline at end of file From fd35cd3add3028b9e96641c10dd6197eda62c66a Mon Sep 17 00:00:00 2001 From: mattromano Date: Mon, 14 Apr 2025 08:20:15 -0700 Subject: [PATCH 14/14] re add to realtime run --- .github/workflows/dbt_run_scheduled_non_realtime.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_scheduled_non_realtime.yml b/.github/workflows/dbt_run_scheduled_non_realtime.yml index e08ea41..8c310f5 100644 --- a/.github/workflows/dbt_run_scheduled_non_realtime.yml +++ b/.github/workflows/dbt_run_scheduled_non_realtime.yml @@ -43,4 +43,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "kaia_models,tag:non_realtime" --exclude "silver__fact_traces2" \ No newline at end of file + dbt run -m "kaia_models,tag:non_realtime" \ No newline at end of file