From 45a68debed36be9e41a4885fbf318c727dbb4b75 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 6 Aug 2025 12:40:17 -0400 Subject: [PATCH 01/17] AN-6231/modernize incremental predicates (#442) * upd incr predicate on silver core models * upd rest of the models * rm predicate from d+i model --- macros/dbt/get_merge.sql | 4 ++++ models/evm/silver/core/silver_evm__created_contracts.sql | 1 + .../core/complete/streamline__complete_get_evm_blocks.sql | 1 + .../core/complete/streamline__complete_get_evm_receipts.sql | 1 + .../core/complete/streamline__complete_get_evm_traces.sql | 1 + models/gold/core/core__dim_address_mapping.sql | 1 + models/gold/core/core__ez_token_transfers.sql | 2 +- models/gold/core/core__ez_transaction_actors.sql | 2 +- models/silver/core/silver__streamline_blocks.sql | 1 + models/silver/core/silver__streamline_collections.sql | 1 + models/silver/core/silver__streamline_events.sql | 1 + .../silver/core/silver__streamline_transaction_results.sql | 2 +- models/silver/core/silver__streamline_transactions.sql | 1 + .../silver/core/silver__streamline_transactions_final.sql | 1 + models/silver/curated/silver__flow_evm_address_map.sql | 1 + models/silver/defi/silver__swaps_aggregator.sql | 1 + models/silver/transfers/silver__bridge_complete.sql | 1 + models/silver/transfers/silver__token_transfers.sql | 2 +- package-lock.yml | 6 +++--- packages.yml | 2 +- 20 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 macros/dbt/get_merge.sql diff --git a/macros/dbt/get_merge.sql b/macros/dbt/get_merge.sql new file mode 100644 index 0000000..e3c75ef --- /dev/null +++ b/macros/dbt/get_merge.sql @@ -0,0 +1,4 @@ +{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%} + {% set merge_sql = fsc_utils.get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %} + {{ return(merge_sql) }} +{% endmacro %} diff --git a/models/evm/silver/core/silver_evm__created_contracts.sql b/models/evm/silver/core/silver_evm__created_contracts.sql index 5a31cd3..5e4f038 100644 --- a/models/evm/silver/core/silver_evm__created_contracts.sql +++ b/models/evm/silver/core/silver_evm__created_contracts.sql @@ -1,5 +1,6 @@ {{ config ( materialized = "incremental", + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], unique_key = "created_contract_address", merge_exclude_columns = ["inserted_timestamp"], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_timestamp, tx_hash, created_contract_address, creator_address), SUBSTRING(created_contract_address, creator_address)", diff --git a/models/evm/streamline/core/complete/streamline__complete_get_evm_blocks.sql b/models/evm/streamline/core/complete/streamline__complete_get_evm_blocks.sql index 2d4ea66..05813ed 100644 --- a/models/evm/streamline/core/complete/streamline__complete_get_evm_blocks.sql +++ b/models/evm/streamline/core/complete/streamline__complete_get_evm_blocks.sql @@ -2,6 +2,7 @@ -- depends_on: {{ ref('bronze_evm__FR_blocks') }} {{ config ( materialized = "incremental", + incremental_predicates = ["dynamic_range_predicate", "partition_key"], unique_key = "block_number", cluster_by = "ROUND(block_number, -3)", merge_exclude_columns = ["inserted_timestamp"], diff --git a/models/evm/streamline/core/complete/streamline__complete_get_evm_receipts.sql b/models/evm/streamline/core/complete/streamline__complete_get_evm_receipts.sql index 07d101e..ffc7c97 100644 --- a/models/evm/streamline/core/complete/streamline__complete_get_evm_receipts.sql +++ b/models/evm/streamline/core/complete/streamline__complete_get_evm_receipts.sql @@ -2,6 +2,7 @@ -- depends_on: {{ ref('bronze_evm__FR_receipts') }} {{ config ( materialized = "incremental", + incremental_predicates = ["dynamic_range_predicate", "partition_key"], unique_key = "block_number", cluster_by = "ROUND(block_number, -3)", merge_exclude_columns = ["inserted_timestamp"], diff --git a/models/evm/streamline/core/complete/streamline__complete_get_evm_traces.sql b/models/evm/streamline/core/complete/streamline__complete_get_evm_traces.sql index 3b0d012..35ae4f4 100644 --- a/models/evm/streamline/core/complete/streamline__complete_get_evm_traces.sql +++ b/models/evm/streamline/core/complete/streamline__complete_get_evm_traces.sql @@ -2,6 +2,7 @@ -- depends_on: {{ ref('bronze_evm__FR_traces') }} {{ config ( materialized = "incremental", + incremental_predicates = ["dynamic_range_predicate", "partition_key"], unique_key = "block_number", cluster_by = "ROUND(block_number, -3)", merge_exclude_columns = ["inserted_timestamp"], diff --git a/models/gold/core/core__dim_address_mapping.sql b/models/gold/core/core__dim_address_mapping.sql index 9de9bc5..5e45c24 100644 --- a/models/gold/core/core__dim_address_mapping.sql +++ b/models/gold/core/core__dim_address_mapping.sql @@ -1,5 +1,6 @@ {{ config ( materialized = 'incremental', + incremental_predicates = ["dynamic_range_predicate", "block_timestamp_associated::DATE"], incremental_strategy = 'merge', merge_exclude_columns = ['inserted_timestamp'], unique_key = 'dim_address_mapping_id', diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index b6ffe73..e14172e 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'merge', merge_exclude_columns = ['inserted_timestamp'], - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], cluster_by = ['block_timestamp::date', 'modified_timestamp::date'], unique_key = "ez_token_transfers_id", post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,sender,recipient,token_contract);", diff --git a/models/gold/core/core__ez_transaction_actors.sql b/models/gold/core/core__ez_transaction_actors.sql index b7693bc..b95ec49 100644 --- a/models/gold/core/core__ez_transaction_actors.sql +++ b/models/gold/core/core__ez_transaction_actors.sql @@ -3,7 +3,7 @@ unique_key = 'ez_transaction_actors_id', incremental_strategy = 'merge', merge_exclude_columns = ['inserted_timestamp'], - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], cluster_by = 'block_timestamp::date', post_hook = 'ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,actors);', tags = ['scheduled_non_core'] diff --git a/models/silver/core/silver__streamline_blocks.sql b/models/silver/core/silver__streamline_blocks.sql index 74b7d51..a82182f 100644 --- a/models/silver/core/silver__streamline_blocks.sql +++ b/models/silver/core/silver__streamline_blocks.sql @@ -5,6 +5,7 @@ materialized = 'incremental', unique_key = "block_number", incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate", "_partition_by_block_id"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = "block_timestamp::date", tags = ['streamline_load', 'core', 'scheduled_core'] diff --git a/models/silver/core/silver__streamline_collections.sql b/models/silver/core/silver__streamline_collections.sql index d7d1bc4..077b528 100644 --- a/models/silver/core/silver__streamline_collections.sql +++ b/models/silver/core/silver__streamline_collections.sql @@ -3,6 +3,7 @@ materialized = 'incremental', unique_key = "collection_id", incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate", "block_number"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['_inserted_timestamp :: DATE', 'block_number'], tags = ['streamline_load', 'core', 'scheduled_core'] diff --git a/models/silver/core/silver__streamline_events.sql b/models/silver/core/silver__streamline_events.sql index 341f15b..5c72df1 100644 --- a/models/silver/core/silver__streamline_events.sql +++ b/models/silver/core/silver__streamline_events.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = 'event_id', incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = "block_timestamp::date", post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,event_id,event_contract,event_type);", diff --git a/models/silver/core/silver__streamline_transaction_results.sql b/models/silver/core/silver__streamline_transaction_results.sql index 2c17f89..013cd29 100644 --- a/models/silver/core/silver__streamline_transaction_results.sql +++ b/models/silver/core/silver__streamline_transaction_results.sql @@ -1,7 +1,7 @@ -- depends_on: {{ ref('bronze__streamline_transaction_results') }} {{ config( materialized = 'incremental', - incremental_predicates = ['DBT_INTERNAL_DEST.block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], + incremental_predicates = ["dynamic_range_predicate", "_partition_by_block_id"], unique_key = "tx_id", incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], diff --git a/models/silver/core/silver__streamline_transactions.sql b/models/silver/core/silver__streamline_transactions.sql index cf7b680..dcb497c 100644 --- a/models/silver/core/silver__streamline_transactions.sql +++ b/models/silver/core/silver__streamline_transactions.sql @@ -3,6 +3,7 @@ materialized = 'incremental', unique_key = "tx_id", incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate", "_partition_by_block_id"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = "_inserted_timestamp::date", tags = ['streamline_load', 'core', 'scheduled_core'] diff --git a/models/silver/core/silver__streamline_transactions_final.sql b/models/silver/core/silver__streamline_transactions_final.sql index 7321de9..404c2ba 100644 --- a/models/silver/core/silver__streamline_transactions_final.sql +++ b/models/silver/core/silver__streamline_transactions_final.sql @@ -3,6 +3,7 @@ materialized = 'incremental', unique_key = "tx_id", incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate", "_partition_by_block_id"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = "block_timestamp::date", post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,proposer,payer,authorizers);", diff --git a/models/silver/curated/silver__flow_evm_address_map.sql b/models/silver/curated/silver__flow_evm_address_map.sql index 6f0a698..2458d74 100644 --- a/models/silver/curated/silver__flow_evm_address_map.sql +++ b/models/silver/curated/silver__flow_evm_address_map.sql @@ -1,5 +1,6 @@ {{ config( materialized = 'incremental', + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], unique_key = 'flow_evm_address_map_id', incremental_strategy = 'merge', merge_exclude_columns = ['inserted_timestamp'], diff --git a/models/silver/defi/silver__swaps_aggregator.sql b/models/silver/defi/silver__swaps_aggregator.sql index 835e826..d9a29f0 100644 --- a/models/silver/defi/silver__swaps_aggregator.sql +++ b/models/silver/defi/silver__swaps_aggregator.sql @@ -1,5 +1,6 @@ {{ config( materialized = 'incremental', + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['inserted_timestamp::DATE'], diff --git a/models/silver/transfers/silver__bridge_complete.sql b/models/silver/transfers/silver__bridge_complete.sql index 1c66e4b..5c76630 100644 --- a/models/silver/transfers/silver__bridge_complete.sql +++ b/models/silver/transfers/silver__bridge_complete.sql @@ -1,5 +1,6 @@ {{ config( materialized = 'incremental', + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::date'], diff --git a/models/silver/transfers/silver__token_transfers.sql b/models/silver/transfers/silver__token_transfers.sql index f0dd543..ca20a22 100644 --- a/models/silver/transfers/silver__token_transfers.sql +++ b/models/silver/transfers/silver__token_transfers.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'merge', merge_exclude_columns = ['inserted_timestamp'], - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], cluster_by = ['block_timestamp::date', 'modified_timestamp::date'], unique_key = "token_transfers_id", tags = ['scheduled_non_core'] diff --git a/package-lock.yml b/package-lock.yml index acb1731..68f92db 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -6,11 +6,11 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: d3cf679e079f0cf06142de9386f215e55fe26b3b + revision: 87e00eb90acddcc7a34aa8e67e3b3bac86b262e6 - package: get-select/dbt_snowflake_query_tags version: 2.5.0 - package: calogica/dbt_date version: 0.7.2 - git: https://github.com/FlipsideCrypto/livequery-models.git - revision: b024188be4e9c6bc00ed77797ebdc92d351d620e -sha1_hash: 3fb8d6ca492a03f5aef6f281508aaa0b34c989d4 + revision: 2651a45b7e123f7bd421bcc0e7e2a7bcbaf7652f +sha1_hash: a1cc3545d7ef13fcf5b3908a9e888b4421018792 diff --git a/packages.yml b/packages.yml index c5ea06d..1b272c8 100644 --- a/packages.yml +++ b/packages.yml @@ -6,6 +6,6 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.32.0 + revision: v1.35.1 - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"] From 4603b205910944fa8403aa55f932ffd9e273ade5 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 11 Aug 2025 14:05:06 -0400 Subject: [PATCH 02/17] only not null if tx succeeded (#463) --- models/gold/core/core__ez_token_transfers.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/gold/core/core__ez_token_transfers.yml b/models/gold/core/core__ez_token_transfers.yml index b0fffdb..fb5dc33 100644 --- a/models/gold/core/core__ez_token_transfers.yml +++ b/models/gold/core/core__ez_token_transfers.yml @@ -54,7 +54,8 @@ models: - name: TOKEN_CONTRACT description: "{{ doc('token_contract') }}" tests: - - not_null + - not_null: + where: tx_succeeded - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING From 198f4a2f86b567e4dc23e9323006329608218622 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 27 Aug 2025 12:20:15 -0600 Subject: [PATCH 03/17] disable rewards GHAs (#466) * disable flow rewards gha * excl gold rewards from dbt test --- .github/workflows/dbt_run_scheduled_streamline_non_core.yml | 4 +--- .../dbt_run_streamline_external_points_balances_realtime.yml | 3 ++- .github/workflows/dbt_run_streamline_external_realtime.yml | 3 ++- .github/workflows/dbt_test.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dbt_run_scheduled_streamline_non_core.yml b/.github/workflows/dbt_run_scheduled_streamline_non_core.yml index 65ac3a1..14097a0 100644 --- a/.github/workflows/dbt_run_scheduled_streamline_non_core.yml +++ b/.github/workflows/dbt_run_scheduled_streamline_non_core.yml @@ -3,9 +3,7 @@ run-name: dbt_run_scheduled_streamline_non_core on: workflow_dispatch: - schedule: - # Daily at 06:00 UTC - - cron: "0 6 * * *" + # Disabled 2025-08-27 due to no responses from the points (snag) API env: SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" diff --git a/.github/workflows/dbt_run_streamline_external_points_balances_realtime.yml b/.github/workflows/dbt_run_streamline_external_points_balances_realtime.yml index 65f5197..da4fce5 100644 --- a/.github/workflows/dbt_run_streamline_external_points_balances_realtime.yml +++ b/.github/workflows/dbt_run_streamline_external_points_balances_realtime.yml @@ -8,7 +8,8 @@ on: workflow_dispatch: schedule: # Daily at 0, 1, 2, 3, 4 UTC - - cron: "0 0,1,2,3,4 * * *" + # Disabled 2025-08-27 due to no responses from the points (snag) API + # - cron: "0 0,1,2,3,4 * * *" env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_streamline_external_realtime.yml b/.github/workflows/dbt_run_streamline_external_realtime.yml index 1f77f5f..bfdd3e3 100644 --- a/.github/workflows/dbt_run_streamline_external_realtime.yml +++ b/.github/workflows/dbt_run_streamline_external_realtime.yml @@ -7,7 +7,8 @@ on: workflow_dispatch: schedule: # Runs hourly - - cron: "0 * * * *" + # Disabled 2025-08-27 due to no responses from the points (snag) API + # - cron: "0 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index 2646e64..89a44de 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -45,7 +45,7 @@ jobs: - name: Run DBT Jobs run: | - dbt test -s "flow_models,models/gold" tag:observability tag:evm_gap_test --vars '{"TEST_RANGE": True, "START_HEIGHT": ${{ vars.STREAMLINE_START_BLOCK }}, "END_HEIGHT": 100000000}' + dbt test -s "flow_models,models/gold" tag:observability tag:evm_gap_test --exclude "flow_models,models/gold/rewards" --vars '{"TEST_RANGE": True, "START_HEIGHT": ${{ vars.STREAMLINE_START_BLOCK }}, "END_HEIGHT": 100000000}' continue-on-error: true - name: Log test results From efaab90e83149a78ccf2a15fcd14ddfb0c7fa10b Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 29 Aug 2025 11:14:41 -0600 Subject: [PATCH 04/17] re-enable points requests (#467) --- .github/workflows/dbt_run_scheduled_streamline_non_core.yml | 4 +++- .../dbt_run_streamline_external_points_balances_realtime.yml | 3 +-- .github/workflows/dbt_run_streamline_external_realtime.yml | 3 +-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dbt_run_scheduled_streamline_non_core.yml b/.github/workflows/dbt_run_scheduled_streamline_non_core.yml index 14097a0..abe2449 100644 --- a/.github/workflows/dbt_run_scheduled_streamline_non_core.yml +++ b/.github/workflows/dbt_run_scheduled_streamline_non_core.yml @@ -3,7 +3,9 @@ run-name: dbt_run_scheduled_streamline_non_core on: workflow_dispatch: - # Disabled 2025-08-27 due to no responses from the points (snag) API + schedule: + # Once daily at 06:00 UTC + - cron: "0 6 * * *" env: SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" diff --git a/.github/workflows/dbt_run_streamline_external_points_balances_realtime.yml b/.github/workflows/dbt_run_streamline_external_points_balances_realtime.yml index da4fce5..65f5197 100644 --- a/.github/workflows/dbt_run_streamline_external_points_balances_realtime.yml +++ b/.github/workflows/dbt_run_streamline_external_points_balances_realtime.yml @@ -8,8 +8,7 @@ on: workflow_dispatch: schedule: # Daily at 0, 1, 2, 3, 4 UTC - # Disabled 2025-08-27 due to no responses from the points (snag) API - # - cron: "0 0,1,2,3,4 * * *" + - cron: "0 0,1,2,3,4 * * *" env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_streamline_external_realtime.yml b/.github/workflows/dbt_run_streamline_external_realtime.yml index bfdd3e3..1f77f5f 100644 --- a/.github/workflows/dbt_run_streamline_external_realtime.yml +++ b/.github/workflows/dbt_run_streamline_external_realtime.yml @@ -7,8 +7,7 @@ on: workflow_dispatch: schedule: # Runs hourly - # Disabled 2025-08-27 due to no responses from the points (snag) API - # - cron: "0 * * * *" + - cron: "0 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" From 18a9c42094f3ddae4114c11f94376ddc36330317 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:18:54 -0600 Subject: [PATCH 05/17] rm tx entries from gha (#468) --- .../dbt_run_scheduled_streamline_non_core.yml | 15 ++++++++------- .../dbt_run_streamline_external_realtime.yml | 4 +++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dbt_run_scheduled_streamline_non_core.yml b/.github/workflows/dbt_run_scheduled_streamline_non_core.yml index abe2449..1bdbb8e 100644 --- a/.github/workflows/dbt_run_scheduled_streamline_non_core.yml +++ b/.github/workflows/dbt_run_scheduled_streamline_non_core.yml @@ -45,14 +45,15 @@ jobs: run: > dbt run -s tag:streamline_non_core - - name: Test DBT Models - run: > - dbt test -s tag:streamline_non_core - continue-on-error: true + # Temporarily disabled 2025-08-29 during Snag API issue triage + # - name: Test DBT Models + # run: > + # dbt test -s tag:streamline_non_core + # continue-on-error: true - - name: Log test results - run: | - python python/dbt_test_alert.py + # - name: Log test results + # run: | + # python python/dbt_test_alert.py - name: Store logs uses: actions/upload-artifact@v4 diff --git a/.github/workflows/dbt_run_streamline_external_realtime.yml b/.github/workflows/dbt_run_streamline_external_realtime.yml index 1f77f5f..5db306c 100644 --- a/.github/workflows/dbt_run_streamline_external_realtime.yml +++ b/.github/workflows/dbt_run_streamline_external_realtime.yml @@ -45,9 +45,11 @@ jobs: pip install -r requirements.txt dbt deps + # 1+streamline__transaction_entries_realtime removed from dbt cmd temporarily + # 2025-08-29 during Snag API issue triage - name: Request Storefront Items and Transaction Entries from Snag API run: > - dbt run -s streamline__minting_assets_realtime 1+streamline__transaction_entries_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + dbt run -s streamline__minting_assets_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' - name: Store logs uses: actions/upload-artifact@v4 From 44a87ef52722689abb1e194c7de79c56f1e6f52e Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:33:11 -0600 Subject: [PATCH 06/17] fix if execute block (#469) --- .github/workflows/dbt_run_streamline_external_realtime.yml | 4 +--- .../streamline__transaction_entries_realtime.sql | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dbt_run_streamline_external_realtime.yml b/.github/workflows/dbt_run_streamline_external_realtime.yml index 5db306c..1f77f5f 100644 --- a/.github/workflows/dbt_run_streamline_external_realtime.yml +++ b/.github/workflows/dbt_run_streamline_external_realtime.yml @@ -45,11 +45,9 @@ jobs: pip install -r requirements.txt dbt deps - # 1+streamline__transaction_entries_realtime removed from dbt cmd temporarily - # 2025-08-29 during Snag API issue triage - name: Request Storefront Items and Transaction Entries from Snag API run: > - dbt run -s streamline__minting_assets_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + dbt run -s streamline__minting_assets_realtime 1+streamline__transaction_entries_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' - name: Store logs uses: actions/upload-artifact@v4 diff --git a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql index f3b4606..ada0938 100644 --- a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql +++ b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql @@ -23,13 +23,13 @@ SELECT entry_id, ROW_NUMBER() over ( + PARTITION BY partition_key ORDER BY - partition_key DESC, - INDEX ASC + INDEX DESC ) AS rn FROM {{ ref('silver_api__transaction_entries') }} - WHERE _inserted_timestamp >= SYSDATE() - INTERVAL '3 days' + WHERE _inserted_timestamp >= SYSDATE() - INTERVAL '10 days' ) SELECT entry_id From 86c227a83ebdfbd54d49c0683eeb2ea7dcf00bdd Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:34:47 -0600 Subject: [PATCH 07/17] stick with 3d (#470) --- .../streamline__transaction_entries_realtime.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql index ada0938..5110293 100644 --- a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql +++ b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql @@ -29,7 +29,7 @@ ) AS rn FROM {{ ref('silver_api__transaction_entries') }} - WHERE _inserted_timestamp >= SYSDATE() - INTERVAL '10 days' + WHERE _inserted_timestamp >= SYSDATE() - INTERVAL '3 days' ) SELECT entry_id From e22e62ff56d07d54038660f721aab7c80e33e5c6 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:37:16 -0600 Subject: [PATCH 08/17] 1w (#471) --- .../streamline__transaction_entries_realtime.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql index 5110293..b0653c0 100644 --- a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql +++ b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql @@ -29,7 +29,7 @@ ) AS rn FROM {{ ref('silver_api__transaction_entries') }} - WHERE _inserted_timestamp >= SYSDATE() - INTERVAL '3 days' + WHERE _inserted_timestamp >= SYSDATE() - INTERVAL '7 days' ) SELECT entry_id From e0a7d7c1d03e99abb1bbaa94ea157f3baa1c205e Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:44:16 -0600 Subject: [PATCH 09/17] up rn declaration (#472) --- .../streamline__transaction_entries_realtime.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql index b0653c0..fcf6f34 100644 --- a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql +++ b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql @@ -23,13 +23,13 @@ SELECT entry_id, ROW_NUMBER() over ( - PARTITION BY partition_key ORDER BY + partition_key DESC, INDEX DESC ) AS rn FROM {{ ref('silver_api__transaction_entries') }} - WHERE _inserted_timestamp >= SYSDATE() - INTERVAL '7 days' + WHERE _inserted_timestamp >= SYSDATE() - INTERVAL '10 days' ) SELECT entry_id From 4e0616780353bd046168f7c4bd16c1614172ebe3 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 4 Sep 2025 08:56:47 -0600 Subject: [PATCH 10/17] set to 100 (#473) --- .../streamline__transaction_entries_realtime.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql index fcf6f34..4d7968b 100644 --- a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql +++ b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql @@ -49,7 +49,7 @@ SELECT {{ var( 'API_LIMIT', - 1000 + 100 ) }} AS api_limit, '{{ starting_after }}' AS starting_after, DATE_PART('EPOCH', SYSDATE()) :: INTEGER AS partition_key, From 84b6ab5f965c6550be6107709755a906e4f1e764 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 4 Sep 2025 08:58:12 -0600 Subject: [PATCH 11/17] set to 2 weeks (#474) --- .../streamline__transaction_entries_realtime.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql index 4d7968b..27721a6 100644 --- a/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql +++ b/models/streamline/external/snag/transaction_entries/streamline__transaction_entries_realtime.sql @@ -29,7 +29,7 @@ ) AS rn FROM {{ ref('silver_api__transaction_entries') }} - WHERE _inserted_timestamp >= SYSDATE() - INTERVAL '10 days' + WHERE _inserted_timestamp >= SYSDATE() - INTERVAL '14 days' ) SELECT entry_id From 9e6e6ecafa2b95649e649d51e3129bc1daa076ff Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 4 Sep 2025 09:15:54 -0600 Subject: [PATCH 12/17] run points request every 15 mins (#475) --- .github/workflows/dbt_run_streamline_external_realtime.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dbt_run_streamline_external_realtime.yml b/.github/workflows/dbt_run_streamline_external_realtime.yml index 1f77f5f..4966db7 100644 --- a/.github/workflows/dbt_run_streamline_external_realtime.yml +++ b/.github/workflows/dbt_run_streamline_external_realtime.yml @@ -6,8 +6,8 @@ description: | on: workflow_dispatch: schedule: - # Runs hourly - - cron: "0 * * * *" + # Runs every 15 minutes + - cron: "0,15,30,45 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" From a2c0db832be6cd27eeb8ed69e502f460189d5cc3 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:08:47 -0400 Subject: [PATCH 13/17] chg request rate to every 6 mins (#476) --- .github/workflows/dbt_run_streamline_external_realtime.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dbt_run_streamline_external_realtime.yml b/.github/workflows/dbt_run_streamline_external_realtime.yml index 4966db7..b3bcfae 100644 --- a/.github/workflows/dbt_run_streamline_external_realtime.yml +++ b/.github/workflows/dbt_run_streamline_external_realtime.yml @@ -6,8 +6,8 @@ description: | on: workflow_dispatch: schedule: - # Runs every 15 minutes - - cron: "0,15,30,45 * * * *" + # Runs every 6 minutes + - cron: "0,6,12,18,24,30,36,42,48,54 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" From 6bc260c8389cad190436da4aacb9fc00bdfaa136 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:12:57 -0400 Subject: [PATCH 14/17] rm chainwalkers from source list (#477) --- models/sources.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/models/sources.yml b/models/sources.yml index 64bed45..5f9e744 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -1,13 +1,6 @@ version: 2 sources: - - name: prod - database: chainwalkers - schema: prod - tables: - - name: flow_blocks - - name: flow_txs - - name: bronze_streamline database: streamline schema: | From 7341100552bbf2974f19727bdcf9fda8882e1243 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:17:30 -0400 Subject: [PATCH 15/17] rm old bronze views (#478) --- models/bronze/bronze__blocks.sql | 23 ----------------------- models/bronze/bronze__transactions.sql | 24 ------------------------ 2 files changed, 47 deletions(-) delete mode 100644 models/bronze/bronze__blocks.sql delete mode 100644 models/bronze/bronze__transactions.sql diff --git a/models/bronze/bronze__blocks.sql b/models/bronze/bronze__blocks.sql deleted file mode 100644 index f5aa142..0000000 --- a/models/bronze/bronze__blocks.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config ( - materialized = 'view', - tags = ['scheduled'] -) }} - -SELECT - record_id, - offset_id, - block_id, - block_timestamp, - network, - chain_id, - tx_count, - header, - ingested_at AS _ingested_at, - _inserted_timestamp -FROM - {{ source( - 'prod', - 'flow_blocks' - ) }} -WHERE - _inserted_timestamp :: DATE >= '2022-05-01' diff --git a/models/bronze/bronze__transactions.sql b/models/bronze/bronze__transactions.sql deleted file mode 100644 index 495e032..0000000 --- a/models/bronze/bronze__transactions.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config ( - materialized = 'view', - tags = ['scheduled'] -) }} - -SELECT - record_id, - tx_id, - tx_block_index, - offset_id, - block_id, - block_timestamp, - network, - chain_id, - tx, - ingested_at AS _ingested_at, - _inserted_timestamp -FROM - {{ source( - 'prod', - 'flow_txs' - ) }} -WHERE - _inserted_timestamp :: DATE >= '2022-05-01' From fc75a1860d979b4a69512080e9ecbc2a5e9fe31a Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 12 Sep 2025 11:55:19 -0600 Subject: [PATCH 16/17] AN-6551/EVM Testnet (#480) * init evm testnet models * upd sources.yml * fix _missing_receipts ref * set min block * upd dev streamline resources * gha and rename schema to jut testnet * fix refs * fix refs 2 * tests * rm space --------- Co-authored-by: Austin --- .github/workflows/dbt_run_evm_testnet.yml | 53 +++ ...bt_run_streamline_evm_testnet_realtime.yml | 53 +++ macros/streamline/api_integrations.sql | 4 +- macros/streamline/streamline_udfs.sql | 2 +- .../testnet/bronze_evm__FR_testnet_blocks.sql | 8 + .../bronze_evm__FR_testnet_receipts.sql | 8 + .../testnet/bronze_evm__FR_testnet_traces.sql | 9 + .../testnet/bronze_evm__testnet_blocks.sql | 8 + .../testnet/bronze_evm__testnet_receipts.sql | 8 + .../testnet/bronze_evm__testnet_traces.sql | 9 + .../testnet/core/testnet__fact_evm_blocks.sql | 64 +++ .../testnet/core/testnet__fact_evm_blocks.yml | 56 +++ .../core/testnet__fact_evm_event_logs.sql | 223 +++++++++++ .../core/testnet__fact_evm_event_logs.yml | 46 +++ .../testnet/core/testnet__fact_evm_traces.sql | 379 ++++++++++++++++++ .../testnet/core/testnet__fact_evm_traces.yml | 62 +++ .../core/testnet__fact_evm_transactions.sql | 348 ++++++++++++++++ .../core/testnet__fact_evm_transactions.yml | 68 ++++ ...est_gold_testnet_evm__fact_blocks_full.sql | 9 + ...est_gold_testnet_evm__fact_blocks_full.yml | 145 +++++++ ...t_gold_testnet_evm__fact_blocks_recent.sql | 16 + ...t_gold_testnet_evm__fact_blocks_recent.yml | 147 +++++++ ...gold_testnet_evm__fact_event_logs_full.sql | 9 + ...gold_testnet_evm__fact_event_logs_full.yml | 101 +++++ ...ld_testnet_evm__fact_event_logs_recent.sql | 16 + ...ld_testnet_evm__fact_event_logs_recent.yml | 100 +++++ ...est_gold_testnet_evm__fact_traces_full.sql | 9 + ...est_gold_testnet_evm__fact_traces_full.yml | 120 ++++++ ...t_gold_testnet_evm__fact_traces_recent.sql | 16 + ...t_gold_testnet_evm__fact_traces_recent.yml | 120 ++++++ ...ld_testnet_evm__fact_transactions_full.sql | 9 + ...ld_testnet_evm__fact_transactions_full.yml | 125 ++++++ ..._testnet_evm__fact_transactions_recent.sql | 16 + ..._testnet_evm__fact_transactions_recent.yml | 124 ++++++ .../core/silver_evm__testnet_blocks.sql | 42 ++ .../core/silver_evm__testnet_receipts.sql | 61 +++ .../core/silver_evm__testnet_traces.sql | 141 +++++++ .../core/silver_evm__testnet_transactions.sql | 53 +++ .../testnet/_evm_testnet_block_lookback.sql | 11 + ...mline__complete_get_evm_testnet_blocks.sql | 44 ++ ...ine__complete_get_evm_testnet_receipts.sql | 43 ++ ...mline__complete_get_evm_testnet_traces.sql | 45 +++ ...mline__get_evm_testnet_blocks_realtime.sql | 89 ++++ ...ine__get_evm_testnet_receipts_realtime.sql | 99 +++++ ...mline__get_evm_testnet_traces_realtime.sql | 101 +++++ .../retry/_missing_testnet_receipts.sql | 16 + .../streamline__evm_testnet_blocks.sql | 14 + .../streamline__evm_testnet_chainhead.sql | 28 ++ models/sources.yml | 3 + 49 files changed, 3277 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/dbt_run_evm_testnet.yml create mode 100644 .github/workflows/dbt_run_streamline_evm_testnet_realtime.yml create mode 100644 models/evm/bronze/testnet/bronze_evm__FR_testnet_blocks.sql create mode 100644 models/evm/bronze/testnet/bronze_evm__FR_testnet_receipts.sql create mode 100644 models/evm/bronze/testnet/bronze_evm__FR_testnet_traces.sql create mode 100644 models/evm/bronze/testnet/bronze_evm__testnet_blocks.sql create mode 100644 models/evm/bronze/testnet/bronze_evm__testnet_receipts.sql create mode 100644 models/evm/bronze/testnet/bronze_evm__testnet_traces.sql create mode 100644 models/evm/gold/testnet/core/testnet__fact_evm_blocks.sql create mode 100644 models/evm/gold/testnet/core/testnet__fact_evm_blocks.yml create mode 100644 models/evm/gold/testnet/core/testnet__fact_evm_event_logs.sql create mode 100644 models/evm/gold/testnet/core/testnet__fact_evm_event_logs.yml create mode 100644 models/evm/gold/testnet/core/testnet__fact_evm_traces.sql create mode 100644 models/evm/gold/testnet/core/testnet__fact_evm_traces.yml create mode 100644 models/evm/gold/testnet/core/testnet__fact_evm_transactions.sql create mode 100644 models/evm/gold/testnet/core/testnet__fact_evm_transactions.yml create mode 100644 models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_full.sql create mode 100644 models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_full.yml create mode 100644 models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_recent.sql create mode 100644 models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_recent.yml create mode 100644 models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_full.sql create mode 100644 models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_full.yml create mode 100644 models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_recent.sql create mode 100644 models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_recent.yml create mode 100644 models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_full.sql create mode 100644 models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_full.yml create mode 100644 models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_recent.sql create mode 100644 models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_recent.yml create mode 100644 models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_full.sql create mode 100644 models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_full.yml create mode 100644 models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_recent.sql create mode 100644 models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_recent.yml create mode 100644 models/evm/silver/testnet/core/silver_evm__testnet_blocks.sql create mode 100644 models/evm/silver/testnet/core/silver_evm__testnet_receipts.sql create mode 100644 models/evm/silver/testnet/core/silver_evm__testnet_traces.sql create mode 100644 models/evm/silver/testnet/core/silver_evm__testnet_transactions.sql create mode 100644 models/evm/streamline/testnet/_evm_testnet_block_lookback.sql create mode 100644 models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_blocks.sql create mode 100644 models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_receipts.sql create mode 100644 models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_traces.sql create mode 100644 models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_blocks_realtime.sql create mode 100644 models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_receipts_realtime.sql create mode 100644 models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_traces_realtime.sql create mode 100644 models/evm/streamline/testnet/retry/_missing_testnet_receipts.sql create mode 100644 models/evm/streamline/testnet/streamline__evm_testnet_blocks.sql create mode 100644 models/evm/streamline/testnet/streamline__evm_testnet_chainhead.sql diff --git a/.github/workflows/dbt_run_evm_testnet.yml b/.github/workflows/dbt_run_evm_testnet.yml new file mode 100644 index 0000000..95f6ede --- /dev/null +++ b/.github/workflows/dbt_run_evm_testnet.yml @@ -0,0 +1,53 @@ +name: dbt_run_evm +run-name: dbt_run_evm + +on: + workflow_dispatch: + schedule: + # Every 4 hours at 30 minutes past the hour + - cron: "30 */4 * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Update evm tables + run: | + dbt run -s tag:evm_testnet + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.EVM_SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_evm_testnet_realtime.yml b/.github/workflows/dbt_run_streamline_evm_testnet_realtime.yml new file mode 100644 index 0000000..c4d40c9 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_evm_testnet_realtime.yml @@ -0,0 +1,53 @@ +name: dbt_run_streamline_evm_testnet_realtime +run-name: dbt_run_streamline_evm_testnet_realtime + +on: + workflow_dispatch: + schedule: + # Every 4 hours + - cron: "0 */4 * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -s 2+tag:streamline_evm_testnet_realtime + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.EVM_SLACK_WEBHOOK_URL }} diff --git a/macros/streamline/api_integrations.sql b/macros/streamline/api_integrations.sql index 376eb7e..d9dc5f4 100644 --- a/macros/streamline/api_integrations.sql +++ b/macros/streamline/api_integrations.sql @@ -41,8 +41,8 @@ {% do run_query(sql) %} {% set sql %} - CREATE api integration IF NOT EXISTS aws_flow_evm_api_dev api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::704693948482:role/flow-api-stg-rolesnowflakeudfsAF733095-tPEdygwPC6IV' api_allowed_prefixes = ( - 'https://pfv9lhg3kg.execute-api.us-east-1.amazonaws.com/stg/' + CREATE api integration IF NOT EXISTS aws_flow_evm_api_dev api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::704693948482:role/flow-api-stg-rolesnowflakeudfsAF733095-ybejBONVMTd4' api_allowed_prefixes = ( + 'https://2hcu4hei27.execute-api.us-east-1.amazonaws.com/stg/' ) enabled = TRUE; {% endset %} {% do run_query(sql) %} diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql index 5a97a05..bea1a25 100644 --- a/macros/streamline/streamline_udfs.sql +++ b/macros/streamline/streamline_udfs.sql @@ -88,7 +88,7 @@ {% if target.name == "prod" %} aws_flow_evm_api_prod AS 'https://rajpkbgko9.execute-api.us-east-1.amazonaws.com/prod/udf_bulk_rest_api' {% else %} - aws_flow_evm_api_dev AS 'https://pfv9lhg3kg.execute-api.us-east-1.amazonaws.com/stg/udf_bulk_rest_api' + aws_flow_evm_api_dev AS 'https://2hcu4hei27.execute-api.us-east-1.amazonaws.com/stg/udf_bulk_rest_api' {%- endif %}; {% endmacro %} diff --git a/models/evm/bronze/testnet/bronze_evm__FR_testnet_blocks.sql b/models/evm/bronze/testnet/bronze_evm__FR_testnet_blocks.sql new file mode 100644 index 0000000..af7a840 --- /dev/null +++ b/models/evm/bronze/testnet/bronze_evm__FR_testnet_blocks.sql @@ -0,0 +1,8 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_FR_query_v2( + model = "evm_testnet_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/bronze/testnet/bronze_evm__FR_testnet_receipts.sql b/models/evm/bronze/testnet/bronze_evm__FR_testnet_receipts.sql new file mode 100644 index 0000000..41251fe --- /dev/null +++ b/models/evm/bronze/testnet/bronze_evm__FR_testnet_receipts.sql @@ -0,0 +1,8 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_FR_query_v2( + model = "evm_testnet_receipts", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/bronze/testnet/bronze_evm__FR_testnet_traces.sql b/models/evm/bronze/testnet/bronze_evm__FR_testnet_traces.sql new file mode 100644 index 0000000..c8eeebc --- /dev/null +++ b/models/evm/bronze/testnet/bronze_evm__FR_testnet_traces.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['traces'] +) }} + +{{ streamline_external_table_FR_query_v2( + model = "evm_testnet_traces", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/bronze/testnet/bronze_evm__testnet_blocks.sql b/models/evm/bronze/testnet/bronze_evm__testnet_blocks.sql new file mode 100644 index 0000000..6af9bbf --- /dev/null +++ b/models/evm/bronze/testnet/bronze_evm__testnet_blocks.sql @@ -0,0 +1,8 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_query_v2( + model = "evm_testnet_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/bronze/testnet/bronze_evm__testnet_receipts.sql b/models/evm/bronze/testnet/bronze_evm__testnet_receipts.sql new file mode 100644 index 0000000..38af2d6 --- /dev/null +++ b/models/evm/bronze/testnet/bronze_evm__testnet_receipts.sql @@ -0,0 +1,8 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_query_v2( + model = "evm_testnet_receipts", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/bronze/testnet/bronze_evm__testnet_traces.sql b/models/evm/bronze/testnet/bronze_evm__testnet_traces.sql new file mode 100644 index 0000000..eccbac5 --- /dev/null +++ b/models/evm/bronze/testnet/bronze_evm__testnet_traces.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['traces'] +) }} + +{{ streamline_external_table_query_v2( + model = "evm_testnet_traces", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/gold/testnet/core/testnet__fact_evm_blocks.sql b/models/evm/gold/testnet/core/testnet__fact_evm_blocks.sql new file mode 100644 index 0000000..b14d0fb --- /dev/null +++ b/models/evm/gold/testnet/core/testnet__fact_evm_blocks.sql @@ -0,0 +1,64 @@ +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['block_timestamp::DATE'], + tags = ['evm_testnet'] +) }} + +SELECT + block_number, + block_json :hash :: STRING AS block_hash, + utils.udf_hex_to_int( + block_json :timestamp :: STRING + ) :: TIMESTAMP AS block_timestamp, + 'testnet' AS network, + ARRAY_SIZE( + block_json :transactions + ) AS tx_count, + utils.udf_hex_to_int( + block_json :size :: STRING + ) :: bigint AS SIZE, + block_json :miner :: STRING AS miner, + block_json :mixHash :: STRING AS mix_hash, + block_json :extraData :: STRING AS extra_data, + block_json :parentHash :: STRING AS parent_hash, + utils.udf_hex_to_int( + block_json :gasUsed :: STRING + ) :: bigint AS gas_used, + utils.udf_hex_to_int( + block_json :gasLimit :: STRING + ) :: bigint AS gas_limit, + utils.udf_hex_to_int( + block_json :baseFeePerGas :: STRING + ) :: bigint AS base_fee_per_gas, + utils.udf_hex_to_int( + block_json :difficulty :: STRING + ) :: bigint AS difficulty, + utils.udf_hex_to_int( + block_json :totalDifficulty :: STRING + ) :: bigint AS total_difficulty, + block_json :sha3Uncles :: STRING AS sha3_uncles, + block_json :uncles AS uncle_blocks, + utils.udf_hex_to_int( + block_json :nonce :: STRING + ) :: bigint AS nonce, + block_json :receiptsRoot :: STRING AS receipts_root, + block_json :stateRoot :: STRING AS state_root, + block_json :transactionsRoot :: STRING AS transactions_root, + block_json :logsBloom :: STRING AS logs_bloom, + {{ dbt_utils.generate_surrogate_key(['block_number']) }} AS fact_blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver_evm__testnet_blocks') }} +WHERE 1=1 + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }} + ) +{% endif %} \ No newline at end of file diff --git a/models/evm/gold/testnet/core/testnet__fact_evm_blocks.yml b/models/evm/gold/testnet/core/testnet__fact_evm_blocks.yml new file mode 100644 index 0000000..2088c2d --- /dev/null +++ b/models/evm/gold/testnet/core/testnet__fact_evm_blocks.yml @@ -0,0 +1,56 @@ +version: 2 +models: + - name: testnet__fact_evm_blocks + description: '{{ doc("evm_blocks_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("evm_block_number") }}' + - name: BLOCK_HASH + description: '{{ doc("evm_blocks_hash") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("evm_block_timestamp") }}' + - name: NETWORK + description: '{{ doc("evm_network") }}' + - name: TX_COUNT + description: '{{ doc("evm_tx_count") }}' + - name: SIZE + description: '{{ doc("evm_size") }}' + - name: MINER + description: '{{ doc("evm_miner") }}' + - name: BASE_FEE_PER_GAS + description: '{{ doc("evm_base_fee_per_gas") }}' + - name: MIX_HASH + description: '{{ doc("evm_mix_hash") }}' + - name: EXTRA_DATA + description: '{{ doc("evm_extra_data") }}' + - name: PARENT_HASH + description: '{{ doc("evm_parent_hash") }}' + - name: GAS_USED + description: '{{ doc("evm_gas_used") }}' + - name: GAS_LIMIT + description: '{{ doc("evm_gas_limit") }}' + - name: DIFFICULTY + description: '{{ doc("evm_difficulty") }}' + - name: TOTAL_DIFFICULTY + description: '{{ doc("evm_total_difficulty") }}' + - name: SHA3_UNCLES + description: '{{ doc("evm_sha3_uncles") }}' + - name: UNCLE_BLOCKS + description: '{{ doc("evm_uncle_blocks") }}' + - name: NONCE + description: '{{ doc("evm_blocks_nonce") }}' + - name: RECEIPTS_ROOT + description: '{{ doc("evm_receipts_root") }}' + - name: STATE_ROOT + description: '{{ doc("evm_state_root") }}' + - name: TRANSACTIONS_ROOT + description: '{{ doc("evm_transactions_root") }}' + - name: LOGS_BLOOM + description: '{{ doc("evm_logs_bloom") }}' + - name: FACT_BLOCKS_ID + description: '{{ doc("evm_pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("evm_inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/evm/gold/testnet/core/testnet__fact_evm_event_logs.sql b/models/evm/gold/testnet/core/testnet__fact_evm_event_logs.sql new file mode 100644 index 0000000..1144042 --- /dev/null +++ b/models/evm/gold/testnet/core/testnet__fact_evm_event_logs.sql @@ -0,0 +1,223 @@ +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['block_timestamp::DATE'], + tags = ['evm_testnet'] +) }} + +WITH base AS ( + + SELECT + block_number, + {% if uses_receipts_by_hash %} + tx_hash, + {% else %} + receipts_json :transactionHash :: STRING AS tx_hash, + {% endif %} + receipts_json, + receipts_json :logs AS full_logs + FROM + {{ ref('silver_evm__testnet_receipts') }} + WHERE + 1 = 1 + AND ARRAY_SIZE(receipts_json :logs) > 0 + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }}) + {% endif %} +), +flattened_logs AS ( + SELECT + block_number, + tx_hash, + lower(receipts_json :from :: STRING) AS origin_from_address, + lower(receipts_json :to :: STRING) AS origin_to_address, + CASE + WHEN receipts_json :status :: STRING = '0x1' THEN TRUE + WHEN receipts_json :status :: STRING = '1' THEN TRUE + WHEN receipts_json :status :: STRING = '0x0' THEN FALSE + WHEN receipts_json :status :: STRING = '0' THEN FALSE + ELSE NULL + END AS tx_succeeded, + VALUE :address :: STRING AS contract_address, + VALUE :blockHash :: STRING AS block_hash, + VALUE :blockNumber :: STRING AS block_number_hex, + VALUE :data :: STRING AS DATA, + utils.udf_hex_to_int( + VALUE :logIndex :: STRING + ) :: INT AS event_index, + VALUE :removed :: BOOLEAN AS event_removed, + VALUE :topics AS topics, + VALUE :transactionHash :: STRING AS transaction_hash, + utils.udf_hex_to_int( + VALUE :transactionIndex :: STRING + ) :: INT AS transaction_index + FROM + base, + LATERAL FLATTEN ( + input => full_logs + ) +), +new_logs AS ( + SELECT + l.block_number, + b.block_timestamp, + l.tx_hash, + l.transaction_index AS tx_position, + l.event_index, + l.contract_address, + l.topics, + l.topics [0] :: STRING AS topic_0, + l.topics [1] :: STRING AS topic_1, + l.topics [2] :: STRING AS topic_2, + l.topics [3] :: STRING AS topic_3, + l.data, + l.event_removed, + txs.from_address AS origin_from_address, + txs.to_address AS origin_to_address, + txs.origin_function_signature, + l.tx_succeeded + FROM + flattened_logs l + LEFT JOIN {{ ref('testnet__fact_evm_blocks') }} + b + ON l.block_number = b.block_number + +{% if is_incremental() %} +AND b.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +LEFT JOIN {{ ref('testnet__fact_evm_transactions') }} +txs +ON l.tx_hash = txs.tx_hash +AND l.block_number = txs.block_number + +{% if is_incremental() %} +AND txs.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + b.block_timestamp AS block_timestamp_heal, + t.tx_hash, + t.tx_position, + t.event_index, + t.contract_address, + t.topics, + t.topic_0, + t.topic_1, + t.topic_2, + t.topic_3, + t.data, + t.event_removed, + txs.from_address AS origin_from_address_heal, + txs.to_address AS origin_to_address_heal, + txs.origin_function_signature AS origin_function_signature_heal, + t.tx_succeeded + FROM + {{ this }} + t + LEFT JOIN {{ ref('testnet__fact_evm_transactions') }} + txs + ON t.tx_hash = txs.tx_hash + AND t.block_number = txs.block_number + LEFT JOIN {{ ref('testnet__fact_evm_blocks') }} + b + ON t.block_number = b.block_number + WHERE + t.block_timestamp IS NULL + OR t.origin_function_signature IS NULL +) +{% endif %}, +all_logs AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + tx_position, + event_index, + contract_address, + topics, + topic_0, + topic_1, + topic_2, + topic_3, + DATA, + event_removed, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_succeeded + FROM + new_logs + +{% if is_incremental() %} +UNION ALL +SELECT + block_number, + block_timestamp_heal AS block_timestamp, + tx_hash, + tx_position, + event_index, + contract_address, + topics, + topic_0, + topic_1, + topic_2, + topic_3, + DATA, + event_removed, + origin_from_address_heal AS origin_from_address, + origin_to_address_heal AS origin_to_address, + origin_function_signature_heal AS origin_function_signature, + tx_succeeded +FROM + missing_data +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + tx_position, + event_index, + contract_address, + topics, + topic_0, + topic_1, + topic_2, + topic_3, + DATA, + event_removed, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_succeeded, + {{ dbt_utils.generate_surrogate_key(['tx_hash','event_index']) }} AS fact_event_logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + all_logs qualify ROW_NUMBER() over ( + PARTITION BY fact_event_logs_id + ORDER BY + block_number DESC, + block_timestamp DESC nulls last, + origin_function_signature DESC nulls last + ) = 1 \ No newline at end of file diff --git a/models/evm/gold/testnet/core/testnet__fact_evm_event_logs.yml b/models/evm/gold/testnet/core/testnet__fact_evm_event_logs.yml new file mode 100644 index 0000000..f54e192 --- /dev/null +++ b/models/evm/gold/testnet/core/testnet__fact_evm_event_logs.yml @@ -0,0 +1,46 @@ +version: 2 +models: + - name: testnet__fact_evm_event_logs + description: '{{ doc("evm_logs_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("evm_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("evm_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("evm_tx_hash") }}' + - name: TX_POSITION + description: '{{ doc("evm_tx_position") }}' + - name: EVENT_INDEX + description: '{{ doc("evm_event_index") }}' + - name: CONTRACT_ADDRESS + description: '{{ doc("evm_logs_contract_address") }}' + - name: TOPICS + description: '{{ doc("evm_topics") }}' + - name: TOPIC_0 + description: '{{ doc("evm_topic_0") }}' + - name: TOPIC_1 + description: '{{ doc("evm_topic_1") }}' + - name: TOPIC_2 + description: '{{ doc("evm_topic_2") }}' + - name: TOPIC_3 + description: '{{ doc("evm_topic_3") }}' + - name: DATA + description: '{{ doc("evm_logs_data") }}' + - name: EVENT_REMOVED + description: '{{ doc("evm_event_removed") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("evm_from_address") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("evm_to_address") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("evm_origin_sig") }}' + - name: TX_SUCCEEDED + description: '{{ doc("evm_tx_succeeded") }}' + - name: FACT_EVENT_LOGS_ID + description: '{{ doc("evm_pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("evm_inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/evm/gold/testnet/core/testnet__fact_evm_traces.sql b/models/evm/gold/testnet/core/testnet__fact_evm_traces.sql new file mode 100644 index 0000000..5a28215 --- /dev/null +++ b/models/evm/gold/testnet/core/testnet__fact_evm_traces.sql @@ -0,0 +1,379 @@ +{{ config( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['block_timestamp::DATE'], + tags = ['evm_testnet'] +) }} + +WITH silver_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + parent_trace_address, + trace_address_array, + trace_json, + traces_id, + 'regular' AS source + FROM {{ ref('silver_evm__testnet_traces') }} + WHERE 1 = 1 + {% if is_incremental() %} + AND modified_timestamp > ( + SELECT COALESCE(MAX(modified_timestamp), '1970-01-01'::TIMESTAMP) AS modified_timestamp + FROM {{ this }} + ) + {% endif %} +), + +sub_traces AS ( + SELECT + block_number, + tx_position, + parent_trace_address, + COUNT(*) AS sub_traces + FROM silver_traces + GROUP BY + block_number, + tx_position, + parent_trace_address +), + +trace_index_array AS ( + SELECT + block_number, + tx_position, + trace_address, + ARRAY_AGG(flat_value) AS number_array + FROM ( + SELECT + block_number, + tx_position, + trace_address, + IFF(VALUE::STRING = 'ORIGIN', -1, VALUE::INT) AS flat_value + FROM silver_traces, + LATERAL FLATTEN(input => trace_address_array) + ) + GROUP BY + block_number, + tx_position, + trace_address +), + +trace_index_sub_traces AS ( + SELECT + b.block_number, + b.tx_position, + b.trace_address, + IFNULL(sub_traces, 0) AS sub_traces, + number_array, + ROW_NUMBER() OVER ( + PARTITION BY b.block_number, b.tx_position + ORDER BY number_array ASC + ) - 1 AS trace_index, + b.trace_json, + b.traces_id, + b.source + FROM silver_traces b + LEFT JOIN sub_traces s + ON b.block_number = s.block_number + AND b.tx_position = s.tx_position + AND b.trace_address = s.parent_trace_address + JOIN trace_index_array n + ON b.block_number = n.block_number + AND b.tx_position = n.tx_position + AND b.trace_address = n.trace_address +), + +errored_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + trace_json + FROM trace_index_sub_traces + WHERE trace_json:error::STRING IS NOT NULL +), + +error_logic AS ( + SELECT + b0.block_number, + b0.tx_position, + b0.trace_address, + b0.trace_json:error::STRING AS error, + b1.trace_json:error::STRING AS any_error, + b2.trace_json:error::STRING AS origin_error + FROM trace_index_sub_traces b0 + LEFT JOIN errored_traces b1 + ON b0.block_number = b1.block_number + AND b0.tx_position = b1.tx_position + AND b0.trace_address RLIKE CONCAT('^', b1.trace_address, '(_[0-9]+)*$') + LEFT JOIN errored_traces b2 + ON b0.block_number = b2.block_number + AND b0.tx_position = b2.tx_position + AND b2.trace_address = 'ORIGIN' +), + +aggregated_errors AS ( + SELECT + block_number, + tx_position, + trace_address, + error, + IFF( + MAX(any_error) IS NULL AND error IS NULL AND origin_error IS NULL, + TRUE, + FALSE + ) AS trace_succeeded + FROM error_logic + GROUP BY + block_number, + tx_position, + trace_address, + error, + origin_error +), + +json_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + sub_traces, + number_array, + trace_index, + trace_succeeded, + trace_json:error::STRING AS error_reason, + trace_json:revertReason::STRING AS revert_reason, + lower(trace_json:from::STRING) AS from_address, + lower(trace_json:to::STRING) AS to_address, + IFNULL(trace_json:value::STRING, '0x0') AS value_hex, + IFNULL(utils.udf_hex_to_int(trace_json:value::STRING), '0') AS value_precise_raw, + utils.udf_decimal_adjust(value_precise_raw, 18) AS value_precise, + value_precise::FLOAT AS value, + utils.udf_hex_to_int(trace_json:gas::STRING)::INT AS gas, + utils.udf_hex_to_int(trace_json:gasUsed::STRING)::INT AS gas_used, + trace_json:input::STRING AS input, + trace_json:output::STRING AS output, + trace_json:type::STRING AS type, + traces_id + FROM trace_index_sub_traces + JOIN aggregated_errors USING ( + block_number, + tx_position, + trace_address + ) +), + +incremental_traces AS ( + SELECT + f.block_number, + t.tx_hash, + t.block_timestamp, + t.origin_function_signature, + t.from_address AS origin_from_address, + t.to_address AS origin_to_address, + f.tx_position, + f.trace_index, + f.from_address AS from_address, + f.to_address AS to_address, + f.value_hex, + f.value_precise_raw, + f.value_precise, + f.value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.sub_traces, + f.error_reason, + f.revert_reason, + f.traces_id, + f.trace_succeeded, + f.trace_address, + t.tx_succeeded + FROM json_traces f + LEFT OUTER JOIN {{ ref('testnet__fact_evm_transactions') }} t + ON f.tx_position = t.tx_position + AND f.block_number = t.block_number + {% if is_incremental() %} + AND t.modified_timestamp >= ( + SELECT DATEADD('hour', -24, MAX(modified_timestamp)) + FROM {{ this }} + ) + {% endif %} +) +{% if is_incremental() %}, +overflow_blocks AS ( + SELECT DISTINCT block_number + FROM silver_traces + WHERE source = 'overflow' +), + +heal_missing_data AS ( + SELECT + t.block_number, + txs.tx_hash, + txs.block_timestamp AS block_timestamp_heal, + txs.origin_function_signature AS origin_function_signature_heal, + txs.from_address AS origin_from_address_heal, + txs.to_address AS origin_to_address_heal, + t.tx_position, + t.trace_index, + t.from_address, + t.to_address, + t.value_hex, + t.value_precise_raw, + t.value_precise, + t.value, + t.gas, + t.gas_used, + t.input, + t.output, + t.type, + t.sub_traces, + t.error_reason, + t.revert_reason, + t.fact_traces_id AS traces_id, + t.trace_succeeded, + t.trace_address, + txs.tx_succeeded AS tx_succeeded_heal + FROM {{ this }} t + JOIN {{ ref('testnet__fact_evm_transactions') }} txs + ON t.tx_position = txs.tx_position + AND t.block_number = txs.block_number + WHERE t.tx_position IS NULL + OR t.block_timestamp IS NULL + OR t.tx_succeeded IS NULL + +) +{% endif %}, +all_traces AS ( + SELECT + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + value, + gas, + gas_used, + input, + output, + type, + sub_traces, + error_reason, + revert_reason, + trace_succeeded, + trace_address, + tx_succeeded + FROM incremental_traces + +{% if is_incremental() %} + + UNION ALL + + SELECT + block_number, + tx_hash, + block_timestamp_heal AS block_timestamp, + origin_function_signature_heal AS origin_function_signature, + origin_from_address_heal AS origin_from_address, + origin_to_address_heal AS origin_to_address, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + value, + gas, + gas_used, + input, + output, + type, + sub_traces, + error_reason, + revert_reason, + trace_succeeded, + trace_address, + tx_succeeded_heal AS tx_succeeded + FROM heal_missing_data + + UNION ALL + + SELECT + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + value, + gas, + gas_used, + input, + output, + type, + sub_traces, + error_reason, + revert_reason, + trace_succeeded, + trace_address, + tx_succeeded + FROM {{ this }} + JOIN overflow_blocks USING (block_number) +{% endif %} +) + +SELECT + block_number, + block_timestamp, + tx_hash, + tx_position, + trace_index, + from_address, + to_address, + input, + output, + type, + trace_address, + sub_traces, + value, + value_precise_raw, + value_precise, + value_hex, + gas, + gas_used, + origin_from_address, + origin_to_address, + origin_function_signature, + trace_succeeded, + error_reason, + revert_reason, + tx_succeeded, + {{ dbt_utils.generate_surrogate_key(['tx_hash', 'trace_index']) }} AS fact_traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM all_traces +QUALIFY (ROW_NUMBER() OVER ( + PARTITION BY block_number, tx_position, trace_index + ORDER BY modified_timestamp DESC, block_timestamp DESC NULLS LAST +)) = 1 \ No newline at end of file diff --git a/models/evm/gold/testnet/core/testnet__fact_evm_traces.yml b/models/evm/gold/testnet/core/testnet__fact_evm_traces.yml new file mode 100644 index 0000000..6a6cd02 --- /dev/null +++ b/models/evm/gold/testnet/core/testnet__fact_evm_traces.yml @@ -0,0 +1,62 @@ +version: 2 +models: + - name: testnet__fact_evm_traces + description: '{{ doc("evm_traces_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("evm_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("evm_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("evm_tx_hash") }}' + - name: TX_POSITION + description: '{{ doc("evm_tx_position") }}' + - name: TRACE_INDEX + description: '{{ doc("evm_trace_index") }}' + - name: FROM_ADDRESS + description: '{{ doc("evm_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("evm_to_address") }}' + - name: INPUT + description: '{{ doc("evm_traces_input") }}' + - name: OUTPUT + description: '{{ doc("evm_traces_output") }}' + - name: TYPE + description: '{{ doc("evm_traces_type") }}' + - name: TRACE_ADDRESS + description: '{{ doc("evm_trace_address") }}' + - name: SUB_TRACES + description: '{{ doc("evm_sub_traces") }}' + - name: VALUE + description: '{{ doc("evm_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("evm_precise_amount_unadjusted") }}' + - name: VALUE_PRECISE + description: '{{ doc("evm_precise_amount_adjusted") }}' + - name: VALUE_HEX + description: '{{ doc("evm_value_hex") }}' + - name: GAS + description: '{{ doc("evm_traces_gas") }}' + - name: GAS_USED + description: '{{ doc("evm_traces_gas_used") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("evm_traces_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("evm_traces_to") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("evm_origin_sig") }}' + - name: TRACE_SUCCEEDED + description: '{{ doc("evm_trace_succeeded") }}' + - name: ERROR_REASON + description: '{{ doc("evm_trace_error_reason") }}' + - name: REVERT_REASON + description: '{{ doc("evm_revert_reason") }}' + - name: TX_SUCCEEDED + description: '{{ doc("evm_tx_succeeded") }}' + - name: FACT_TRACES_ID + description: '{{ doc("evm_pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("evm_inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/evm/gold/testnet/core/testnet__fact_evm_transactions.sql b/models/evm/gold/testnet/core/testnet__fact_evm_transactions.sql new file mode 100644 index 0000000..0fc7b0c --- /dev/null +++ b/models/evm/gold/testnet/core/testnet__fact_evm_transactions.sql @@ -0,0 +1,348 @@ +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['block_timestamp::DATE'], + tags = ['evm_testnet'] +) }} + +WITH base AS ( + + SELECT + block_number, + tx_position, + transaction_json + FROM + {{ ref('silver_evm__testnet_transactions') }} + +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }}) + {% endif %} + ), + transactions_fields AS ( + SELECT + block_number, + tx_position, + transaction_json :blockHash :: STRING AS block_hash, + transaction_json :blockNumber :: STRING AS block_number_hex, + lower(transaction_json :from :: STRING) AS from_address, + utils.udf_hex_to_int( + transaction_json :gas :: STRING + ) :: bigint AS gas_limit, + utils.udf_hex_to_int( + transaction_json :gasPrice :: STRING + ) :: bigint AS gas_price, + transaction_json :hash :: STRING AS tx_hash, + transaction_json :input :: STRING AS input_data, + LEFT( + input_data, + 10 + ) AS origin_function_signature, + utils.udf_hex_to_int( + transaction_json :nonce :: STRING + ) :: bigint AS nonce, + transaction_json :r :: STRING AS r, + transaction_json :s :: STRING AS s, + lower(transaction_json :to :: STRING) AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + utils.udf_hex_to_int( + transaction_json :transactionIndex :: STRING + ) :: bigint AS transaction_index, + utils.udf_hex_to_int( + transaction_json :type :: STRING + ) :: bigint AS tx_type, + utils.udf_hex_to_int( + transaction_json :v :: STRING + ) :: bigint AS v, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + transaction_json :maxFeePerGas :: STRING + ) + ) / pow( + 10, + 9 + ) AS max_fee_per_gas, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + transaction_json :maxPriorityFeePerGas :: STRING + ) + ) / pow( + 10, + 9 + ) AS max_priority_fee_per_gas, + utils.udf_hex_to_int( + transaction_json :value :: STRING + ) AS value_precise_raw, + utils.udf_decimal_adjust( + value_precise_raw, + 18 + ) AS value_precise, + value_precise :: FLOAT AS VALUE, + utils.udf_hex_to_int(transaction_json :yParity :: STRING):: bigint AS y_parity, + transaction_json :accessList AS access_list, + FROM + base + ), + new_transactions AS ( + SELECT + txs.block_number, + txs.block_hash, + b.block_timestamp, + txs.tx_hash, + txs.from_address, + txs.to_address, + txs.origin_function_signature, + txs.value, + txs.value_precise_raw, + txs.value_precise, + txs.max_fee_per_gas, + txs.max_priority_fee_per_gas, + txs.y_parity, + txs.access_list, + utils.udf_decimal_adjust( + txs.gas_price * utils.udf_hex_to_int( + r.receipts_json :gasUsed :: STRING + ) :: bigint, + 18 + ) AS tx_fee_precise, + COALESCE( + tx_fee_precise :: FLOAT, + 0 + ) AS tx_fee, + CASE + WHEN r.receipts_json :status :: STRING = '0x1' THEN TRUE + WHEN r.receipts_json :status :: STRING = '1' THEN TRUE + WHEN r.receipts_json :status :: STRING = '0x0' THEN FALSE + WHEN r.receipts_json :status :: STRING = '0' THEN FALSE + ELSE NULL + END AS tx_succeeded, + txs.tx_type, + txs.nonce, + txs.tx_position, + txs.input_data, + txs.gas_price / pow( + 10, + 9 + ) AS gas_price, + utils.udf_hex_to_int( + r.receipts_json :gasUsed :: STRING + ) :: bigint AS gas_used, + txs.gas_limit, + utils.udf_hex_to_int( + r.receipts_json :cumulativeGasUsed :: STRING + ) :: bigint AS cumulative_gas_used, + utils.udf_hex_to_int( + r.receipts_json :effectiveGasPrice :: STRING + ) :: bigint AS effective_gas_price, + txs.r, + txs.s, + txs.v + FROM + transactions_fields txs + LEFT JOIN {{ ref('testnet__fact_evm_blocks') }} + b + ON txs.block_number = b.block_number + +{% if is_incremental() %} +AND b.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +LEFT JOIN {{ ref('silver_evm__testnet_receipts') }} +r +ON txs.block_number = r.block_number +AND txs.tx_hash = r.receipts_json :transactionHash :: STRING + +{% if is_incremental() %} +AND r.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + b.block_timestamp AS block_timestamp_heal, + t.tx_hash, + t.from_address, + t.to_address, + t.origin_function_signature, + t.value, + t.value_precise_raw, + t.value_precise, + t.max_fee_per_gas, + t.max_priority_fee_per_gas, + t.y_parity, + t.access_list, + utils.udf_decimal_adjust( + t.gas_price * utils.udf_hex_to_int( + r.receipts_json :gasUsed :: STRING + ) :: bigint, + 9 + ) AS tx_fee_precise_heal, + COALESCE( + tx_fee_precise_heal :: FLOAT, + 0 + ) AS tx_fee_heal, + CASE + WHEN r.receipts_json :status :: STRING = '0x1' THEN TRUE + WHEN r.receipts_json :status :: STRING = '0x0' THEN FALSE + ELSE NULL + END AS tx_succeeded_heal, + t.tx_type, + t.nonce, + t.tx_position, + t.input_data, + t.gas_price, + utils.udf_hex_to_int( + r.receipts_json :gasUsed :: STRING + ) :: bigint AS gas_used_heal, + t.gas_limit, + utils.udf_hex_to_int( + r.receipts_json :cumulativeGasUsed :: STRING + ) :: bigint AS cumulative_gas_used_heal, + utils.udf_hex_to_int( + r.receipts_json :effectiveGasPrice :: STRING + ) :: bigint AS effective_gas_price_heal, + t.r, + t.s, + t.v + FROM + {{ this }} + t + LEFT JOIN {{ ref('testnet__fact_evm_blocks') }} + b + ON t.block_number = b.block_number + LEFT JOIN {{ ref('silver_evm__testnet_receipts') }} + r + ON t.block_number = r.block_number + AND t.tx_hash = r.receipts_json :transactionHash :: STRING + WHERE + t.block_timestamp IS NULL + OR t.tx_succeeded IS NULL +) +{% endif %}, +all_transactions AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + from_address, + to_address, + origin_function_signature, + VALUE, + value_precise_raw, + value_precise, + max_fee_per_gas, + max_priority_fee_per_gas, + y_parity, + access_list, + tx_fee, + tx_fee_precise, + tx_succeeded, + tx_type, + nonce, + tx_position, + input_data, + gas_price, + gas_used, + gas_limit, + cumulative_gas_used, + effective_gas_price, + r, + s, + v + FROM + new_transactions + +{% if is_incremental() %} +UNION ALL +SELECT + block_number, + block_timestamp_heal AS block_timestamp, + tx_hash, + from_address, + to_address, + origin_function_signature, + VALUE, + value_precise_raw, + value_precise, + max_fee_per_gas, + max_priority_fee_per_gas, + y_parity, + access_list, + tx_fee_heal AS tx_fee, + tx_fee_precise_heal AS tx_fee_precise, + tx_succeeded_heal AS tx_succeeded, + tx_type, + nonce, + tx_position, + input_data, + gas_price, + gas_used_heal AS gas_used, + gas_limit, + cumulative_gas_used_heal AS cumulative_gas_used, + effective_gas_price_heal AS effective_gas_price, + r, + s, + v +FROM + missing_data +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + from_address, + to_address, + origin_function_signature, + VALUE, + value_precise_raw, + value_precise, + tx_fee, + tx_fee_precise, + tx_succeeded, + tx_type, + nonce, + tx_position, + input_data, + gas_price, + gas_used, + gas_limit, + cumulative_gas_used, + effective_gas_price, + max_fee_per_gas, + max_priority_fee_per_gas, + y_parity, + access_list, + r, + s, + v, + {{ dbt_utils.generate_surrogate_key(['tx_hash']) }} AS fact_transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + all_transactions qualify ROW_NUMBER() over ( + PARTITION BY fact_transactions_id + ORDER BY + block_number DESC, + block_timestamp DESC nulls last, + tx_succeeded DESC nulls last + ) = 1 \ No newline at end of file diff --git a/models/evm/gold/testnet/core/testnet__fact_evm_transactions.yml b/models/evm/gold/testnet/core/testnet__fact_evm_transactions.yml new file mode 100644 index 0000000..763cea8 --- /dev/null +++ b/models/evm/gold/testnet/core/testnet__fact_evm_transactions.yml @@ -0,0 +1,68 @@ +version: 2 +models: + - name: testnet__fact_evm_transactions + description: '{{ doc("evm_tx_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("evm_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("evm_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("evm_tx_hash") }}' + - name: FROM_ADDRESS + description: '{{ doc("evm_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("evm_to_address") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("evm_tx_origin_sig") }}' + - name: VALUE + description: '{{ doc("evm_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("evm_precise_amount_unadjusted") }}' + - name: VALUE_PRECISE + description: '{{ doc("evm_precise_amount_adjusted") }}' + - name: TX_FEE + description: '{{ doc("evm_tx_fee") }}' + - name: TX_FEE_PRECISE + description: '{{ doc("evm_tx_fee_precise") }}' + - name: TX_SUCCEEDED + description: '{{ doc("evm_tx_succeeded") }}' + - name: TX_TYPE + description: '{{ doc("evm_tx_type") }}' + - name: NONCE + description: '{{ doc("evm_tx_nonce") }}' + - name: TX_POSITION + description: '{{ doc("evm_tx_position") }}' + - name: INPUT_DATA + description: '{{ doc("evm_tx_input_data") }}' + - name: GAS_PRICE + description: '{{ doc("evm_tx_gas_price") }}' + - name: GAS_USED + description: '{{ doc("evm_tx_gas_used") }}' + - name: GAS_LIMIT + description: '{{ doc("evm_tx_gas_limit") }}' + - name: CUMULATIVE_GAS_USED + description: '{{ doc("evm_cumulative_gas_used") }}' + - name: EFFECTIVE_GAS_PRICE + description: '{{ doc("evm_effective_gas_price") }}' + - name: R + description: '{{ doc("evm_r") }}' + - name: S + description: '{{ doc("evm_s") }}' + - name: V + description: '{{ doc("evm_v") }}' + - name: MAX_FEE_PER_GAS + description: '{{ doc("evm_max_fee_per_gas") }}' + - name: MAX_PRIORITY_FEE_PER_GAS + description: '{{ doc("evm_max_priority_fee_per_gas") }}' + - name: Y_PARITY + description: '{{ doc("evm_y_parity") }}' + - name: ACCESS_LIST + description: '{{ doc("evm_access_list") }}' + - name: FACT_TRANSACTIONS_ID + description: '{{ doc("evm_pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("evm_inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_full.sql b/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_full.sql new file mode 100644 index 0000000..7fdfdac --- /dev/null +++ b/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('testnet__fact_evm_blocks') }} \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_full.yml b/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_full.yml new file mode 100644 index 0000000..9d45613 --- /dev/null +++ b/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_full.yml @@ -0,0 +1,145 @@ +version: 2 +models: + - name: test_gold_testnet_evm__fact_blocks_full + description: "This is a view used to test all of the gold testnet fact blocks model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: NETWORK + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: ^[a-zA-Z0-9_]+$ + - name: TX_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SIZE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MINER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: EXTRA_DATA + tests: + - not_null + - name: PARENT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_LIMIT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SHA3_UNCLES + tests: + - not_null + - name: UNCLE_BLOCKS + tests: + - not_null + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: RECEIPTS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: STATE_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TRANSACTIONS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: LOGS_BLOOM + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FACT_BLOCKS_ID + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_unique + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_recent.sql b/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_recent.sql new file mode 100644 index 0000000..2886348 --- /dev/null +++ b/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('testnet__fact_evm_blocks') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_testnet_block_lookback') }} + ) diff --git a/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_recent.yml b/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_recent.yml new file mode 100644 index 0000000..11b8bea --- /dev/null +++ b/models/evm/gold/tests/testnet_blocks/test_gold_testnet_evm__fact_blocks_recent.yml @@ -0,0 +1,147 @@ +version: 2 +models: + - name: test_gold_testnet_evm__fact_blocks_recent + description: "This is a view used to test the last three days of gold testnet fact blocks." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">10" + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: NETWORK + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: ^[a-zA-Z0-9_]+$ + - name: TX_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SIZE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MINER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: EXTRA_DATA + tests: + - not_null + - name: PARENT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_LIMIT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SHA3_UNCLES + tests: + - not_null + - name: UNCLE_BLOCKS + tests: + - not_null + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: RECEIPTS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: STATE_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TRANSACTIONS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: LOGS_BLOOM + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FACT_BLOCKS_ID + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_unique + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_full.sql b/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_full.sql new file mode 100644 index 0000000..b42ab0d --- /dev/null +++ b/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('testnet__fact_evm_event_logs') }} \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_full.yml b/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_full.yml new file mode 100644 index 0000000..33a4450 --- /dev/null +++ b/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_full.yml @@ -0,0 +1,101 @@ +version: 2 +models: + - name: test_gold_testnet_evm__fact_event_logs_full + description: "This is a view used to test all of the gold testnet fact event logs model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - EVENT_INDEX + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: EVENT_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + - events_match_txs: + transactions_model: ref('test_gold_evm__fact_transactions_full') + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - fsc_utils.tx_block_count: + config: + severity: error + error_if: "!=0" + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TOPICS + tests: + - not_null + - name: DATA + tests: + - not_null + - name: EVENT_REMOVED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_EVENT_LOGS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_recent.sql b/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_recent.sql new file mode 100644 index 0000000..20378e0 --- /dev/null +++ b/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('testnet__fact_evm_event_logs') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_testnet_block_lookback') }} + ) diff --git a/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_recent.yml b/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_recent.yml new file mode 100644 index 0000000..37445af --- /dev/null +++ b/models/evm/gold/tests/testnet_event_logs/test_gold_testnet_evm__fact_event_logs_recent.yml @@ -0,0 +1,100 @@ +version: 2 +models: + - name: test_gold_testnet_evm__fact_event_logs_recent + description: "This is a view used to test the last three days of gold testnet fact event logs." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - EVENT_INDEX + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: EVENT_INDEX + - events_match_txs: + transactions_model: ref('test_gold_testnet_evm__fact_transactions_recent') + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - fsc_utils.tx_block_count: + config: + severity: error + error_if: "!=0" + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TOPICS + tests: + - not_null + - name: DATA + tests: + - not_null + - name: EVENT_REMOVED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_EVENT_LOGS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_full.sql b/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_full.sql new file mode 100644 index 0000000..e12fd78 --- /dev/null +++ b/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('testnet__fact_evm_traces') }} \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_full.yml b/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_full.yml new file mode 100644 index 0000000..1c28824 --- /dev/null +++ b/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_full.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold_testnet_evm__fact_traces_full + description: "This is a view used to test all of the gold testnet fact traces model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 AND TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_recent.sql b/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_recent.sql new file mode 100644 index 0000000..979c20a --- /dev/null +++ b/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('testnet__fact_evm_traces') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_testnet_block_lookback') }} + ) diff --git a/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_recent.yml b/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_recent.yml new file mode 100644 index 0000000..d5eccab --- /dev/null +++ b/models/evm/gold/tests/testnet_traces/test_gold_testnet_evm__fact_traces_recent.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold_testnet_evm__fact_traces_recent + description: "This is a view used to test the last three days of gold testnet fact traces." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_full.sql b/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_full.sql new file mode 100644 index 0000000..49fdc5b --- /dev/null +++ b/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('testnet__fact_evm_transactions') }} \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_full.yml b/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_full.yml new file mode 100644 index 0000000..b6e8790 --- /dev/null +++ b/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_full.yml @@ -0,0 +1,125 @@ +version: 2 +models: + - name: test_gold_testnet_evm__fact_transactions_full + description: "This is a view used to test all of the gold testnet fact transactions model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: TX_POSITION + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + - txs_match_blocks: + blocks_model: ref('test_gold_testnet_evm__fact_blocks_full') + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: TX_FEE + tests: + - not_null + - name: TX_FEE_PRECISE + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: TX_TYPE + tests: + - not_null + - name: NONCE + tests: + - not_null + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: INPUT_DATA + tests: + - not_null + - name: GAS_PRICE + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: GAS_LIMIT + tests: + - not_null + - name: CUMULATIVE_GAS_USED + tests: + - not_null + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null + - name: R + tests: + - not_null + - name: S + tests: + - not_null + - name: V + tests: + - not_null + - name: FACT_TRANSACTIONS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_recent.sql b/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_recent.sql new file mode 100644 index 0000000..e33ee2b --- /dev/null +++ b/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('testnet__fact_evm_transactions') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_testnet_block_lookback') }} + ) diff --git a/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_recent.yml b/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_recent.yml new file mode 100644 index 0000000..3e264f8 --- /dev/null +++ b/models/evm/gold/tests/testnet_transactions/test_gold_testnet_evm__fact_transactions_recent.yml @@ -0,0 +1,124 @@ +version: 2 +models: + - name: test_gold_testnet_evm__fact_transactions_recent + description: "This is a view used to test the last three days of gold testnet fact transactions." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: TX_POSITION + - txs_match_blocks: + blocks_model: ref('test_gold_testnet_evm__fact_blocks_recent') + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: TX_FEE + tests: + - not_null + - name: TX_FEE_PRECISE + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: TX_TYPE + tests: + - not_null + - name: NONCE + tests: + - not_null + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: INPUT_DATA + tests: + - not_null + - name: GAS_PRICE + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: GAS_LIMIT + tests: + - not_null + - name: CUMULATIVE_GAS_USED + tests: + - not_null + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null + - name: R + tests: + - not_null + - name: S + tests: + - not_null + - name: V + tests: + - not_null + - name: FACT_TRANSACTIONS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/silver/testnet/core/silver_evm__testnet_blocks.sql b/models/evm/silver/testnet/core/silver_evm__testnet_blocks.sql new file mode 100644 index 0000000..f873b21 --- /dev/null +++ b/models/evm/silver/testnet/core/silver_evm__testnet_blocks.sql @@ -0,0 +1,42 @@ +-- depends_on: {{ ref('bronze_evm__testnet_blocks') }} + +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['modified_timestamp::DATE','partition_key'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['evm_testnet'] +) }} + +WITH bronze_blocks AS ( + SELECT + block_number, + partition_key, + DATA:result AS block_json, + _inserted_timestamp + FROM + {% if is_incremental() %} + {{ ref('bronze_evm__testnet_blocks') }} + WHERE _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1900-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM {{ this }} + ) AND DATA:result IS NOT NULL + {% else %} + {{ ref('bronze_evm__FR_testnet_blocks') }} + WHERE DATA:result IS NOT NULL + {% endif %} +) + +SELECT + block_number, + partition_key, + block_json, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['block_number']) }} AS blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM bronze_blocks +QUALIFY ROW_NUMBER() OVER (PARTITION BY blocks_id ORDER BY _inserted_timestamp DESC) = 1 \ No newline at end of file diff --git a/models/evm/silver/testnet/core/silver_evm__testnet_receipts.sql b/models/evm/silver/testnet/core/silver_evm__testnet_receipts.sql new file mode 100644 index 0000000..0fdcfd1 --- /dev/null +++ b/models/evm/silver/testnet/core/silver_evm__testnet_receipts.sql @@ -0,0 +1,61 @@ +-- depends_on: {{ ref('bronze_evm__testnet_receipts') }} +-- depends_on: {{ ref('bronze_evm__FR_testnet_receipts') }} +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['modified_timestamp::DATE','partition_key'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['evm_testnet'] +) }} + +WITH bronze AS ( + + SELECT + block_number, + DATA, + partition_key, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__testnet_receipts') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) AND data:result[0] is not null +{% else %} + {{ ref('bronze_evm__FR_testnet_receipts') }} + WHERE data:result[0] is not null +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 +), +flat_receipts AS ( + SELECT + block_number, + partition_key, + index :: INT AS array_index, + value AS receipts_json, + _inserted_timestamp + FROM bronze, + LATERAL FLATTEN(input => data:result) AS receipt +) +SELECT + block_number, + partition_key, + array_index, + receipts_json, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['block_number','array_index']) }} AS receipts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM flat_receipts + +QUALIFY(ROW_NUMBER() OVER (PARTITION BY block_number, array_index ORDER BY _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/evm/silver/testnet/core/silver_evm__testnet_traces.sql b/models/evm/silver/testnet/core/silver_evm__testnet_traces.sql new file mode 100644 index 0000000..9dd5a24 --- /dev/null +++ b/models/evm/silver/testnet/core/silver_evm__testnet_traces.sql @@ -0,0 +1,141 @@ +-- depends_on: {{ ref('bronze_evm__testnet_traces') }} +-- depends_on: {{ ref('bronze_evm__FR_testnet_traces') }} + +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['modified_timestamp::DATE','partition_key'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['evm_testnet'] +) }} + + +WITH bronze_traces AS ( + + SELECT + value:"BLOCK_NUMBER"::INT AS block_number, + partition_key, + value:array_index::INT AS tx_position, + DATA :result AS full_traces, + _inserted_timestamp + FROM + + {% if is_incremental()%} + {{ ref('bronze_evm__testnet_traces') }} + WHERE + DATA :result IS NOT NULL + AND _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1900-01-01') _inserted_timestamp + FROM + {{ this }} + ) + {% else %} + {{ ref('bronze_evm__FR_testnet_traces') }} + WHERE DATA :result IS NOT NULL + {% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position +ORDER BY + _inserted_timestamp DESC)) = 1 + +), +flatten_traces AS ( + SELECT + block_number, + tx_position, + partition_key, + IFF( + path IN ( + 'result', + 'result.value', + 'result.type', + 'result.to', + 'result.input', + 'result.gasUsed', + 'result.gas', + 'result.from', + 'result.output', + 'result.error', + 'result.revertReason', + 'result.time', + 'gasUsed', + 'gas', + 'type', + 'to', + 'from', + 'value', + 'input', + 'error', + 'output', + 'time', + 'revertReason' + ), + 'ORIGIN', + REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') + ) AS trace_address, + _inserted_timestamp, + OBJECT_AGG( + key, + VALUE + ) AS trace_json, + CASE + WHEN trace_address = 'ORIGIN' THEN NULL + WHEN POSITION( + '_' IN trace_address + ) = 0 THEN 'ORIGIN' + ELSE REGEXP_REPLACE( + trace_address, + '_[0-9]+$', + '', + 1, + 1 + ) + END AS parent_trace_address, + SPLIT( + trace_address, + '_' + ) AS trace_address_array + FROM + bronze_traces txs, + TABLE( + FLATTEN( + input => PARSE_JSON( + txs.full_traces + ), + recursive => TRUE + ) + ) f + WHERE + f.index IS NULL + AND f.key != 'calls' + AND f.path != 'result' + GROUP BY + block_number, + tx_position, + partition_key, + trace_address, + _inserted_timestamp +) +SELECT + block_number, + tx_position, + trace_address, + parent_trace_address, + trace_address_array, + trace_json, + partition_key, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + + ['tx_position'] + + ['trace_address'] + ) }} AS traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + flatten_traces qualify(ROW_NUMBER() over(PARTITION BY traces_id +ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/evm/silver/testnet/core/silver_evm__testnet_transactions.sql b/models/evm/silver/testnet/core/silver_evm__testnet_transactions.sql new file mode 100644 index 0000000..4c68032 --- /dev/null +++ b/models/evm/silver/testnet/core/silver_evm__testnet_transactions.sql @@ -0,0 +1,53 @@ +-- depends_on: {{ ref('bronze_evm__testnet_blocks') }} + +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['modified_timestamp::DATE','partition_key'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['evm_testnet'] +) }} + +WITH flat_txs as ( + SELECT + block_number, + partition_key, + _inserted_timestamp, + data + FROM + {% if is_incremental() %} + {{ ref('bronze_evm__testnet_blocks') }} + WHERE _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1900-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM {{ this }} + ) AND data:result:transactions[0] is not null + {% else %} + {{ ref('bronze_evm__FR_testnet_blocks') }} + WHERE data:result:transactions[0] is not null + {% endif %} +), +bronze_transactions AS ( + SELECT + block_number, + partition_key, + index :: INT AS tx_position, + value AS transaction_json, + _inserted_timestamp + FROM flat_txs, + LATERAL FLATTEN(input => data:result:transactions) AS tx +) + +SELECT + block_number, + partition_key, + tx_position, + transaction_json, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['block_number','tx_position']) }} AS transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM bronze_transactions +QUALIFY ROW_NUMBER() OVER (PARTITION BY transactions_id ORDER BY _inserted_timestamp DESC) = 1 \ No newline at end of file diff --git a/models/evm/streamline/testnet/_evm_testnet_block_lookback.sql b/models/evm/streamline/testnet/_evm_testnet_block_lookback.sql new file mode 100644 index 0000000..adfc0e8 --- /dev/null +++ b/models/evm/streamline/testnet/_evm_testnet_block_lookback.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = "ephemeral" +) }} + +SELECT + MIN(block_number) AS block_number +FROM + {{ ref("testnet__fact_evm_blocks") }} +WHERE + block_timestamp >= DATEADD('hour', -72, TRUNCATE(SYSDATE(), 'HOUR')) + AND block_timestamp < DATEADD('hour', -71, TRUNCATE(SYSDATE(), 'HOUR')) diff --git a/models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_blocks.sql b/models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_blocks.sql new file mode 100644 index 0000000..0039186 --- /dev/null +++ b/models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_blocks.sql @@ -0,0 +1,44 @@ +-- depends_on: {{ ref('bronze_evm__testnet_blocks') }} +-- depends_on: {{ ref('bronze_evm__FR_testnet_blocks') }} +{{ config ( + materialized = "incremental", + incremental_predicates = ["dynamic_range_predicate", "partition_key"], + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_complete_evm_testnet'] +) }} + +SELECT + block_number, + utils.udf_hex_to_int(DATA :result :number :: STRING) as blockNumber, + partition_key, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number::STRING'] + ) }} AS complete_evm_blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__testnet_blocks') }} +WHERE + _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01' :: timestamp_ntz + ) +{% else %} + {{ ref('bronze_evm__FR_testnet_blocks') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_receipts.sql b/models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_receipts.sql new file mode 100644 index 0000000..f44f0ee --- /dev/null +++ b/models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_receipts.sql @@ -0,0 +1,43 @@ +-- depends_on: {{ ref('bronze_evm__testnet_receipts') }} +-- depends_on: {{ ref('bronze_evm__FR_testnet_receipts') }} +{{ config ( + materialized = "incremental", + incremental_predicates = ["dynamic_range_predicate", "partition_key"], + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_complete_evm_testnet'] +) }} + +SELECT + block_number, + partition_key, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number::STRING'] + ) }} AS complete_evm_receipts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__testnet_receipts') }} +WHERE + _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01' :: timestamp_ntz + ) +{% else %} + {{ ref('bronze_evm__FR_testnet_receipts') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_traces.sql b/models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_traces.sql new file mode 100644 index 0000000..552fd46 --- /dev/null +++ b/models/evm/streamline/testnet/complete/streamline__complete_get_evm_testnet_traces.sql @@ -0,0 +1,45 @@ +-- depends_on: {{ ref('bronze_evm__testnet_traces') }} +-- depends_on: {{ ref('bronze_evm__FR_testnet_traces') }} +{{ config ( + materialized = "incremental", + incremental_predicates = ["dynamic_range_predicate", "partition_key"], + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_complete_evm_testnet'] +) }} + +SELECT + value:"BLOCK_NUMBER"::INT AS block_number, + partition_key, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number::STRING'] + ) }} AS complete_evm_traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__testnet_traces') }} +WHERE + _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01' :: timestamp_ntz + ) +{% else %} + {{ ref('bronze_evm__FR_testnet_traces') }} +WHERE + TRUE +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_blocks_realtime.sql b/models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_blocks_realtime.sql new file mode 100644 index 0000000..b6a70bf --- /dev/null +++ b/models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_blocks_realtime.sql @@ -0,0 +1,89 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_testnet_blocks", + "sql_limit" :"25000", + "producer_batch_size" :"5000", + "worker_batch_size" :"1000", + "sql_source" :"{{this.identifier}}" } + ), + tags = ['streamline_realtime_evm_testnet'] +) }} + +WITH last_3_days AS ( + + SELECT + GREATEST(ZEROIFNULL(block_number), 67860000) AS block_number + FROM + {{ ref("_evm_testnet_block_lookback") }} +), +tbl AS ( + SELECT + block_number + FROM + {{ ref('streamline__evm_testnet_blocks') }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref('streamline__complete_get_evm_testnet_blocks') }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) + AND blockNumber IS NOT NULL +) +SELECT + block_number, + DATE_PART(epoch_second, SYSDATE()) :: STRING AS request_timestamp, + '{{ invocation_id }}' AS _invocation_id, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'id', + block_number, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT( + utils.udf_int_to_hex(block_number), + TRUE -- Include transactions + ) + ), + 'Vault/prod/flow/quicknode/testnet' + ) AS request +FROM + tbl +ORDER BY + block_number DESC diff --git a/models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_receipts_realtime.sql b/models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_receipts_realtime.sql new file mode 100644 index 0000000..84003c0 --- /dev/null +++ b/models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_receipts_realtime.sql @@ -0,0 +1,99 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_testnet_receipts", + "sql_limit" :"25000", + "producer_batch_size" :"5000", + "worker_batch_size" :"1000", + "sql_source" :"{{this.identifier}}" } + ), + tags = ['streamline_realtime_evm_testnet'] +) }} + +WITH last_3_days AS ( + + SELECT + GREATEST(ZEROIFNULL(block_number), 67860000) AS block_number + FROM + {{ ref("_evm_testnet_block_lookback") }} +), +tbl AS ( + + SELECT + block_number + FROM + {{ ref('streamline__evm_testnet_blocks') }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref('streamline__complete_get_evm_testnet_receipts') }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) +), +ready_blocks AS ( + SELECT + block_number + FROM + tbl + UNION ALL + SELECT + block_number + FROM + {{ ref("_missing_testnet_receipts") }} +) +SELECT + block_number, + DATE_PART(epoch_second, SYSDATE())::STRING AS request_timestamp, + '{{ invocation_id }}' AS _invocation_id, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'id', + block_number, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockReceipts', + 'params', + ARRAY_CONSTRUCT( + utils.udf_int_to_hex(block_number) + ) + ), + 'Vault/prod/flow/quicknode/testnet' + ) AS request +FROM + ready_blocks +ORDER BY + block_number DESC diff --git a/models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_traces_realtime.sql b/models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_traces_realtime.sql new file mode 100644 index 0000000..98e4e2e --- /dev/null +++ b/models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_traces_realtime.sql @@ -0,0 +1,101 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_testnet_traces", + "sql_limit" :"25000", + "producer_batch_size" :"2000", + "worker_batch_size" :"1000", + "sql_source" :"{{this.identifier}}", + "exploded_key": tojson(["result"])} + ), + tags = ['streamline_realtime_evm_testnet'] +) }} + + +WITH last_3_days AS ( + + SELECT + GREATEST(ZEROIFNULL(block_number), 67860000) AS block_number + FROM + {{ ref("_evm_testnet_block_lookback") }} +), +tbl AS ( + + SELECT + block_number + FROM + {{ ref('streamline__evm_testnet_blocks') }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref('streamline__complete_get_evm_testnet_traces') }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) +), +ready_blocks AS ( + SELECT + block_number + FROM + tbl +) +SELECT + block_number, + DATE_PART(epoch_second, SYSDATE())::STRING AS request_timestamp, + '{{ invocation_id }}' AS _invocation_id, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'id', + block_number, + 'jsonrpc', + '2.0', + 'method', + 'debug_traceBlockByNumber', + 'params', + ARRAY_CONSTRUCT( + utils.udf_int_to_hex(block_number), + OBJECT_CONSTRUCT( + 'tracer', 'callTracer', + 'timeout', '180s' + ) + ) + ), + 'Vault/prod/flow/quicknode/testnet' + ) AS request +FROM + ready_blocks +ORDER BY + block_number DESC +limit 25000 \ No newline at end of file diff --git a/models/evm/streamline/testnet/retry/_missing_testnet_receipts.sql b/models/evm/streamline/testnet/retry/_missing_testnet_receipts.sql new file mode 100644 index 0000000..3907092 --- /dev/null +++ b/models/evm/streamline/testnet/retry/_missing_testnet_receipts.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "ephemeral" +) }} + +SELECT + DISTINCT block_number AS block_number +FROM + {{ ref("testnet__fact_evm_transactions") }} +WHERE + tx_succeeded IS NULL + AND block_number > ( + SELECT + block_number + FROM + {{ ref("_evm_testnet_block_lookback") }} + ) \ No newline at end of file diff --git a/models/evm/streamline/testnet/streamline__evm_testnet_blocks.sql b/models/evm/streamline/testnet/streamline__evm_testnet_blocks.sql new file mode 100644 index 0000000..36c6ab7 --- /dev/null +++ b/models/evm/streamline/testnet/streamline__evm_testnet_blocks.sql @@ -0,0 +1,14 @@ +{{ config( + materialized = "view", + tags = ['streamline_realtime_evm_testnet'] +) }} + +SELECT + _id AS block_number +FROM + {{ source( + 'silver_crosschain', + 'number_sequence' + ) }} +WHERE + _id <= (SELECT block_number FROM {{ ref('streamline__evm_testnet_chainhead') }}) \ No newline at end of file diff --git a/models/evm/streamline/testnet/streamline__evm_testnet_chainhead.sql b/models/evm/streamline/testnet/streamline__evm_testnet_chainhead.sql new file mode 100644 index 0000000..058c71b --- /dev/null +++ b/models/evm/streamline/testnet/streamline__evm_testnet_chainhead.sql @@ -0,0 +1,28 @@ +{{ config ( + materialized = 'table', + tags = ['streamline_realtime_evm_testnet'] +) }} + +SELECT + live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', 'LiveQuery' + ), + OBJECT_CONSTRUCT( + 'id', + 0, + 'jsonrpc', + '2.0', + 'method', + 'eth_blockNumber', + 'params', + [] + ), + 'Vault/prod/flow/quicknode/testnet' + ) AS resp, + utils.udf_hex_to_int( + resp :data :result :: STRING + ) AS block_number \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 5f9e744..49cb564 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -126,6 +126,9 @@ sources: - name: evm_traces_v2 - name: evm_decoded_logs - name: topshot_metadata + - name: evm_testnet_blocks + - name: evm_testnet_receipts + - name: evm_testnet_traces - name: crosschain_silver database: crosschain From 60b600b4fa90b7916b1e191962ca9f0bc29de1f5 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 12 Sep 2025 14:21:21 -0600 Subject: [PATCH 17/17] Rename workflow to dbt_run_evm_testnet (#481) --- .github/workflows/dbt_run_evm_testnet.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dbt_run_evm_testnet.yml b/.github/workflows/dbt_run_evm_testnet.yml index 95f6ede..16e882c 100644 --- a/.github/workflows/dbt_run_evm_testnet.yml +++ b/.github/workflows/dbt_run_evm_testnet.yml @@ -1,5 +1,5 @@ -name: dbt_run_evm -run-name: dbt_run_evm +name: dbt_run_evm_testnet +run-name: dbt_run_evm_testnet on: workflow_dispatch: @@ -50,4 +50,4 @@ jobs: if: failure() uses: ./.github/workflows/slack_notify.yml secrets: - SLACK_WEBHOOK_URL: ${{ secrets.EVM_SLACK_WEBHOOK_URL }} \ No newline at end of file + SLACK_WEBHOOK_URL: ${{ secrets.EVM_SLACK_WEBHOOK_URL }}