DAT2-17/avax-external-tables (#402)

* remove streamline * sources * exclude
2026-02-06 11:06:55 +00:00 · 2025-11-18 13:23:55 -07:00 · 2025-11-18 13:23:55 -07:00 · 7c5b6e21e5
commit 7c5b6e21e5
parent 03eed647e6
30 changed files with 4 additions and 451 deletions
--- a/.github/workflows/dbt_run_scheduled_main.yml
+++ b/.github/workflows/dbt_run_scheduled_main.yml
@ -15,5 +15,5 @@ jobs:
    with:
      command_name: Run Main Models
      command: |
-        dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:silver,tag:balances" "fsc_evm,tag:gold,tag:balances" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" "fsc_evm,tag:streamline,tag:token_reads,tag:realtime" "fsc_evm,tag:streamline,tag:token_reads,tag:complete" "fsc_evm,tag:streamline,tag:balances,tag:complete" "fsc_evm,tag:streamline,tag:balances,tag:records" "fsc_evm,tag:streamline,tag:balances,tag:realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}'
+        dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:silver,tag:balances" "fsc_evm,tag:gold,tag:balances" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" "fsc_evm,tag:streamline,tag:token_reads,tag:realtime" "fsc_evm,tag:streamline,tag:token_reads,tag:complete" "fsc_evm,tag:streamline,tag:balances,tag:complete" "fsc_evm,tag:streamline,tag:balances,tag:records" "fsc_evm,tag:streamline,tag:balances,tag:realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' --exclude "fsc_evm,tag:receipts_by_hash"
    secrets: inherit
--- a/dbt_project.yml
+++ b/dbt_project.yml
@ -73,8 +73,6 @@ models:
        +enabled: true
      core:
        +enabled: true # enable subpackages, as needed
-        bronze:
-          +enabled: false
      github_actions:
        +enabled: true
      labels:
@ -92,13 +90,7 @@ models:
      abis:
        +enabled: true
      decoded_logs:
-        +enabled: false
-        gold:
-          +enabled: true
-        silver:
-          +enabled: true
-        streamline:
-          +enabled: true
+        +enabled: true
    curated_package:
      +enabled: false
      defi:
--- a/models/sources.yml
+++ b/models/sources.yml
@ -9,15 +9,9 @@ sources:
      - name: receipts
      - name: blocks
      - name: transactions
-      - name: debug_traceblockbynumber
+      - name: traces
      - name: decoded_logs
      - name: confirm_blocks
-      - name: blocks_v2
-      - name: transactions_v2
-      - name: receipts_v2
-      - name: traces_v2
-      - name: confirm_blocks_v2
-      - name: decoded_logs_v2
      - name: contract_abis
      - name: dexalot_blocks
      - name: dexalot_transactions
--- a/models/streamline/bronze/core/bronze__blocks.sql
+++ b/models/streamline/bronze/core/bronze__blocks.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query(
-    source_name = 'blocks',
-    source_version = 'v2'
-) }}
--- a/models/streamline/bronze/core/bronze__blocks_fr.sql
+++ b/models/streamline/bronze/core/bronze__blocks_fr.sql
@ -1,29 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','phase_1']
-) }}
-
-SELECT
-    partition_key,
-    block_number,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
-FROM
-    {{ ref('bronze__blocks_fr_v2') }}
-UNION ALL
-SELECT
-    _partition_by_block_id AS partition_key,
-    block_number,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
-FROM
-   {{ ref('bronze__blocks_fr_v1') }}
--- a/models/streamline/bronze/core/bronze__blocks_fr_v1.sql
+++ b/models/streamline/bronze/core/bronze__blocks_fr_v1.sql
@ -1,16 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_fr(
-    source_name = 'blocks',
-    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
-    partition_join_key = "_partition_by_block_id",
-    block_number = false
-) }}
--- a/models/streamline/bronze/core/bronze__blocks_fr_v2.sql
+++ b/models/streamline/bronze/core/bronze__blocks_fr_v2.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_fr(
-    source_name = 'blocks',
-    source_version = 'v2'
-) }}
--- a/models/streamline/bronze/core/bronze__confirm_blocks.sql
+++ b/models/streamline/bronze/core/bronze__confirm_blocks.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','confirm_blocks','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query(
-    source_name = 'confirm_blocks',
-    source_version = 'v2'
-) }}
--- a/models/streamline/bronze/core/bronze__confirm_blocks_fr.sql
+++ b/models/streamline/bronze/core/bronze__confirm_blocks_fr.sql
@ -1,29 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','confirm_blocks','phase_1']
-) }}
-
-SELECT
-    partition_key,
-    block_number,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
-FROM
-    {{ ref('bronze__confirm_blocks_fr_v2') }}
-UNION ALL
-SELECT
-    _partition_by_block_id AS partition_key,
-    block_number,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
-FROM
-   {{ ref('bronze__confirm_blocks_fr_v1') }}
--- a/models/streamline/bronze/core/bronze__confirm_blocks_fr_v1.sql
+++ b/models/streamline/bronze/core/bronze__confirm_blocks_fr_v1.sql
@ -1,17 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','streamline_v1','confirm_blocks','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_fr(
-    source_name = 'confirm_blocks',
-    source_version = '',
-    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
-    partition_join_key = "_partition_by_block_id",
-    block_number = false
-) }}
--- a/models/streamline/bronze/core/bronze__confirm_blocks_fr_v2.sql
+++ b/models/streamline/bronze/core/bronze__confirm_blocks_fr_v2.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','confirm_blocks','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_fr(
-    source_name = 'confirm_blocks',
-    source_version = 'v2'
-) }}
--- a/models/streamline/bronze/core/bronze__receipts.sql
+++ b/models/streamline/bronze/core/bronze__receipts.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','receipts','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query(
-    source_name = 'receipts',
-    source_version = 'v2'
-) }}
--- a/models/streamline/bronze/core/bronze__receipts_by_hash.sql
+++ b/models/streamline/bronze/core/bronze__receipts_by_hash.sql
--- a/models/streamline/bronze/core/bronze__receipts_by_hash_fr.sql
+++ b/models/streamline/bronze/core/bronze__receipts_by_hash_fr.sql
--- a/models/streamline/bronze/core/bronze__receipts_fr.sql
+++ b/models/streamline/bronze/core/bronze__receipts_fr.sql
@ -1,34 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','receipts','phase_1']
-) }}
-
-SELECT
-    partition_key,
-    block_number,
-    array_index,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
-FROM
-    {{ ref('bronze__receipts_fr_v2') }}
-UNION ALL
-SELECT
-    _partition_by_block_id AS partition_key,
-    block_number,
-    COALESCE(
-        VALUE :array_index :: INT,
-        TRY_TO_NUMBER(utils.udf_hex_to_int(VALUE :data :"transactionIndex" :: STRING))
-    ) AS array_index,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
-FROM
-   {{ ref('bronze__receipts_fr_v1') }}
--- a/models/streamline/bronze/core/bronze__receipts_fr_v1.sql
+++ b/models/streamline/bronze/core/bronze__receipts_fr_v1.sql
@ -1,17 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','streamline_v1','receipts','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_fr(
-    source_name = 'receipts',
-    source_version = '',
-    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
-    partition_join_key = "_partition_by_block_id",
-    block_number = false
-) }}
--- a/models/streamline/bronze/core/bronze__receipts_fr_v2.sql
+++ b/models/streamline/bronze/core/bronze__receipts_fr_v2.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','receipts','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_fr(
-    source_name = 'receipts',
-    source_version = 'v2'
-) }}
--- a/models/streamline/bronze/core/bronze__traces.sql
+++ b/models/streamline/bronze/core/bronze__traces.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query(
-    source_name = 'traces',
-    source_version = 'v2'
-) }}
--- a/models/streamline/bronze/core/bronze__traces_fr.sql
+++ b/models/streamline/bronze/core/bronze__traces_fr.sql
@ -1,31 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','phase_1']
-) }}
-
-SELECT
-    partition_key,
-    block_number,
-    array_index,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
-FROM
-    {{ ref('bronze__traces_fr_v2') }}
-UNION ALL
-SELECT
-    _partition_by_block_id AS partition_key,
-    block_number,
-    VALUE :"array_index" :: INT AS array_index,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
-FROM
-   {{ ref('bronze__traces_fr_v1') }}
--- a/models/streamline/bronze/core/bronze__traces_fr_v1.sql
+++ b/models/streamline/bronze/core/bronze__traces_fr_v1.sql
@ -1,16 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_fr(
-    source_name = 'debug_traceblockbynumber',
-    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
-    partition_join_key = "_partition_by_block_id",
-    block_number = false
-) }}
--- a/models/streamline/bronze/core/bronze__traces_fr_v2.sql
+++ b/models/streamline/bronze/core/bronze__traces_fr_v2.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_fr(
-    source_name = 'traces',
-    source_version = 'v2'
-) }}
--- a/models/streamline/bronze/core/bronze__transactions.sql
+++ b/models/streamline/bronze/core/bronze__transactions.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query(
-    source_name = 'transactions',
-    source_version = 'v2'
-) }}
--- a/models/streamline/bronze/core/bronze__transactions_fr.sql
+++ b/models/streamline/bronze/core/bronze__transactions_fr.sql
@ -1,29 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','phase_1']
-) }}
-
-SELECT
-    partition_key,
-    block_number,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
-FROM
-    {{ ref('bronze__transactions_fr_v2') }}
-UNION ALL
-SELECT
-    _partition_by_block_id AS partition_key,
-    block_number,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
-FROM
-   {{ ref('bronze__transactions_fr_v1') }}
--- a/models/streamline/bronze/core/bronze__transactions_fr_v1.sql
+++ b/models/streamline/bronze/core/bronze__transactions_fr_v1.sql
@ -1,16 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_fr(
-    source_name = 'transactions',
-    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
-    partition_join_key = "_partition_by_block_id",
-    block_number = false
-) }}
--- a/models/streamline/bronze/core/bronze__transactions_fr_v2.sql
+++ b/models/streamline/bronze/core/bronze__transactions_fr_v2.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','phase_1']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_fr(
-    source_name = 'transactions',
-    source_version = 'v2'
-) }}
--- a/models/streamline/bronze/decoder/bronze__decoded_logs.sql
+++ b/models/streamline/bronze/decoder/bronze__decoded_logs.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','decoded_logs','phase_2']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_decoder(
-    source_name = 'decoded_logs',
-    source_version = 'v2'
-) }}
--- a/models/streamline/bronze/decoder/bronze__decoded_logs_fr.sql
+++ b/models/streamline/bronze/decoder/bronze__decoded_logs_fr.sql
@ -1,18 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','decoded_logs','phase_2']
-) }}
-
-SELECT
-    *
-FROM
-    {{ ref('bronze__decoded_logs_fr_v2') }}
-UNION ALL
-SELECT
-    *
-FROM
-    {{ ref('bronze__decoded_logs_fr_v1') }}
--- a/models/streamline/bronze/decoder/bronze__decoded_logs_fr_v1.sql
+++ b/models/streamline/bronze/decoder/bronze__decoded_logs_fr_v1.sql
@ -1,13 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','decoded_logs','streamline_v1','phase_2']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_decoder_fr(
-    source_name = 'decoded_logs'
-) }}
--- a/models/streamline/bronze/decoder/bronze__decoded_logs_fr_v2.sql
+++ b/models/streamline/bronze/decoder/bronze__decoded_logs_fr_v2.sql
@ -1,14 +0,0 @@
-{# Log configuration details #}
-{{ fsc_evm.log_model_details() }}
-
-{# Set up dbt configuration #}
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','decoded_logs','phase_2']
-) }}
-
-{# Main query starts here #}
-{{ fsc_evm.streamline_external_table_query_decoder_fr(
-    source_name = 'decoded_logs',
-    source_version = 'v2'
-) }}
--- a/packages.yml
+++ b/packages.yml
@ -1,3 +1,3 @@
 packages:
  - git: https://github.com/FlipsideCrypto/fsc-evm.git
-    revision: v4.34.0
+    revision: v4.36.0