diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index 1434c4b..f6c4fdf 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -15,5 +15,5 @@ jobs: with: command_name: Run Main Models command: | - dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:silver,tag:balances" "fsc_evm,tag:gold,tag:balances" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" "fsc_evm,tag:streamline,tag:token_reads,tag:realtime" "fsc_evm,tag:streamline,tag:token_reads,tag:complete" "fsc_evm,tag:streamline,tag:balances,tag:complete" "fsc_evm,tag:streamline,tag:balances,tag:records" "fsc_evm,tag:streamline,tag:balances,tag:realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' + dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:silver,tag:balances" "fsc_evm,tag:gold,tag:balances" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" "fsc_evm,tag:streamline,tag:token_reads,tag:realtime" "fsc_evm,tag:streamline,tag:token_reads,tag:complete" "fsc_evm,tag:streamline,tag:balances,tag:complete" "fsc_evm,tag:streamline,tag:balances,tag:records" "fsc_evm,tag:streamline,tag:balances,tag:realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' --exclude "fsc_evm,tag:receipts_by_hash" secrets: inherit \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 777a50f..b7844fb 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -73,8 +73,6 @@ models: +enabled: true core: +enabled: true # enable subpackages, as needed - bronze: - +enabled: false github_actions: +enabled: true labels: @@ -92,13 +90,7 @@ models: abis: +enabled: true decoded_logs: - +enabled: false - gold: - +enabled: true - silver: - +enabled: true - streamline: - +enabled: true + +enabled: true curated_package: +enabled: false stats: diff --git a/models/sources.yml b/models/sources.yml index 93e673f..b04e68b 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -9,16 +9,9 @@ sources: - name: receipts - name: blocks - name: transactions - - name: debug_traceblockbynumber - name: traces - name: decoded_logs - name: confirm_blocks - - name: blocks_v2 - - name: transactions_v2 - - name: receipts_v2 - - name: traces_v2 - - name: confirm_blocks_v2 - - name: decoded_logs_v2 - name: contract_abis - name: token_reads - name: balances_erc20 diff --git a/models/streamline/bronze/core/bronze__blocks.sql b/models/streamline/bronze/core/bronze__blocks.sql deleted file mode 100644 index b5611e0..0000000 --- a/models/streamline/bronze/core/bronze__blocks.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query( - source_name = 'blocks', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__blocks_fr.sql b/models/streamline/bronze/core/bronze__blocks_fr.sql deleted file mode 100644 index d190ec7..0000000 --- a/models/streamline/bronze/core/bronze__blocks_fr.sql +++ /dev/null @@ -1,29 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{{ config ( - materialized = 'view', - tags = ['bronze','core','phase_1'] -) }} - -SELECT - partition_key, - block_number, - VALUE, - DATA, - metadata, - file_name, - _inserted_timestamp -FROM - {{ ref('bronze__blocks_fr_v2') }} -UNION ALL -SELECT - _partition_by_block_id AS partition_key, - block_number, - VALUE, - DATA :result AS DATA, - metadata, - file_name, - _inserted_timestamp -FROM - {{ ref('bronze__blocks_fr_v1') }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__blocks_fr_v1.sql b/models/streamline/bronze/core/bronze__blocks_fr_v1.sql deleted file mode 100644 index 6b0b0bc..0000000 --- a/models/streamline/bronze/core/bronze__blocks_fr_v1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','streamline_v1','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_fr( - source_name = 'blocks', - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", - partition_join_key = "_partition_by_block_id", - block_number = false -) }} diff --git a/models/streamline/bronze/core/bronze__blocks_fr_v2.sql b/models/streamline/bronze/core/bronze__blocks_fr_v2.sql deleted file mode 100644 index 32dcd1d..0000000 --- a/models/streamline/bronze/core/bronze__blocks_fr_v2.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_fr( - source_name = 'blocks', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__confirm_blocks.sql b/models/streamline/bronze/core/bronze__confirm_blocks.sql deleted file mode 100644 index 2927de0..0000000 --- a/models/streamline/bronze/core/bronze__confirm_blocks.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','confirm_blocks','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query( - source_name = 'confirm_blocks', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__confirm_blocks_fr.sql b/models/streamline/bronze/core/bronze__confirm_blocks_fr.sql deleted file mode 100644 index 5b4a5e8..0000000 --- a/models/streamline/bronze/core/bronze__confirm_blocks_fr.sql +++ /dev/null @@ -1,29 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{{ config ( - materialized = 'view', - tags = ['bronze','core','confirm_blocks','phase_1'] -) }} - -SELECT - partition_key, - block_number, - VALUE, - DATA, - metadata, - file_name, - _inserted_timestamp -FROM - {{ ref('bronze__confirm_blocks_fr_v2') }} -UNION ALL -SELECT - _partition_by_block_id AS partition_key, - block_number, - VALUE, - DATA, - metadata, - file_name, - _inserted_timestamp -FROM - {{ ref('bronze__confirm_blocks_fr_v1') }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__confirm_blocks_fr_v1.sql b/models/streamline/bronze/core/bronze__confirm_blocks_fr_v1.sql deleted file mode 100644 index b65db0d..0000000 --- a/models/streamline/bronze/core/bronze__confirm_blocks_fr_v1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','streamline_v1','confirm_blocks','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_fr( - source_name = 'confirm_blocks', - source_version = '', - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", - partition_join_key = "_partition_by_block_id", - block_number = false -) }} diff --git a/models/streamline/bronze/core/bronze__confirm_blocks_fr_v2.sql b/models/streamline/bronze/core/bronze__confirm_blocks_fr_v2.sql deleted file mode 100644 index f14b458..0000000 --- a/models/streamline/bronze/core/bronze__confirm_blocks_fr_v2.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','confirm_blocks','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_fr( - source_name = 'confirm_blocks', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__receipts.sql b/models/streamline/bronze/core/bronze__receipts.sql deleted file mode 100644 index 4e167d2..0000000 --- a/models/streamline/bronze/core/bronze__receipts.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','receipts','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query( - source_name = 'receipts', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__receipts_by_hash.sql b/models/streamline/bronze/core/bronze__receipts_by_hash.sql deleted file mode 100644 index e69de29..0000000 diff --git a/models/streamline/bronze/core/bronze__receipts_by_hash_fr.sql b/models/streamline/bronze/core/bronze__receipts_by_hash_fr.sql deleted file mode 100644 index e69de29..0000000 diff --git a/models/streamline/bronze/core/bronze__receipts_fr.sql b/models/streamline/bronze/core/bronze__receipts_fr.sql deleted file mode 100644 index 9b46789..0000000 --- a/models/streamline/bronze/core/bronze__receipts_fr.sql +++ /dev/null @@ -1,34 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{{ config ( - materialized = 'view', - tags = ['bronze','core','receipts','phase_1'] -) }} - -SELECT - partition_key, - block_number, - array_index, - VALUE, - DATA, - metadata, - file_name, - _inserted_timestamp -FROM - {{ ref('bronze__receipts_fr_v2') }} -UNION ALL -SELECT - _partition_by_block_id AS partition_key, - block_number, - COALESCE( - VALUE :array_index :: INT, - TRY_TO_NUMBER(utils.udf_hex_to_int(VALUE :data :"transactionIndex" :: STRING)) - ) AS array_index, - VALUE, - DATA, - metadata, - file_name, - _inserted_timestamp -FROM - {{ ref('bronze__receipts_fr_v1') }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__receipts_fr_v1.sql b/models/streamline/bronze/core/bronze__receipts_fr_v1.sql deleted file mode 100644 index edba56d..0000000 --- a/models/streamline/bronze/core/bronze__receipts_fr_v1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','streamline_v1','receipts','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_fr( - source_name = 'receipts', - source_version = '', - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", - partition_join_key = "_partition_by_block_id", - block_number = false -) }} diff --git a/models/streamline/bronze/core/bronze__receipts_fr_v2.sql b/models/streamline/bronze/core/bronze__receipts_fr_v2.sql deleted file mode 100644 index cf74f3c..0000000 --- a/models/streamline/bronze/core/bronze__receipts_fr_v2.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','receipts','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_fr( - source_name = 'receipts', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__traces.sql b/models/streamline/bronze/core/bronze__traces.sql deleted file mode 100644 index 9803133..0000000 --- a/models/streamline/bronze/core/bronze__traces.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query( - source_name = 'traces', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__traces_fr.sql b/models/streamline/bronze/core/bronze__traces_fr.sql deleted file mode 100644 index 74b915f..0000000 --- a/models/streamline/bronze/core/bronze__traces_fr.sql +++ /dev/null @@ -1,31 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{{ config ( - materialized = 'view', - tags = ['bronze','core','phase_1'] -) }} - -SELECT - partition_key, - block_number, - array_index, - VALUE, - DATA, - metadata, - file_name, - _inserted_timestamp -FROM - {{ ref('bronze__traces_fr_v2') }} -UNION ALL -SELECT - _partition_by_block_id AS partition_key, - block_number, - VALUE :"array_index" :: INT AS array_index, - VALUE, - DATA, - metadata, - file_name, - _inserted_timestamp -FROM - {{ ref('bronze__traces_fr_v1') }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__traces_fr_v1.sql b/models/streamline/bronze/core/bronze__traces_fr_v1.sql deleted file mode 100644 index c557d2e..0000000 --- a/models/streamline/bronze/core/bronze__traces_fr_v1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','streamline_v1','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_fr( - source_name = 'traces', - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", - partition_join_key = "_partition_by_block_id", - block_number = false -) }} diff --git a/models/streamline/bronze/core/bronze__traces_fr_v2.sql b/models/streamline/bronze/core/bronze__traces_fr_v2.sql deleted file mode 100644 index facbd5d..0000000 --- a/models/streamline/bronze/core/bronze__traces_fr_v2.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_fr( - source_name = 'traces', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__transactions.sql b/models/streamline/bronze/core/bronze__transactions.sql deleted file mode 100644 index df93172..0000000 --- a/models/streamline/bronze/core/bronze__transactions.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query( - source_name = 'transactions', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__transactions_fr.sql b/models/streamline/bronze/core/bronze__transactions_fr.sql deleted file mode 100644 index ccb3e1b..0000000 --- a/models/streamline/bronze/core/bronze__transactions_fr.sql +++ /dev/null @@ -1,29 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{{ config ( - materialized = 'view', - tags = ['bronze','core','phase_1'] -) }} - -SELECT - partition_key, - block_number, - VALUE, - DATA, - metadata, - file_name, - _inserted_timestamp -FROM - {{ ref('bronze__transactions_fr_v2') }} -UNION ALL -SELECT - _partition_by_block_id AS partition_key, - block_number, - VALUE, - DATA, - metadata, - file_name, - _inserted_timestamp -FROM - {{ ref('bronze__transactions_fr_v1') }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__transactions_fr_v1.sql b/models/streamline/bronze/core/bronze__transactions_fr_v1.sql deleted file mode 100644 index 54e50ad..0000000 --- a/models/streamline/bronze/core/bronze__transactions_fr_v1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','streamline_v1','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_fr( - source_name = 'transactions', - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", - partition_join_key = "_partition_by_block_id", - block_number = false -) }} diff --git a/models/streamline/bronze/core/bronze__transactions_fr_v2.sql b/models/streamline/bronze/core/bronze__transactions_fr_v2.sql deleted file mode 100644 index 25da349..0000000 --- a/models/streamline/bronze/core/bronze__transactions_fr_v2.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','core','phase_1'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_fr( - source_name = 'transactions', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/decoder/bronze__decoded_logs.sql b/models/streamline/bronze/decoder/bronze__decoded_logs.sql deleted file mode 100644 index 1d5264a..0000000 --- a/models/streamline/bronze/decoder/bronze__decoded_logs.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','decoded_logs','phase_2'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_decoder( - source_name = 'decoded_logs', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/decoder/bronze__decoded_logs_fr.sql b/models/streamline/bronze/decoder/bronze__decoded_logs_fr.sql deleted file mode 100644 index 03e1cfb..0000000 --- a/models/streamline/bronze/decoder/bronze__decoded_logs_fr.sql +++ /dev/null @@ -1,18 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','decoded_logs','phase_2'] -) }} - -SELECT - * -FROM - {{ ref('bronze__decoded_logs_fr_v2') }} -UNION ALL -SELECT - * -FROM - {{ ref('bronze__decoded_logs_fr_v1') }} \ No newline at end of file diff --git a/models/streamline/bronze/decoder/bronze__decoded_logs_fr_v1.sql b/models/streamline/bronze/decoder/bronze__decoded_logs_fr_v1.sql deleted file mode 100644 index 20e47dd..0000000 --- a/models/streamline/bronze/decoder/bronze__decoded_logs_fr_v1.sql +++ /dev/null @@ -1,13 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','decoded_logs','streamline_v1','phase_2'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_decoder_fr( - source_name = 'decoded_logs' -) }} \ No newline at end of file diff --git a/models/streamline/bronze/decoder/bronze__decoded_logs_fr_v2.sql b/models/streamline/bronze/decoder/bronze__decoded_logs_fr_v2.sql deleted file mode 100644 index c849a83..0000000 --- a/models/streamline/bronze/decoder/bronze__decoded_logs_fr_v2.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# Log configuration details #} -{{ fsc_evm.log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','decoded_logs','phase_2'] -) }} - -{# Main query starts here #} -{{ fsc_evm.streamline_external_table_query_decoder_fr( - source_name = 'decoded_logs', - source_version = 'v2' -) }} \ No newline at end of file diff --git a/packages.yml b/packages.yml index 219d6c4..8052282 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: v4.34.0 + revision: v4.36.0