From 27f1bdc15d5ec9c4611a3759e22fd49aee8a2f7f Mon Sep 17 00:00:00 2001 From: drethereum <71602799+drethereum@users.noreply.github.com> Date: Mon, 17 Mar 2025 10:16:53 -0600 Subject: [PATCH] AN-5797/sl-upgrade-eth (#1030) * sl2 upgrade models * updates for blocks txns and silver * external table vars and blocks structure * integration test * remove --- .../dbt_run_streamline_chainhead.yml | 6 +- .../dbt_run_streamline_history_adhoc.yml | 5 +- dbt_project.yml | 63 ++++++ macros/decoder/decoded_logs_history.sql | 17 +- macros/decoder/run_decoded_history.sql | 4 +- .../streamline_decoder_complete_v0.sql | 49 ++++ .../streamline_external_table_query_v0.sql | 210 ++++++++++++++++++ ...eamline_external_table_queries_decoder.sql | 101 +++++++++ .../streamline_external_table_queries.sql | 141 ++++++++++++ .../main_package/logging/bronze.sql | 36 +++ .../main_package/logging/complete.sql | 29 +++ .../main_package/logging/logging.sql | 36 +++ .../main_package/logging/requests.sql | 55 +++++ .../streamline/set_default_variables.sql | 47 ++++ .../streamline/set_streamline_parameters.sql | 60 +++++ macros/streamline/models.sql | 181 +-------------- models/silver/core/silver__blocks.sql | 54 ++--- .../silver/core/silver__confirmed_blocks.sql | 6 +- models/silver/core/silver__decoded_logs.sql | 6 +- models/silver/core/silver__receipts.sql | 6 +- models/silver/core/silver__traces.sql | 8 +- models/silver/core/silver__transactions.sql | 6 +- .../abis/bronze__streamline_contract_abis.sql | 2 +- ...bronze__streamline_fr_contract_abis_v1.sql | 2 +- ...bronze__streamline_fr_contract_abis_v2.sql | 2 +- .../bronze__streamline_eth_balances.sql | 2 +- .../bronze__streamline_fr_eth_balances_v1.sql | 2 +- .../bronze__streamline_fr_eth_balances_v2.sql | 2 +- ...ronze__streamline_fr_token_balances_v1.sql | 2 +- ...ronze__streamline_fr_token_balances_v2.sql | 2 +- .../bronze__streamline_token_balances.sql | 2 +- .../bronze__streamline_beacon_blobs.sql | 2 +- .../bronze__streamline_beacon_blocks.sql | 2 +- .../bronze__streamline_beacon_validators.sql | 2 +- .../bronze__streamline_fr_beacon_blobs_v2.sql | 2 +- ...bronze__streamline_fr_beacon_blocks_v1.sql | 2 +- ...bronze__streamline_fr_beacon_blocks_v2.sql | 2 +- ...ze__streamline_fr_beacon_validators_v1.sql | 2 +- ...ze__streamline_fr_beacon_validators_v2.sql | 2 +- .../streamline/bronze/core/bronze__blocks.sql | 39 ++++ .../bronze/core/bronze__blocks_fr.sql | 31 +++ .../bronze/core/bronze__blocks_fr_v1.sql | 40 ++++ .../bronze/core/bronze__blocks_fr_v2.sql | 40 ++++ .../bronze/core/bronze__confirm_blocks.sql | 39 ++++ .../bronze/core/bronze__confirm_blocks_fr.sql | 28 +++ .../core/bronze__confirm_blocks_fr_v1.sql | 40 ++++ .../core/bronze__confirm_blocks_fr_v2.sql | 40 ++++ .../bronze/core/bronze__receipts.sql | 39 ++++ .../bronze/core/bronze__receipts_fr.sql | 38 ++++ .../bronze/core/bronze__receipts_fr_v1.sql | 40 ++++ .../bronze/core/bronze__receipts_fr_v2.sql | 40 ++++ .../bronze/core/bronze__streamline_blocks.sql | 7 - .../bronze__streamline_confirmed_blocks.sql | 7 - .../core/bronze__streamline_receipts.sql | 7 - .../bronze/core/bronze__streamline_traces.sql | 7 - .../core/bronze__streamline_transactions.sql | 7 - .../streamline/bronze/core/bronze__traces.sql | 39 ++++ .../bronze/core/bronze__traces_fr.sql | 30 +++ .../bronze/core/bronze__traces_fr_v1.sql | 40 ++++ .../bronze/core/bronze__traces_fr_v2.sql | 40 ++++ .../bronze/core/bronze__transactions.sql | 39 ++++ .../bronze/core/bronze__transactions_fr.sql | 28 +++ .../core/bronze__transactions_fr_v1.sql | 40 ++++ .../core/bronze__transactions_fr_v2.sql | 40 ++++ .../core/fr/bronze__streamline_fr_blocks.sql | 6 - .../fr/bronze__streamline_fr_blocks_v1.sql | 9 - .../fr/bronze__streamline_fr_blocks_v2.sql | 7 - ...bronze__streamline_fr_confirmed_blocks.sql | 6 - ...nze__streamline_fr_confirmed_blocks_v1.sql | 9 - ...nze__streamline_fr_confirmed_blocks_v2.sql | 8 - .../fr/bronze__streamline_fr_receipts.sql | 6 - .../fr/bronze__streamline_fr_receipts_v1.sql | 9 - .../fr/bronze__streamline_fr_receipts_v2.sql | 7 - .../core/fr/bronze__streamline_fr_traces.sql | 6 - .../fr/bronze__streamline_fr_traces_v1.sql | 9 - .../fr/bronze__streamline_fr_traces_v2.sql | 8 - .../fr/bronze__streamline_fr_transactions.sql | 6 - .../bronze__streamline_fr_transactions_v1.sql | 9 - .../bronze__streamline_fr_transactions_v2.sql | 8 - .../bronze__streamline_decoded_logs.sql | 6 - .../bronze__streamline_fr_decoded_logs.sql | 13 -- .../bronze__streamline_fr_decoded_logs_v1.sql | 6 - .../bronze__streamline_fr_decoded_logs_v2.sql | 6 - .../decoded_logs/bronze__decoded_logs.sql | 23 ++ .../decoded_logs/bronze__decoded_logs_fr.sql | 20 ++ .../bronze__decoded_logs_fr_v1.sql | 23 ++ .../bronze__decoded_logs_fr_v2.sql | 23 ++ .../bronze__streamline_decoded_traces.sql | 2 +- .../bronze__streamline_fr_decoded_traces.sql | 0 ...ronze__streamline_fr_decoded_traces_v1.sql | 2 +- ...ronze__streamline_fr_decoded_traces_v2.sql | 2 +- .../reads/bronze__streamline_fr_reads_v1.sql | 2 +- .../reads/bronze__streamline_fr_reads_v2.sql | 2 +- .../bronze/reads/bronze__streamline_reads.sql | 2 +- .../streamline/silver/_24_hour_lookback.sql | 24 +- models/streamline/silver/_block_lookback.sql | 9 +- models/streamline/silver/_block_ranges.sql | 52 ++++- .../streamline/silver/_max_block_by_date.sql | 24 +- .../streamline/silver/_max_block_by_hour.sql | 35 ++- .../complete/streamline__blocks_complete.sql | 50 +++++ .../complete/streamline__complete_blocks.sql | 11 - .../streamline__complete_confirmed_blocks.sql | 10 - .../streamline__complete_receipts.sql | 11 - .../complete/streamline__complete_traces.sql | 11 - .../streamline__complete_transactions.sql | 11 - .../streamline__confirm_blocks_complete.sql | 50 +++++ .../streamline__receipts_complete.sql | 50 +++++ .../complete/streamline__traces_complete.sql | 50 +++++ .../streamline__transactions_complete.sql | 50 +++++ ...treamline__blocks_transactions_history.sql | 116 +++++++++- .../streamline__confirm_blocks_history.sql | 133 +++++++++++ .../history/streamline__receipts_history.sql | 119 ++++++++-- .../history/streamline__traces_history.sql | 117 +++++++++- ...reamline__blocks_transactions_realtime.sql | 137 ++++++++++-- .../streamline__confirm_blocks_realtime.sql | 138 ++++++++++++ .../streamline__confirmed_blocks_realtime.sql | 20 -- .../streamline__receipts_realtime.sql | 137 ++++++++++-- .../realtime/streamline__traces_realtime.sql | 132 +++++++++-- .../silver/core/retry/_missing_receipts.sql | 32 ++- .../silver/core/retry/_missing_traces.sql | 2 +- .../silver/core/retry/_missing_txs.sql | 38 +++- .../silver/core/retry/_unconfirmed_blocks.sql | 32 ++- .../silver/core/streamline__blocks.sql | 33 ++- .../silver/core/streamline__get_chainhead.sql | 55 ++++- .../silver/core/streamline__get_chainhead.yml | 9 + .../streamline__complete_decoded_logs.sql | 13 -- .../streamline__decoded_logs_complete.sql | 50 +++++ .../streamline__decoded_logs_realtime.sql | 110 +++++++++ .../streamline__complete_decoded_traces.sql | 2 +- .../streamline__decode_traces_realtime.sql | 0 .../streamline__decode_logs_realtime.sql | 89 -------- package-lock.yml | 4 +- packages.yml | 2 +- 133 files changed, 3441 insertions(+), 717 deletions(-) create mode 100644 macros/fsc_evm_temp/_legacy/streamline_decoder_complete_v0.sql create mode 100644 macros/fsc_evm_temp/_legacy/streamline_external_table_query_v0.sql create mode 100644 macros/fsc_evm_temp/decoder_package/streamline_external_table_queries_decoder.sql create mode 100644 macros/fsc_evm_temp/main_package/bronze/streamline_external_table_queries.sql create mode 100644 macros/fsc_evm_temp/main_package/logging/bronze.sql create mode 100644 macros/fsc_evm_temp/main_package/logging/complete.sql create mode 100644 macros/fsc_evm_temp/main_package/logging/logging.sql create mode 100644 macros/fsc_evm_temp/main_package/logging/requests.sql create mode 100644 macros/fsc_evm_temp/main_package/streamline/set_default_variables.sql create mode 100644 macros/fsc_evm_temp/main_package/streamline/set_streamline_parameters.sql create mode 100644 models/streamline/bronze/core/bronze__blocks.sql create mode 100644 models/streamline/bronze/core/bronze__blocks_fr.sql create mode 100644 models/streamline/bronze/core/bronze__blocks_fr_v1.sql create mode 100644 models/streamline/bronze/core/bronze__blocks_fr_v2.sql create mode 100644 models/streamline/bronze/core/bronze__confirm_blocks.sql create mode 100644 models/streamline/bronze/core/bronze__confirm_blocks_fr.sql create mode 100644 models/streamline/bronze/core/bronze__confirm_blocks_fr_v1.sql create mode 100644 models/streamline/bronze/core/bronze__confirm_blocks_fr_v2.sql create mode 100644 models/streamline/bronze/core/bronze__receipts.sql create mode 100644 models/streamline/bronze/core/bronze__receipts_fr.sql create mode 100644 models/streamline/bronze/core/bronze__receipts_fr_v1.sql create mode 100644 models/streamline/bronze/core/bronze__receipts_fr_v2.sql delete mode 100644 models/streamline/bronze/core/bronze__streamline_blocks.sql delete mode 100644 models/streamline/bronze/core/bronze__streamline_confirmed_blocks.sql delete mode 100644 models/streamline/bronze/core/bronze__streamline_receipts.sql delete mode 100644 models/streamline/bronze/core/bronze__streamline_traces.sql delete mode 100644 models/streamline/bronze/core/bronze__streamline_transactions.sql create mode 100644 models/streamline/bronze/core/bronze__traces.sql create mode 100644 models/streamline/bronze/core/bronze__traces_fr.sql create mode 100644 models/streamline/bronze/core/bronze__traces_fr_v1.sql create mode 100644 models/streamline/bronze/core/bronze__traces_fr_v2.sql create mode 100644 models/streamline/bronze/core/bronze__transactions.sql create mode 100644 models/streamline/bronze/core/bronze__transactions_fr.sql create mode 100644 models/streamline/bronze/core/bronze__transactions_fr_v1.sql create mode 100644 models/streamline/bronze/core/bronze__transactions_fr_v2.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_blocks.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_blocks_v1.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_blocks_v2.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks_v1.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks_v2.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_receipts.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_receipts_v1.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_receipts_v2.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_traces.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_traces_v1.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_traces_v2.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_transactions.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_transactions_v1.sql delete mode 100644 models/streamline/bronze/core/fr/bronze__streamline_fr_transactions_v2.sql delete mode 100644 models/streamline/bronze/decoder/bronze__streamline_decoded_logs.sql delete mode 100644 models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs.sql delete mode 100644 models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs_v1.sql delete mode 100644 models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs_v2.sql create mode 100644 models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs.sql create mode 100644 models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr.sql create mode 100644 models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr_v1.sql create mode 100644 models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr_v2.sql rename models/streamline/bronze/decoder/{ => decoded_traces}/bronze__streamline_decoded_traces.sql (60%) rename models/streamline/bronze/decoder/{ => decoded_traces}/bronze__streamline_fr_decoded_traces.sql (100%) rename models/streamline/bronze/decoder/{ => decoded_traces}/bronze__streamline_fr_decoded_traces_v1.sql (58%) rename models/streamline/bronze/decoder/{ => decoded_traces}/bronze__streamline_fr_decoded_traces_v2.sql (59%) create mode 100644 models/streamline/silver/core/complete/streamline__blocks_complete.sql delete mode 100644 models/streamline/silver/core/complete/streamline__complete_blocks.sql delete mode 100644 models/streamline/silver/core/complete/streamline__complete_confirmed_blocks.sql delete mode 100644 models/streamline/silver/core/complete/streamline__complete_receipts.sql delete mode 100644 models/streamline/silver/core/complete/streamline__complete_traces.sql delete mode 100644 models/streamline/silver/core/complete/streamline__complete_transactions.sql create mode 100644 models/streamline/silver/core/complete/streamline__confirm_blocks_complete.sql create mode 100644 models/streamline/silver/core/complete/streamline__receipts_complete.sql create mode 100644 models/streamline/silver/core/complete/streamline__traces_complete.sql create mode 100644 models/streamline/silver/core/complete/streamline__transactions_complete.sql create mode 100644 models/streamline/silver/core/history/streamline__confirm_blocks_history.sql create mode 100644 models/streamline/silver/core/realtime/streamline__confirm_blocks_realtime.sql delete mode 100644 models/streamline/silver/core/realtime/streamline__confirmed_blocks_realtime.sql create mode 100644 models/streamline/silver/core/streamline__get_chainhead.yml delete mode 100644 models/streamline/silver/decoder/complete/streamline__complete_decoded_logs.sql create mode 100644 models/streamline/silver/decoder/decoded_logs/complete/streamline__decoded_logs_complete.sql create mode 100644 models/streamline/silver/decoder/decoded_logs/realtime/streamline__decoded_logs_realtime.sql rename models/streamline/silver/decoder/{ => decoded_traces}/complete/streamline__complete_decoded_traces.sql (92%) rename models/streamline/silver/decoder/{ => decoded_traces}/realtime/streamline__decode_traces_realtime.sql (100%) delete mode 100644 models/streamline/silver/decoder/realtime/streamline__decode_logs_realtime.sql diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index 240e6549..a6e2f89e 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -43,4 +43,8 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_core_realtime" "ethereum_models,tag:streamline_core_complete" \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_core_complete" "ethereum_models,tag:streamline_core_realtime" "ethereum_models,tag:streamline_core_complete_receipts" "ethereum_models,tag:streamline_core_realtime_receipts" "ethereum_models,tag:streamline_core_complete_confirm_blocks" "ethereum_models,tag:streamline_core_realtime_confirm_blocks" + + - name: Run Chainhead Tests + run: | + dbt test -m "ethereum_models,tag:chainhead" \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_history_adhoc.yml b/.github/workflows/dbt_run_streamline_history_adhoc.yml index e80664dc..2ddb7686 100644 --- a/.github/workflows/dbt_run_streamline_history_adhoc.yml +++ b/.github/workflows/dbt_run_streamline_history_adhoc.yml @@ -29,8 +29,7 @@ on: description: DBT Run Command required: true options: - - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_core_history" "ethereum_models,tag:streamline_core_complete" - - dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120,"row_limit":2000000}' -m "ethereum_models,tag:streamline_decoded_logs_history" "ethereum_models,tag:streamline_decoded_logs_complete" + - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_core_complete" "ethereum_models,tag:streamline_core_history" "ethereum_models,tag:streamline_core_complete_receipts" "ethereum_models,tag:streamline_core_history_receipts" "ethereum_models,tag:streamline_core_complete_confirm_blocks" "ethereum_models,tag:streamline_core_history_confirm_blocks" - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_beacon_history" "ethereum_models,tag:streamline_beacon_complete" - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_balances_history" "ethereum_models,tag:streamline_balances_complete" - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_abis_history" "ethereum_models,tag:streamline_abis_complete" @@ -50,8 +49,6 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest diff --git a/dbt_project.yml b/dbt_project.yml index cd871f11..b1f899bd 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -99,3 +99,66 @@ vars: - DBT_CLOUD_ETHEREUM #### STREAMLINE 2.0 END #### + +#### FSC_EVM BEGIN #### +# Visit https://github.com/FlipsideCrypto/fsc-evm/wiki for more information on required and optional variables + + ### GLOBAL VARIABLES BEGIN ### + ## REQUIRED + GLOBAL_PROD_DB_NAME: 'ethereum' + GLOBAL_NODE_SECRET_PATH: 'Vault/prod/ethereum/quicknode/ethereum_mainnet' + GLOBAL_BLOCKS_PER_HOUR: 300 + GLOBAL_USES_STREAMLINE_V1: True + + ### GLOBAL VARIABLES END ### + + ### MAIN_PACKAGE VARIABLES BEGIN ### + + ### CORE ### + ## REQUIRED + + BLOCKS_TRANSACTIONS_REALTIME_EXTERNAL_TABLE: 'blocks_v2' + BLOCKS_TRANSACTIONS_HISTORY_EXTERNAL_TABLE: 'blocks_v2' + TRACES_REALTIME_EXTERNAL_TABLE: 'traces_v2' + TRACES_HISTORY_EXTERNAL_TABLE: 'traces_v2' + RECEIPTS_REALTIME_EXTERNAL_TABLE: 'receipts_v2' + RECEIPTS_HISTORY_EXTERNAL_TABLE: 'receipts_v2' + CONFIRM_BLOCKS_REALTIME_EXTERNAL_TABLE: 'confirm_blocks_v2' + CONFIRM_BLOCKS_HISTORY_EXTERNAL_TABLE: 'confirm_blocks_v2' + + ## OPTIONAL + # GOLD_FULL_REFRESH: True + # SILVER_FULL_REFRESH: True + # BRONZE_FULL_REFRESH: True + + # BLOCKS_COMPLETE_FULL_REFRESH: True + # CONFIRM_BLOCKS_COMPLETE_FULL_REFRESH: True + # TRACES_COMPLETE_FULL_REFRESH: True + # RECEIPTS_COMPLETE_FULL_REFRESH: True + # TRANSACTIONS_COMPLETE_FULL_REFRESH: True + + # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 + # BLOCKS_TRANSACTIONS_HISTORY_TESTING_LIMIT: 3 + # TRACES_REALTIME_TESTING_LIMIT: 3 + # TRACES_HISTORY_TESTING_LIMIT: 3 + # RECEIPTS_REALTIME_TESTING_LIMIT: 3 + # RECEIPTS_HISTORY_TESTING_LIMIT: 3 + # CONFIRM_BLOCKS_REALTIME_TESTING_LIMIT: 3 + # CONFIRM_BLOCKS_HISTORY_TESTING_LIMIT: 3 + + # ### MAIN_PACKAGE VARIABLES END ### + + # ### DECODER_PACKAGE VARIABLES BEGIN ### + + # ## REQUIRED + + # ## OPTIONAL + + # DECODED_LOGS_COMPLETE_FULL_REFRESH: True + + # DECODED_LOGS_REALTIME_TESTING_LIMIT: 3 + # DECODED_LOGS_HISTORY_SQL_LIMIT: 1 #limit per monthly range + + ### DECODER_PACKAGE VARIABLES END ### + +#### FSC_EVM END #### \ No newline at end of file diff --git a/macros/decoder/decoded_logs_history.sql b/macros/decoder/decoded_logs_history.sql index 3aab6cab..4fe2282d 100644 --- a/macros/decoder/decoded_logs_history.sql +++ b/macros/decoder/decoded_logs_history.sql @@ -1,28 +1,26 @@ {% macro decoded_logs_history(backfill_mode=false) %} {%- set params = { - "sql_limit": var("DECODED_LOGS_HISTORY_SQL_LIMIT", 7500000), + "sql_limit": var("DECODED_LOGS_HISTORY_SQL_LIMIT", 8000000), "producer_batch_size": var("DECODED_LOGS_HISTORY_PRODUCER_BATCH_SIZE", 400000), "worker_batch_size": var("DECODED_LOGS_HISTORY_WORKER_BATCH_SIZE", 100000) } -%} {% set wait_time = var("DECODED_LOGS_HISTORY_WAIT_TIME", 60) %} - {% set find_months_query %} SELECT DISTINCT date_trunc('month', block_timestamp)::date as month FROM {{ ref('core__fact_blocks') }} ORDER BY month ASC {% endset %} - {% set results = run_query(find_months_query) %} {% if execute %} {% set months = results.columns[0].values() %} - + {% for month in months %} {% set view_name = 'decoded_logs_history_' ~ month.strftime('%Y_%m') %} - + {% set create_view_query %} create or replace view streamline.{{view_name}} as ( WITH target_blocks AS ( @@ -45,7 +43,7 @@ ), existing_logs_to_exclude AS ( SELECT _log_id - FROM {{ ref('streamline__complete_decoded_logs') }} l + FROM {{ ref('streamline__decoded_logs_complete') }} l INNER JOIN target_blocks b using (block_number) ), candidate_logs AS ( @@ -83,11 +81,9 @@ LIMIT {{ params.sql_limit }} ) {% endset %} - {# Create the view #} {% do run_query(create_view_query) %} {{ log("Created view for month " ~ month.strftime('%Y-%m'), info=True) }} - {% if var("STREAMLINE_INVOKE_STREAMS", false) %} {# Check if rows exist first #} {% set check_rows_query %} @@ -98,7 +94,7 @@ {% set has_rows = results.columns[0].values()[0] %} {% if has_rows %} - {# Invoke streamline since rows exist to decode #} + {# Invoke streamline, if rows exist to decode #} {% set decode_query %} SELECT streamline.udf_bulk_decode_logs_v2( @@ -114,7 +110,6 @@ {% do run_query(decode_query) %} {{ log("Triggered decoding for month " ~ month.strftime('%Y-%m'), info=True) }} - {# Call wait since we actually did some decoding #} {% do run_query("call system$wait(" ~ wait_time ~ ")") %} {{ log("Completed wait after decoding for month " ~ month.strftime('%Y-%m'), info=True) }} @@ -122,7 +117,7 @@ {{ log("No rows to decode for month " ~ month.strftime('%Y-%m'), info=True) }} {% endif %} {% endif %} - + {% endfor %} {% endif %} diff --git a/macros/decoder/run_decoded_history.sql b/macros/decoder/run_decoded_history.sql index 8d148639..2ca60453 100644 --- a/macros/decoder/run_decoded_history.sql +++ b/macros/decoder/run_decoded_history.sql @@ -15,7 +15,7 @@ WHERE SELECT github_actions.workflow_dispatches( 'FlipsideCrypto', - 'ethereum-models', + '{{ blockchain }}' || '-models', 'dbt_run_streamline_decoded_logs_history.yml', NULL ) {% endset %} @@ -23,7 +23,7 @@ WHERE SELECT github_actions.workflow_dispatches( 'FlipsideCrypto', - 'ethereum-models', + '{{ blockchain }}' || '-models', 'dbt_run_streamline_decoded_traces_history.yml', NULL ) {% endset %} diff --git a/macros/fsc_evm_temp/_legacy/streamline_decoder_complete_v0.sql b/macros/fsc_evm_temp/_legacy/streamline_decoder_complete_v0.sql new file mode 100644 index 00000000..3345c18e --- /dev/null +++ b/macros/fsc_evm_temp/_legacy/streamline_decoder_complete_v0.sql @@ -0,0 +1,49 @@ +{% macro v0_streamline_decoded_complete( + model + ) %} +SELECT + block_number, + file_name, + id AS {% if model == 'decoded_logs' %} + _log_id {% elif model == 'decoded_traces' %} + _call_id + {% endif %}, + {{ dbt_utils.generate_surrogate_key( + ['id'] + ) }} AS {% if model == 'decoded_logs' %} + complete_decoded_logs_id {% elif model == 'decoded_traces' %} + complete_decoded_traces_id + {% endif %}, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{% if model == 'decoded_logs' %} + {{ ref('bronze__streamline_decoded_logs') }} + + {% elif model == 'decoded_traces' %} + {{ ref('bronze__streamline_decoded_traces') }} +{% endif %} +WHERE + TO_TIMESTAMP_NTZ(_inserted_timestamp) >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) +{% else %} + {% if model == 'decoded_logs' %} + {{ ref('bronze__streamline_fr_decoded_logs') }} + + {% elif model == 'decoded_traces' %} + {{ ref('bronze__streamline_fr_decoded_traces') }} + {% endif %} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 +{% endmacro %} \ No newline at end of file diff --git a/macros/fsc_evm_temp/_legacy/streamline_external_table_query_v0.sql b/macros/fsc_evm_temp/_legacy/streamline_external_table_query_v0.sql new file mode 100644 index 00000000..68219bb9 --- /dev/null +++ b/macros/fsc_evm_temp/_legacy/streamline_external_table_query_v0.sql @@ -0,0 +1,210 @@ +{% macro v0_streamline_external_table_query( + model, + partition_function, + balances = false, + block_number = true + ) %} + WITH meta AS ( + SELECT + job_created_time AS _inserted_timestamp, + file_name, + {{ partition_function }} AS partition_key + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", model) }}') + ) A + ) + SELECT + s.*, + b.file_name, + b._inserted_timestamp + + {% if balances %}, + r.block_timestamp :: TIMESTAMP AS block_timestamp + {% endif %} + + {% if block_number %}, + COALESCE( + s.value :"BLOCK_NUMBER" :: INT, + s.metadata :request :"data" :id :: INT, + PARSE_JSON( + s.metadata :request :"data" + ) :id :: INT + ) AS block_number + {% endif %} + FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.partition_key = s.partition_key + + {% if balances %} + JOIN {{ ref('_block_ranges') }} + r + ON r.block_number = COALESCE( + s.value :"BLOCK_NUMBER" :: INT, + s.value :"block_number" :: INT + ) + {% endif %} + WHERE + b.partition_key = s.partition_key + AND DATA :error IS NULL + AND DATA IS NOT NULL +{% endmacro %} + +{% macro v0_streamline_external_table_fr_query( + model, + partition_function, + partition_join_key = "partition_key", + balances = false, + block_number = true + ) %} + WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + {{ partition_function }} AS partition_key + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", model) }}' + ) + ) A + ) +SELECT + s.*, + b.file_name, + b._inserted_timestamp + + {% if balances %}, + r.block_timestamp :: TIMESTAMP AS block_timestamp +{% endif %} + +{% if block_number %}, + COALESCE( + s.value :"BLOCK_NUMBER" :: INT, + s.value :"block_number" :: INT, + s.metadata :request :"data" :id :: INT, + PARSE_JSON( + s.metadata :request :"data" + ) :id :: INT + ) AS block_number +{% endif %} +FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.partition_key = s.{{ partition_join_key }} + + {% if balances %} + JOIN {{ ref('_block_ranges') }} + r + ON r.block_number = COALESCE( + s.value :"BLOCK_NUMBER" :: INT, + s.value :"block_number" :: INT + ) + {% endif %} +WHERE + b.partition_key = s.{{ partition_join_key }} + AND DATA :error IS NULL + AND DATA IS NOT NULL +{% endmacro %} + +{% macro v0_streamline_external_table_query_decoder( + model + ) %} + WITH meta AS ( + SELECT + job_created_time AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 6), '_', 1) AS INTEGER) AS _partition_by_block_number, + TO_DATE( + concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)) + ) AS _partition_by_created_date + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", model) }}') + ) A + ) + SELECT + block_number, + id :: STRING AS id, + DATA, + metadata, + b.file_name, + _inserted_timestamp, + s._partition_by_block_number AS _partition_by_block_number, + s._partition_by_created_date AS _partition_by_created_date + FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + WHERE + b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + AND s._partition_by_created_date >= DATEADD('day', -2, CURRENT_TIMESTAMP()) + AND DATA :error IS NULL + AND DATA IS NOT NULL +{% endmacro %} + +{% macro v0_streamline_external_table_fr_query_decoder( + model + ) %} + WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 6), '_', 1) AS INTEGER) AS _partition_by_block_number, + TO_DATE( + concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)) + ) AS _partition_by_created_date + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", model) }}' + ) + ) A + ) +SELECT + block_number, + id :: STRING AS id, + DATA, + metadata, + b.file_name, + _inserted_timestamp, + s._partition_by_block_number AS _partition_by_block_number, + s._partition_by_created_date AS _partition_by_created_date +FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date +WHERE + b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + AND DATA :error IS NULL + AND DATA IS NOT NULL +{% endmacro %} \ No newline at end of file diff --git a/macros/fsc_evm_temp/decoder_package/streamline_external_table_queries_decoder.sql b/macros/fsc_evm_temp/decoder_package/streamline_external_table_queries_decoder.sql new file mode 100644 index 00000000..318a3daa --- /dev/null +++ b/macros/fsc_evm_temp/decoder_package/streamline_external_table_queries_decoder.sql @@ -0,0 +1,101 @@ +{% macro streamline_external_table_query_decoder( + source_name, + source_version + ) %} + + {% if source_version != '' %} + {% set source_version = '_' ~ source_version.lower() %} + {% endif %} + + WITH meta AS ( + SELECT + job_created_time AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 6), '_', 1) AS INTEGER) AS _partition_by_block_number, + TO_DATE( + concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)) + ) AS _partition_by_created_date + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", source_name ~ source_version) }}') + ) A + ) + SELECT + block_number, + id :: STRING AS id, + DATA, + metadata, + b.file_name, + _inserted_timestamp, + s._partition_by_block_number AS _partition_by_block_number, + s._partition_by_created_date AS _partition_by_created_date + FROM + {{ source( + "bronze_streamline", + source_name ~ source_version + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + WHERE + b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + AND s._partition_by_created_date >= DATEADD('day', -2, CURRENT_TIMESTAMP()) + AND DATA :error IS NULL + AND DATA IS NOT NULL +{% endmacro %} + + +{% macro streamline_external_table_query_decoder_fr( + source_name, + source_version + ) %} + + {% if source_version != '' %} + {% set source_version = '_' ~ source_version.lower() %} + {% endif %} + + WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 6), '_', 1) AS INTEGER) AS _partition_by_block_number, + TO_DATE( + concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)) + ) AS _partition_by_created_date + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", source_name ~ source_version) }}' + ) + ) A + ) +SELECT + block_number, + id :: STRING AS id, + DATA, + metadata, + b.file_name, + _inserted_timestamp, + s._partition_by_block_number AS _partition_by_block_number, + s._partition_by_created_date AS _partition_by_created_date +FROM + {{ source( + "bronze_streamline", + source_name ~ source_version + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date +WHERE + b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + AND DATA :error IS NULL + AND DATA IS NOT NULL +{% endmacro %} diff --git a/macros/fsc_evm_temp/main_package/bronze/streamline_external_table_queries.sql b/macros/fsc_evm_temp/main_package/bronze/streamline_external_table_queries.sql new file mode 100644 index 00000000..7d9460e3 --- /dev/null +++ b/macros/fsc_evm_temp/main_package/bronze/streamline_external_table_queries.sql @@ -0,0 +1,141 @@ +{% macro streamline_external_table_query( + source_name, + source_version, + partition_function, + balances, + block_number, + uses_receipts_by_hash + ) %} + + {% if source_version != '' %} + {% set source_version = '_' ~ source_version.lower() %} + {% endif %} + + WITH meta AS ( + SELECT + job_created_time AS _inserted_timestamp, + file_name, + {{ partition_function }} AS partition_key + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", source_name ~ source_version) }}') + ) A + ) + SELECT + s.*, + b.file_name, + b._inserted_timestamp + + {% if balances %}, + r.block_timestamp :: TIMESTAMP AS block_timestamp + {% endif %} + + {% if block_number %}, + COALESCE( + s.value :"BLOCK_NUMBER" :: STRING, + s.metadata :request :"data" :id :: STRING, + PARSE_JSON( + s.metadata :request :"data" + ) :id :: STRING + ) :: INT AS block_number + {% endif %} + {% if uses_receipts_by_hash %}, + s.value :"TX_HASH" :: STRING AS tx_hash + {% endif %} + FROM + {{ source( + "bronze_streamline", + source_name ~ source_version + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.partition_key = s.partition_key + + {% if balances %} + JOIN {{ ref('_block_ranges') }} + r + ON r.block_number = COALESCE( + s.value :"BLOCK_NUMBER" :: INT, + s.value :"block_number" :: INT + ) + {% endif %} + WHERE + b.partition_key = s.partition_key + AND DATA :error :code IS NULL + AND DATA IS NOT NULL +{% endmacro %} + +{% macro streamline_external_table_query_fr( + source_name, + source_version, + partition_function, + partition_join_key, + balances, + block_number, + uses_receipts_by_hash + ) %} + + {% if source_version != '' %} + {% set source_version = '_' ~ source_version.lower() %} + {% endif %} + + WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + {{ partition_function }} AS partition_key + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", source_name ~ source_version) }}' + ) + ) A + ) +SELECT + s.*, + b.file_name, + b._inserted_timestamp + + {% if balances %}, + r.block_timestamp :: TIMESTAMP AS block_timestamp +{% endif %} + +{% if block_number %}, + COALESCE( + s.value :"BLOCK_NUMBER" :: STRING, + s.value :"block_number" :: STRING, + s.metadata :request :"data" :id :: STRING, + PARSE_JSON( + s.metadata :request :"data" + ) :id :: STRING + ) :: INT AS block_number +{% endif %} +{% if uses_receipts_by_hash %}, + s.value :"TX_HASH" :: STRING AS tx_hash +{% endif %} +FROM + {{ source( + "bronze_streamline", + source_name ~ source_version + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.partition_key = s.{{ partition_join_key }} + + {% if balances %} + JOIN {{ ref('_block_ranges') }} + r + ON r.block_number = COALESCE( + s.value :"BLOCK_NUMBER" :: INT, + s.value :"block_number" :: INT + ) + {% endif %} +WHERE + b.partition_key = s.{{ partition_join_key }} + AND DATA :error :code IS NULL + AND DATA IS NOT NULL +{% endmacro %} diff --git a/macros/fsc_evm_temp/main_package/logging/bronze.sql b/macros/fsc_evm_temp/main_package/logging/bronze.sql new file mode 100644 index 00000000..b367deb5 --- /dev/null +++ b/macros/fsc_evm_temp/main_package/logging/bronze.sql @@ -0,0 +1,36 @@ +{% macro log_bronze_details(source_name, source_version, model_type, partition_function, partition_join_key, block_number, uses_receipts_by_hash) %} + +{% if source_version != '' %} + {% set source_version = '_' ~ source_version.lower() %} +{% endif %} +{% if model_type != '' %} + {% set model_type = '_' ~ model_type %} +{% endif %} + +{%- if flags.WHICH == 'compile' and execute -%} + + {{ log("=== Current Variable Settings ===", info=True) }} + {{ log(source_name ~ model_type ~ '_PARTITION_FUNCTION: ' ~ partition_function, info=True) }} + {{ log(source_name ~ model_type ~ '_PARTITION_JOIN_KEY: ' ~ partition_join_key, info=True) }} + {{ log(source_name ~ model_type ~ '_BLOCK_NUMBER: ' ~ block_number, info=True) }} + {% if uses_receipts_by_hash %} + {{ log("USES_RECEIPTS_BY_HASH: " ~ uses_receipts_by_hash, info=True) }} + {% endif %} + + {{ log("", info=True) }} + {{ log("=== Source Details ===", info=True) }} + {{ log("Source: " ~ source('bronze_streamline', source_name.lower() ~ source_version.lower()), info=True) }} + {{ log("", info=True) }} + + {% set config_log = '\n' %} + {% set config_log = config_log ~ '\n=== DBT Model Config ===\n'%} + {% set config_log = config_log ~ '\n{{ config (\n' %} + {% set config_log = config_log ~ ' materialized = "' ~ config.get('materialized') ~ '",\n' %} + {% set config_log = config_log ~ ' tags = ' ~ config.get('tags') | tojson ~ '\n' %} + {% set config_log = config_log ~ ') }}\n' %} + {{ log(config_log, info=True) }} + {{ log("", info=True) }} + +{%- endif -%} + +{% endmacro %} \ No newline at end of file diff --git a/macros/fsc_evm_temp/main_package/logging/complete.sql b/macros/fsc_evm_temp/main_package/logging/complete.sql new file mode 100644 index 00000000..3637b41e --- /dev/null +++ b/macros/fsc_evm_temp/main_package/logging/complete.sql @@ -0,0 +1,29 @@ +{% macro log_complete_details(post_hook, full_refresh_type, uses_receipts_by_hash) %} + +{%- if flags.WHICH == 'compile' and execute -%} + + {% if uses_receipts_by_hash %} + + {{ log("=== Current Variable Settings ===", info=True) }} + {{ log("USES_RECEIPTS_BY_HASH: " ~ uses_receipts_by_hash, info=True) }} + + {% endif %} + + {% set config_log = '\n' %} + {% set config_log = config_log ~ '\n=== DBT Model Config ===\n'%} + {% set config_log = config_log ~ '\n{{ config (\n' %} + {% set config_log = config_log ~ ' materialized = "' ~ config.get('materialized') ~ '",\n' %} + {% set config_log = config_log ~ ' unique_key = "' ~ config.get('unique_key') ~ '",\n' %} + {% set config_log = config_log ~ ' cluster_by = "' ~ config.get('cluster_by') ~ '",\n' %} + {% set config_log = config_log ~ ' merge_update_columns = ' ~ config.get('merge_update_columns') | tojson ~ ',\n' %} + {% set config_log = config_log ~ ' post_hook = "' ~ post_hook ~ '",\n' %} + {% set config_log = config_log ~ ' incremental_predicates = ' ~ config.get('incremental_predicates') | tojson ~ ',\n' %} + {% set config_log = config_log ~ ' full_refresh = ' ~ full_refresh_type ~ ',\n' %} + {% set config_log = config_log ~ ' tags = ' ~ config.get('tags') | tojson ~ '\n' %} + {% set config_log = config_log ~ ') }}\n' %} + {{ log(config_log, info=True) }} + {{ log("", info=True) }} + +{%- endif -%} + +{% endmacro %} \ No newline at end of file diff --git a/macros/fsc_evm_temp/main_package/logging/logging.sql b/macros/fsc_evm_temp/main_package/logging/logging.sql new file mode 100644 index 00000000..f3686852 --- /dev/null +++ b/macros/fsc_evm_temp/main_package/logging/logging.sql @@ -0,0 +1,36 @@ +{% macro log_model_details(vars=false, params=false) %} + +{%- if execute -%} +/* +DBT Model Config: +{{ model.config | tojson(indent=2) }} +*/ + +{% if vars is not false %} + +{% if var('LOG_MODEL_DETAILS', false) %} +{{ log( vars | tojson(indent=2), info=True) }} +{% endif %} +/* +Variables: +{{ vars | tojson(indent=2) }} +*/ +{% endif %} + +{% if params is not false %} + +{% if var('LOG_MODEL_DETAILS', false) %} +{{ log( params | tojson(indent=2), info=True) }} +{% endif %} +/* +Parameters: +{{ params | tojson(indent=2) }} +*/ +{% endif %} + +/* +Raw Code: +{{ model.raw_code }} +*/ +{%- endif -%} +{% endmacro %} \ No newline at end of file diff --git a/macros/fsc_evm_temp/main_package/logging/requests.sql b/macros/fsc_evm_temp/main_package/logging/requests.sql new file mode 100644 index 00000000..85cbbda1 --- /dev/null +++ b/macros/fsc_evm_temp/main_package/logging/requests.sql @@ -0,0 +1,55 @@ +{% macro log_streamline_details(model_name, model_type, node_url, model_quantum_state, sql_limit, testing_limit, order_by_clause, new_build, streamline_params, uses_receipts_by_hash, method, method_params, min_block=0) %} + +{%- if flags.WHICH == 'compile' and execute -%} + + {{ log("=== Current Variable Settings ===", info=True) }} + {{ log("START_UP_BLOCK: " ~ min_block, info=True) }} + {{ log("", info=True) }} + + {{ log("=== API Details ===", info=True) }} + + {{ log("NODE_URL: " ~ node_url, info=True) }} + {{ log("NODE_SECRET_PATH: " ~ var('GLOBAL_NODE_SECRET_PATH'), info=True) }} + {{ log("", info=True) }} + + {{ log("=== Current Variable Settings ===", info=True) }} + + {{ log((model_name ~ '_' ~ model_type ~ '_model_quantum_state').upper() ~ ': ' ~ model_quantum_state, info=True) }} + {{ log((model_name ~ '_' ~ model_type ~ '_sql_limit').upper() ~ ': ' ~ sql_limit, info=True) }} + {{ log((model_name ~ '_' ~ model_type ~ '_testing_limit').upper() ~ ': ' ~ testing_limit, info=True) }} + {{ log((model_name ~ '_' ~ model_type ~ '_order_by_clause').upper() ~ ': ' ~ order_by_clause, info=True) }} + {{ log((model_name ~ '_' ~ model_type ~ '_new_build').upper() ~ ': ' ~ new_build, info=True) }} + {{ log('USES_RECEIPTS_BY_HASH' ~ ': ' ~ uses_receipts_by_hash, info=True) }} + {{ log("", info=True) }} + + {{ log("=== RPC Details ===", info=True) }} + + {{ log(model_name ~ ": {", info=True) }} + {{ log(" method: '" ~ method ~ "',", info=True) }} + {{ log(" method_params: " ~ method_params, info=True) }} + {{ log("}", info=True) }} + {{ log("", info=True) }} + + {% set params_str = streamline_params | tojson %} + {% set params_formatted = params_str | replace('{', '{\n ') | replace('}', '\n }') | replace(', ', ',\n ') %} + + {# Clean up the method_params formatting #} + {% set params_formatted = params_formatted | replace('"method_params": "', '"method_params": "') | replace('\\n', ' ') | replace('\\u0027', "'") %} + + {% set config_log = '\n' %} + {% set config_log = config_log ~ '\n=== DBT Model Config ===\n'%} + {% set config_log = config_log ~ '\n{{ config (\n' %} + {% set config_log = config_log ~ ' materialized = "' ~ config.get('materialized') ~ '",\n' %} + {% set config_log = config_log ~ ' post_hook = fsc_utils.if_data_call_function_v2(\n' %} + {% set config_log = config_log ~ ' func = "streamline.udf_bulk_rest_api_v2",\n' %} + {% set config_log = config_log ~ ' target = "' ~ this.schema ~ '.' ~ this.identifier ~ '",\n' %} + {% set config_log = config_log ~ ' params = ' ~ params_formatted ~ '\n' %} + {% set config_log = config_log ~ ' ),\n' %} + {% set config_log = config_log ~ ' tags = ' ~ config.get('tags') | tojson ~ '\n' %} + {% set config_log = config_log ~ ') }}\n' %} + {{ log(config_log, info=True) }} + {{ log("", info=True) }} + +{%- endif -%} + +{% endmacro %} \ No newline at end of file diff --git a/macros/fsc_evm_temp/main_package/streamline/set_default_variables.sql b/macros/fsc_evm_temp/main_package/streamline/set_default_variables.sql new file mode 100644 index 00000000..598c3b95 --- /dev/null +++ b/macros/fsc_evm_temp/main_package/streamline/set_default_variables.sql @@ -0,0 +1,47 @@ +{% macro set_default_variables_streamline(model_name, model_type) %} + +{%- set node_url = var('GLOBAL_NODE_URL', '{Service}/{Authentication}') -%} +{%- set node_secret_path = var('GLOBAL_NODE_SECRET_PATH', '') -%} +{%- set model_quantum_state = var((model_name ~ '_' ~ model_type ~ '_quantum_state').upper(), 'streamline') -%} +{%- set testing_limit = var((model_name ~ '_' ~ model_type ~ '_testing_limit').upper(), none) -%} +{%- set new_build = var((model_name ~ '_' ~ model_type ~ '_new_build').upper(), false) -%} +{%- set default_order = 'ORDER BY partition_key DESC, block_number DESC' if model_type.lower() == 'realtime' + else 'ORDER BY partition_key ASC, block_number ASC' -%} +{%- set order_by_clause = var((model_name ~ '_' ~ model_type ~ '_order_by_clause').upper(), default_order) -%} +{%- set uses_receipts_by_hash = var('GLOBAL_USES_RECEIPTS_BY_HASH', false) -%} + +{%- set variables = { + 'node_url': node_url, + 'node_secret_path': node_secret_path, + 'model_quantum_state': model_quantum_state, + 'testing_limit': testing_limit, + 'new_build': new_build, + 'order_by_clause': order_by_clause, + 'uses_receipts_by_hash': uses_receipts_by_hash +} -%} + +{{ return(variables) }} + +{% endmacro %} + +{% macro set_default_variables_bronze(source_name, model_type) %} + +{%- set partition_function = var(source_name ~ model_type ~ '_PARTITION_FUNCTION', + "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)") +-%} +{%- set partition_join_key = var(source_name ~ model_type ~ '_PARTITION_JOIN_KEY', 'partition_key') -%} +{%- set block_number = var(source_name ~ model_type ~ '_BLOCK_NUMBER', true) -%} +{%- set balances = var(source_name ~ model_type ~ '_BALANCES', false) -%} +{%- set uses_receipts_by_hash = var('GLOBAL_USES_RECEIPTS_BY_HASH', false) -%} + +{%- set variables = { + 'partition_function': partition_function, + 'partition_join_key': partition_join_key, + 'block_number': block_number, + 'balances': balances, + 'uses_receipts_by_hash': uses_receipts_by_hash +} -%} + +{{ return(variables) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/fsc_evm_temp/main_package/streamline/set_streamline_parameters.sql b/macros/fsc_evm_temp/main_package/streamline/set_streamline_parameters.sql new file mode 100644 index 00000000..6b6e5ff9 --- /dev/null +++ b/macros/fsc_evm_temp/main_package/streamline/set_streamline_parameters.sql @@ -0,0 +1,60 @@ +{% macro set_streamline_parameters(model_name, model_type, multiplier=1) %} + +{%- set rpc_config_details = { + "blocks_transactions": { + "method": 'eth_getBlockByNumber', + "method_params": 'ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), TRUE)', + "exploded_key": ['result', 'result.transactions'] + }, + "receipts_by_hash": { + "method": 'eth_getTransactionReceipt', + "method_params": 'ARRAY_CONSTRUCT(tx_hash)' + }, + "receipts": { + "method": 'eth_getBlockReceipts', + "method_params": 'ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number))', + "exploded_key": ['result'], + "lambdas": 2 + + }, + "traces": { + "method": 'debug_traceBlockByNumber', + "method_params": "ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), OBJECT_CONSTRUCT('tracer', 'callTracer', 'timeout', '120s'))", + "exploded_key": ['result'], + "lambdas": 2 + }, + "confirm_blocks": { + "method": 'eth_getBlockByNumber', + "method_params": 'ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), FALSE)' + } +} -%} + +{%- set rpc_config = rpc_config_details[model_name.lower()] -%} + +{%- set params = { + "external_table": var((model_name ~ '_' ~ model_type ~ '_external_table').upper(), model_name.lower()), + "sql_limit": var((model_name ~ '_' ~ model_type ~ '_sql_limit').upper(), 2 * var('GLOBAL_BLOCKS_PER_HOUR',0) * multiplier), + "producer_batch_size": var((model_name ~ '_' ~ model_type ~ '_producer_batch_size').upper(), 2 * var('GLOBAL_BLOCKS_PER_HOUR',0) * multiplier), + "worker_batch_size": var( + (model_name ~ '_' ~ model_type ~ '_worker_batch_size').upper(), + (2 * var('GLOBAL_BLOCKS_PER_HOUR',0) * multiplier) // (rpc_config.get('lambdas', 1)) + ), + "sql_source": (model_name ~ '_' ~ model_type).lower(), + "method": rpc_config['method'], + "method_params": rpc_config['method_params'] +} -%} + +{%- if rpc_config.get('exploded_key') is not none -%} + {%- do params.update({"exploded_key": tojson(rpc_config['exploded_key'])}) -%} +{%- endif -%} + +{%- if rpc_config.get('lambdas') is not none -%} + {%- do params.update({"lambdas": rpc_config['lambdas']}) -%} +{%- endif -%} + +{{ return(params) }} + +{% endmacro %} + + + diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql index 58d15423..aaa1290f 100644 --- a/macros/streamline/models.sql +++ b/macros/streamline/models.sql @@ -1,63 +1,3 @@ -{% macro decode_logs_history( - start, - stop - ) %} - WITH look_back AS ( - SELECT - block_number - FROM - {{ ref("_24_hour_lookback") }} - ) -SELECT - l.block_number, - l._log_id, - a.abi AS abi, - OBJECT_CONSTRUCT( - 'topics', - l.topics, - 'data', - l.data, - 'address', - l.contract_address - ) AS DATA -FROM - {{ ref("silver__logs") }} - l - INNER JOIN {{ ref("silver__complete_event_abis") }} - a - ON a.parent_contract_address = l.contract_address - and a.event_signature = l.topics[0]::string - and l.block_number between a.start_block and a.end_block -WHERE - ( - l.block_number BETWEEN {{ start }} - AND {{ stop }} - ) - AND l.block_number <= ( - SELECT - block_number - FROM - look_back - ) - AND _log_id NOT IN ( - SELECT - _log_id - FROM - {{ ref("streamline__complete_decode_logs") }} - WHERE - ( - block_number BETWEEN {{ start }} - AND {{ stop }} - ) - AND block_number <= ( - SELECT - block_number - FROM - look_back - ) - ) -{% endmacro %} - {% macro decode_traces_history( start, stop @@ -123,123 +63,4 @@ WHERE look_back )) -{% endmacro %} - -{% macro streamline_external_table_query( - model, - partition_function, - partition_name, - unique_key - ) %} - WITH meta AS ( - SELECT - last_modified AS _inserted_timestamp, - file_name, - {{ partition_function }} AS {{ partition_name }} - FROM - TABLE( - information_schema.external_table_file_registration_history( - start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), - table_name => '{{ source( "bronze_streamline", model) }}') - ) A - ) - SELECT - {{ unique_key }}, - DATA, - _inserted_timestamp, - MD5( - CAST( - COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text - ) - ) AS id, - s.{{ partition_name }}, - s.value AS value - FROM - {{ source( - "bronze_streamline", - model - ) }} - s - JOIN meta b - ON b.file_name = metadata$filename - AND b.{{ partition_name }} = s.{{ partition_name }} - WHERE - b.{{ partition_name }} = s.{{ partition_name }} - AND ( - DATA :error :code IS NULL - OR DATA :error :code NOT IN ( - '-32000', - '-32001', - '-32002', - '-32003', - '-32004', - '-32005', - '-32006', - '-32007', - '-32008', - '-32009', - '-32010', - '-32608' - ) - ) -{% endmacro %} - -{% macro streamline_external_table_FR_query( - model, - partition_function, - partition_name, - unique_key - ) %} - WITH meta AS ( - SELECT - registered_on AS _inserted_timestamp, - file_name, - {{ partition_function }} AS {{ partition_name }} - FROM - TABLE( - information_schema.external_table_files( - table_name => '{{ source( "bronze_streamline", model) }}' - ) - ) A - ) -SELECT - {{ unique_key }}, - DATA, - _inserted_timestamp, - MD5( - CAST( - COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text - ) - ) AS id, - s.{{ partition_name }}, - s.value AS value -FROM - {{ source( - "bronze_streamline", - model - ) }} - s - JOIN meta b - ON b.file_name = metadata$filename - AND b.{{ partition_name }} = s.{{ partition_name }} -WHERE - b.{{ partition_name }} = s.{{ partition_name }} - AND ( - DATA :error :code IS NULL - OR DATA :error :code NOT IN ( - '-32000', - '-32001', - '-32002', - '-32003', - '-32004', - '-32005', - '-32006', - '-32007', - '-32008', - '-32009', - '-32010', - '-32608' - ) - ) -{% endmacro %} - +{% endmacro %} \ No newline at end of file diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index be5ae5ad..ad4ef0b8 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_blocks') }} +-- depends_on: {{ ref('bronze__blocks') }} {{ config( materialized = 'incremental', unique_key = "block_number", @@ -12,47 +12,47 @@ SELECT block_number, utils.udf_hex_to_int( - DATA :result :baseFeePerGas :: STRING + DATA :baseFeePerGas :: STRING ) :: INT AS base_fee_per_gas, utils.udf_hex_to_int( - DATA :result :difficulty :: STRING + DATA :difficulty :: STRING ) :: INT AS difficulty, - DATA :result :extraData :: STRING AS extra_data, + DATA :extraData :: STRING AS extra_data, utils.udf_hex_to_int( - DATA :result :gasLimit :: STRING + DATA :gasLimit :: STRING ) :: INT AS gas_limit, utils.udf_hex_to_int( - DATA :result :gasUsed :: STRING + DATA :gasUsed :: STRING ) :: INT AS gas_used, - DATA :result :hash :: STRING AS HASH, - DATA :result :logsBloom :: STRING AS logs_bloom, - DATA :result :miner :: STRING AS miner, + DATA :hash :: STRING AS HASH, + DATA :logsBloom :: STRING AS logs_bloom, + DATA :miner :: STRING AS miner, utils.udf_hex_to_int( - DATA :result :nonce :: STRING + DATA :nonce :: STRING ) :: INT AS nonce, utils.udf_hex_to_int( - DATA :result :number :: STRING + DATA :number :: STRING ) :: INT AS NUMBER, - DATA :result :parentHash :: STRING AS parent_hash, - DATA :result :receiptsRoot :: STRING AS receipts_root, - DATA :result :sha3Uncles :: STRING AS sha3_uncles, + DATA :parentHash :: STRING AS parent_hash, + DATA :receiptsRoot :: STRING AS receipts_root, + DATA :sha3Uncles :: STRING AS sha3_uncles, utils.udf_hex_to_int( - DATA :result :size :: STRING + DATA :size :: STRING ) :: INT AS SIZE, - DATA :result :stateRoot :: STRING AS state_root, + DATA :stateRoot :: STRING AS state_root, utils.udf_hex_to_int( - DATA :result :timestamp :: STRING + DATA :timestamp :: STRING ) :: TIMESTAMP AS block_timestamp, utils.udf_hex_to_int( - DATA :result :totalDifficulty :: STRING + DATA :totalDifficulty :: STRING ) :: INT AS total_difficulty, ARRAY_SIZE( - DATA :result :transactions + DATA :transactions ) AS tx_count, - DATA :result :transactionsRoot :: STRING AS transactions_root, - DATA :result :uncles AS uncles, - DATA :result :withdrawals AS withdrawals, - DATA :result :withdrawalsRoot :: STRING AS withdrawals_root, + DATA :transactionsRoot :: STRING AS transactions_root, + DATA :uncles AS uncles, + DATA :withdrawals AS withdrawals, + DATA :withdrawalsRoot :: STRING AS withdrawals_root, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['block_number'] @@ -61,15 +61,15 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id, utils.udf_hex_to_int( - DATA :result: blobGasUsed :: STRING + DATA: blobGasUsed :: STRING ) :: INT AS blob_gas_used, utils.udf_hex_to_int( - DATA :result: excessBlobGas :: STRING + DATA: excessBlobGas :: STRING ) :: INT AS excess_blob_gas FROM {% if is_incremental() %} -{{ ref('bronze__streamline_blocks') }} +{{ ref('bronze__blocks') }} WHERE _inserted_timestamp >= ( SELECT @@ -78,7 +78,7 @@ WHERE {{ this }} ) {% else %} - {{ ref('bronze__streamline_fr_blocks') }} + {{ ref('bronze__blocks_fr') }} {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number diff --git a/models/silver/core/silver__confirmed_blocks.sql b/models/silver/core/silver__confirmed_blocks.sql index 9840dc7a..6d18edda 100644 --- a/models/silver/core/silver__confirmed_blocks.sql +++ b/models/silver/core/silver__confirmed_blocks.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_confirmed_blocks') }} +-- depends_on: {{ ref('bronze__confirm_blocks') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', @@ -17,7 +17,7 @@ WITH base AS ( FROM {% if is_incremental() %} -{{ ref('bronze__streamline_confirmed_blocks') }} +{{ ref('bronze__confirm_blocks') }} WHERE _inserted_timestamp >= ( SELECT @@ -31,7 +31,7 @@ WHERE {{ this }} ) {% else %} - {{ ref('bronze__streamline_fr_confirmed_blocks') }} + {{ ref('bronze__confirm_blocks_fr') }} {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number diff --git a/models/silver/core/silver__decoded_logs.sql b/models/silver/core/silver__decoded_logs.sql index 12dbf3c4..20338b18 100644 --- a/models/silver/core/silver__decoded_logs.sql +++ b/models/silver/core/silver__decoded_logs.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_decoded_logs') }} +-- depends_on: {{ ref('bronze__decoded_logs') }} {{ config ( materialized = "incremental", unique_key = ['block_number', 'event_index'], @@ -32,7 +32,7 @@ WITH base_data AS ( FROM {% if is_incremental() %} -{{ ref('bronze__streamline_decoded_logs') }} +{{ ref('bronze__decoded_logs') }} WHERE TO_TIMESTAMP_NTZ(_inserted_timestamp) >= ( SELECT @@ -42,7 +42,7 @@ WHERE ) AND DATA NOT ILIKE '%Event topic is not present in given ABI%' {% else %} - {{ ref('bronze__streamline_fr_decoded_logs') }} + {{ ref('bronze__decoded_logs_fr') }} WHERE _partition_by_block_number <= 4000000 AND DATA NOT ILIKE '%Event topic is not present in given ABI%' diff --git a/models/silver/core/silver__receipts.sql b/models/silver/core/silver__receipts.sql index ba72af49..0b4ff35c 100644 --- a/models/silver/core/silver__receipts.sql +++ b/models/silver/core/silver__receipts.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_receipts') }} +-- depends_on: {{ ref('bronze__receipts') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', @@ -18,7 +18,7 @@ WITH base AS ( FROM {% if is_incremental() %} -{{ ref('bronze__streamline_receipts') }} +{{ ref('bronze__receipts') }} WHERE _inserted_timestamp >= ( SELECT @@ -30,7 +30,7 @@ WHERE DATA ) {% else %} - {{ ref('bronze__streamline_fr_receipts') }} + {{ ref('bronze__receipts_fr') }} WHERE IS_OBJECT( DATA diff --git a/models/silver/core/silver__traces.sql b/models/silver/core/silver__traces.sql index 9619b4b5..142e9a0c 100644 --- a/models/silver/core/silver__traces.sql +++ b/models/silver/core/silver__traces.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_traces') }} +-- depends_on: {{ ref('bronze__traces') }} {{ config ( materialized = "incremental", incremental_strategy = 'delete+insert', @@ -26,7 +26,7 @@ WITH bronze_traces AS ( {% if is_incremental() and not full_reload_mode %} {{ ref( - 'bronze__streamline_traces' + 'bronze__traces' ) }} WHERE _inserted_timestamp >= ( @@ -37,7 +37,7 @@ WHERE ) AND DATA :result IS NOT NULL {% elif is_incremental() and full_reload_mode %} {{ ref( - 'bronze__streamline_fr_traces' + 'bronze__traces_fr' ) }} WHERE partition_key BETWEEN ( @@ -54,7 +54,7 @@ WHERE ) {% else %} {{ ref( - 'bronze__streamline_fr_traces' + 'bronze__traces_fr' ) }} WHERE partition_key <= 3000000 diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index 9e9243c5..77ef81d8 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_transactions') }} +-- depends_on: {{ ref('bronze__transactions') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', @@ -17,7 +17,7 @@ WITH base AS ( FROM {% if is_incremental() %} -{{ ref('bronze__streamline_transactions') }} +{{ ref('bronze__transactions') }} WHERE _inserted_timestamp >= ( SELECT @@ -27,7 +27,7 @@ WHERE ) AND IS_OBJECT(DATA) {% else %} - {{ ref('bronze__streamline_fr_transactions') }} + {{ ref('bronze__transactions_fr') }} WHERE IS_OBJECT(DATA) {% endif %} diff --git a/models/streamline/bronze/abis/bronze__streamline_contract_abis.sql b/models/streamline/bronze/abis/bronze__streamline_contract_abis.sql index 10ca9f7d..2574871b 100644 --- a/models/streamline/bronze/abis/bronze__streamline_contract_abis.sql +++ b/models/streamline/bronze/abis/bronze__streamline_contract_abis.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_query( +{{ v0_streamline_external_table_query( model = "contract_abis_v2", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)" ) }} diff --git a/models/streamline/bronze/abis/bronze__streamline_fr_contract_abis_v1.sql b/models/streamline/bronze/abis/bronze__streamline_fr_contract_abis_v1.sql index 89c776a6..aa66d31b 100644 --- a/models/streamline/bronze/abis/bronze__streamline_fr_contract_abis_v1.sql +++ b/models/streamline/bronze/abis/bronze__streamline_fr_contract_abis_v1.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "contract_abis", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", partition_join_key = "_partition_by_block_id", diff --git a/models/streamline/bronze/abis/bronze__streamline_fr_contract_abis_v2.sql b/models/streamline/bronze/abis/bronze__streamline_fr_contract_abis_v2.sql index 47bbad4c..f2eda0dd 100644 --- a/models/streamline/bronze/abis/bronze__streamline_fr_contract_abis_v2.sql +++ b/models/streamline/bronze/abis/bronze__streamline_fr_contract_abis_v2.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "contract_abis_v2", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" ) }} diff --git a/models/streamline/bronze/balances/bronze__streamline_eth_balances.sql b/models/streamline/bronze/balances/bronze__streamline_eth_balances.sql index 1ee1e379..ba4bfa86 100644 --- a/models/streamline/bronze/balances/bronze__streamline_eth_balances.sql +++ b/models/streamline/bronze/balances/bronze__streamline_eth_balances.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_query( +{{ v0_streamline_external_table_query( model = "eth_balances_v2", partition_function = "TO_NUMBER(SPLIT_PART(file_name, '/', 3))", balances = true diff --git a/models/streamline/bronze/balances/bronze__streamline_fr_eth_balances_v1.sql b/models/streamline/bronze/balances/bronze__streamline_fr_eth_balances_v1.sql index ad0047a4..cf4faf0b 100644 --- a/models/streamline/bronze/balances/bronze__streamline_fr_eth_balances_v1.sql +++ b/models/streamline/bronze/balances/bronze__streamline_fr_eth_balances_v1.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "eth_balances", partition_function = "TO_NUMBER(SPLIT_PART(file_name, '/', 3))", partition_join_key = "_partition_by_block_id", diff --git a/models/streamline/bronze/balances/bronze__streamline_fr_eth_balances_v2.sql b/models/streamline/bronze/balances/bronze__streamline_fr_eth_balances_v2.sql index dbe30459..8013a5cb 100644 --- a/models/streamline/bronze/balances/bronze__streamline_fr_eth_balances_v2.sql +++ b/models/streamline/bronze/balances/bronze__streamline_fr_eth_balances_v2.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "eth_balances_v2", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", balances = true diff --git a/models/streamline/bronze/balances/bronze__streamline_fr_token_balances_v1.sql b/models/streamline/bronze/balances/bronze__streamline_fr_token_balances_v1.sql index 9cf7fcb2..d17e363e 100644 --- a/models/streamline/bronze/balances/bronze__streamline_fr_token_balances_v1.sql +++ b/models/streamline/bronze/balances/bronze__streamline_fr_token_balances_v1.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "token_balances", partition_function = "TO_NUMBER(SPLIT_PART(file_name, '/', 3))", partition_join_key = "_partition_by_block_id", diff --git a/models/streamline/bronze/balances/bronze__streamline_fr_token_balances_v2.sql b/models/streamline/bronze/balances/bronze__streamline_fr_token_balances_v2.sql index f8b790b4..1a89c00a 100644 --- a/models/streamline/bronze/balances/bronze__streamline_fr_token_balances_v2.sql +++ b/models/streamline/bronze/balances/bronze__streamline_fr_token_balances_v2.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "token_balances_v2", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", balances = true diff --git a/models/streamline/bronze/balances/bronze__streamline_token_balances.sql b/models/streamline/bronze/balances/bronze__streamline_token_balances.sql index dd4cfe90..d4aca23b 100644 --- a/models/streamline/bronze/balances/bronze__streamline_token_balances.sql +++ b/models/streamline/bronze/balances/bronze__streamline_token_balances.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_query( +{{ v0_streamline_external_table_query( model = "token_balances_v2", partition_function = "TO_NUMBER(SPLIT_PART(file_name, '/', 3))", balances = true diff --git a/models/streamline/bronze/beacon/bronze__streamline_beacon_blobs.sql b/models/streamline/bronze/beacon/bronze__streamline_beacon_blobs.sql index 72830115..9134f2f2 100644 --- a/models/streamline/bronze/beacon/bronze__streamline_beacon_blobs.sql +++ b/models/streamline/bronze/beacon/bronze__streamline_beacon_blobs.sql @@ -2,7 +2,7 @@ materialized = 'view', tags = ['bronze_beacon_blobs'] ) }} -{{ fsc_evm.streamline_external_table_query( +{{ v0_streamline_external_table_query( model = "beacon_blobs_v2", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)", block_number = false diff --git a/models/streamline/bronze/beacon/bronze__streamline_beacon_blocks.sql b/models/streamline/bronze/beacon/bronze__streamline_beacon_blocks.sql index fe847809..88b02e73 100644 --- a/models/streamline/bronze/beacon/bronze__streamline_beacon_blocks.sql +++ b/models/streamline/bronze/beacon/bronze__streamline_beacon_blocks.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_query( +{{ v0_streamline_external_table_query( model = "beacon_blocks_v2", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)", block_number = false diff --git a/models/streamline/bronze/beacon/bronze__streamline_beacon_validators.sql b/models/streamline/bronze/beacon/bronze__streamline_beacon_validators.sql index 4efe2533..3ec70c31 100644 --- a/models/streamline/bronze/beacon/bronze__streamline_beacon_validators.sql +++ b/models/streamline/bronze/beacon/bronze__streamline_beacon_validators.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_query( +{{ v0_streamline_external_table_query( model = "beacon_validators_v2", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)", block_number = false diff --git a/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blobs_v2.sql b/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blobs_v2.sql index 70ed86ae..9e5d4f9e 100644 --- a/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blobs_v2.sql +++ b/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blobs_v2.sql @@ -2,7 +2,7 @@ materialized = 'view', tags = ['bronze_beacon_blobs'] ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "beacon_blobs_v2", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", block_number = false diff --git a/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blocks_v1.sql b/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blocks_v1.sql index cc6e92d3..acf34ae1 100644 --- a/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blocks_v1.sql +++ b/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blocks_v1.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "beacon_blocks", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", partition_join_key = "_partition_by_slot_id", diff --git a/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blocks_v2.sql b/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blocks_v2.sql index 71c75ee6..d59e2539 100644 --- a/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blocks_v2.sql +++ b/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_blocks_v2.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "beacon_blocks_v2", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", block_number = false diff --git a/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_validators_v1.sql b/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_validators_v1.sql index a34d83a8..c6fbc1cc 100644 --- a/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_validators_v1.sql +++ b/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_validators_v1.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "beacon_validators", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", partition_join_key = "_partition_by_block_id", diff --git a/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_validators_v2.sql b/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_validators_v2.sql index a3629ba2..29e75aaa 100644 --- a/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_validators_v2.sql +++ b/models/streamline/bronze/beacon/bronze__streamline_fr_beacon_validators_v2.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "beacon_validators_v2", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", block_number = false diff --git a/models/streamline/bronze/core/bronze__blocks.sql b/models/streamline/bronze/core/bronze__blocks.sql new file mode 100644 index 00000000..a75a70db --- /dev/null +++ b/models/streamline/bronze/core/bronze__blocks.sql @@ -0,0 +1,39 @@ +{# Set variables #} +{% set source_name = 'BLOCKS' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = '' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = default_vars['partition_function'] %} +{% set partition_join_key = default_vars['partition_join_key'] %} +{% set balances = default_vars['balances'] %} +{% set block_number = default_vars['block_number'] %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__blocks_fr.sql b/models/streamline/bronze/core/bronze__blocks_fr.sql new file mode 100644 index 00000000..9733a667 --- /dev/null +++ b/models/streamline/bronze/core/bronze__blocks_fr.sql @@ -0,0 +1,31 @@ +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +SELECT + partition_key, + block_number, + VALUE, + CASE + WHEN DATA :result IS NULL THEN DATA + ELSE DATA :result + END AS DATA, + metadata, + file_name, + _inserted_timestamp +FROM + {{ ref('bronze__blocks_fr_v2') }} +{% if var('GLOBAL_USES_STREAMLINE_V1', false) %} +UNION ALL +SELECT + _partition_by_block_id AS partition_key, + block_number, + VALUE, + DATA :result AS DATA, + metadata, + file_name, + _inserted_timestamp +FROM + {{ ref('bronze__blocks_fr_v1') }} +{% endif %} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__blocks_fr_v1.sql b/models/streamline/bronze/core/bronze__blocks_fr_v1.sql new file mode 100644 index 00000000..b45639a6 --- /dev/null +++ b/models/streamline/bronze/core/bronze__blocks_fr_v1.sql @@ -0,0 +1,40 @@ +{# Set variables #} +{% set source_name = 'BLOCKS' %} +{% set source_version = '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)" %} +{% set partition_join_key = '_partition_by_block_id' %} +{% set balances = default_vars['balances'] %} +{% set block_number = false %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core_streamline_v1'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_fr( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + partition_join_key = partition_join_key, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} diff --git a/models/streamline/bronze/core/bronze__blocks_fr_v2.sql b/models/streamline/bronze/core/bronze__blocks_fr_v2.sql new file mode 100644 index 00000000..91576bd6 --- /dev/null +++ b/models/streamline/bronze/core/bronze__blocks_fr_v2.sql @@ -0,0 +1,40 @@ +{# Set variables #} +{% set source_name = 'BLOCKS' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = default_vars['partition_function'] %} +{% set partition_join_key = default_vars['partition_join_key'] %} +{% set balances = default_vars['balances'] %} +{% set block_number = default_vars['block_number'] %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_fr( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + partition_join_key = partition_join_key, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__confirm_blocks.sql b/models/streamline/bronze/core/bronze__confirm_blocks.sql new file mode 100644 index 00000000..031fdb2c --- /dev/null +++ b/models/streamline/bronze/core/bronze__confirm_blocks.sql @@ -0,0 +1,39 @@ +{# Set variables #} +{% set source_name = 'CONFIRM_BLOCKS' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = '' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = default_vars['partition_function'] %} +{% set partition_join_key = default_vars['partition_join_key'] %} +{% set balances = default_vars['balances'] %} +{% set block_number = default_vars['block_number'] %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__confirm_blocks_fr.sql b/models/streamline/bronze/core/bronze__confirm_blocks_fr.sql new file mode 100644 index 00000000..ebf8ac69 --- /dev/null +++ b/models/streamline/bronze/core/bronze__confirm_blocks_fr.sql @@ -0,0 +1,28 @@ +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +SELECT + partition_key, + block_number, + VALUE, + DATA, + metadata, + file_name, + _inserted_timestamp +FROM + {{ ref('bronze__confirm_blocks_fr_v2') }} +{% if var('GLOBAL_USES_STREAMLINE_V1', false) %} +UNION ALL +SELECT + _partition_by_block_id AS partition_key, + block_number, + VALUE, + DATA, + metadata, + file_name, + _inserted_timestamp +FROM + {{ ref('bronze__confirm_blocks_fr_v1') }} +{% endif %} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__confirm_blocks_fr_v1.sql b/models/streamline/bronze/core/bronze__confirm_blocks_fr_v1.sql new file mode 100644 index 00000000..a4402e53 --- /dev/null +++ b/models/streamline/bronze/core/bronze__confirm_blocks_fr_v1.sql @@ -0,0 +1,40 @@ +{# Set variables #} +{% set source_name = 'CONFIRM_BLOCKS' %} +{% set source_version = '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)" %} +{% set partition_join_key = '_partition_by_block_id' %} +{% set balances = default_vars['balances'] %} +{% set block_number = false %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core_streamline_v1'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_fr( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + partition_join_key = partition_join_key, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} diff --git a/models/streamline/bronze/core/bronze__confirm_blocks_fr_v2.sql b/models/streamline/bronze/core/bronze__confirm_blocks_fr_v2.sql new file mode 100644 index 00000000..d03c45b9 --- /dev/null +++ b/models/streamline/bronze/core/bronze__confirm_blocks_fr_v2.sql @@ -0,0 +1,40 @@ +{# Set variables #} +{% set source_name = 'CONFIRM_BLOCKS' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = default_vars['partition_function'] %} +{% set partition_join_key = default_vars['partition_join_key'] %} +{% set balances = default_vars['balances'] %} +{% set block_number = default_vars['block_number'] %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_fr( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + partition_join_key = partition_join_key, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__receipts.sql b/models/streamline/bronze/core/bronze__receipts.sql new file mode 100644 index 00000000..21d84b90 --- /dev/null +++ b/models/streamline/bronze/core/bronze__receipts.sql @@ -0,0 +1,39 @@ +{# Set variables #} +{% set source_name = 'RECEIPTS' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = '' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = default_vars['partition_function'] %} +{% set partition_join_key = default_vars['partition_join_key'] %} +{% set balances = default_vars['balances'] %} +{% set block_number = default_vars['block_number'] %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_receipts'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__receipts_fr.sql b/models/streamline/bronze/core/bronze__receipts_fr.sql new file mode 100644 index 00000000..3b861c06 --- /dev/null +++ b/models/streamline/bronze/core/bronze__receipts_fr.sql @@ -0,0 +1,38 @@ +{{ config ( + materialized = 'view', + tags = ['bronze_receipts'] +) }} + +SELECT + partition_key, + block_number, + array_index, + VALUE, + DATA, + metadata, + file_name, + _inserted_timestamp +FROM + {{ ref('bronze__receipts_fr_v2') }} + + {% if var('GLOBAL_USES_STREAMLINE_V1',false) %} + UNION ALL + SELECT + _partition_by_block_id AS partition_key, + block_number, + COALESCE( + VALUE :"array_index" :: INT, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + VALUE :"data" :"transactionIndex" :: STRING + ) + ) + ) AS array_index, + VALUE, + DATA, + metadata, + file_name, + _inserted_timestamp + FROM + {{ ref('bronze__receipts_fr_v1') }} + {% endif %} diff --git a/models/streamline/bronze/core/bronze__receipts_fr_v1.sql b/models/streamline/bronze/core/bronze__receipts_fr_v1.sql new file mode 100644 index 00000000..6c4956e2 --- /dev/null +++ b/models/streamline/bronze/core/bronze__receipts_fr_v1.sql @@ -0,0 +1,40 @@ +{# Set variables #} +{% set source_name = 'RECEIPTS' %} +{% set source_version = '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)" %} +{% set partition_join_key = '_partition_by_block_id' %} +{% set balances = default_vars['balances'] %} +{% set block_number = false %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core_streamline_v1','bronze_receipts'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_fr( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + partition_join_key = partition_join_key, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__receipts_fr_v2.sql b/models/streamline/bronze/core/bronze__receipts_fr_v2.sql new file mode 100644 index 00000000..90e0a071 --- /dev/null +++ b/models/streamline/bronze/core/bronze__receipts_fr_v2.sql @@ -0,0 +1,40 @@ +{# Set variables #} +{% set source_name = 'RECEIPTS' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = default_vars['partition_function'] %} +{% set partition_join_key = default_vars['partition_join_key'] %} +{% set balances = default_vars['balances'] %} +{% set block_number = default_vars['block_number'] %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_receipts'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_fr( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + partition_join_key = partition_join_key, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__streamline_blocks.sql b/models/streamline/bronze/core/bronze__streamline_blocks.sql deleted file mode 100644 index 84d9df83..00000000 --- a/models/streamline/bronze/core/bronze__streamline_blocks.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_query( - model = "blocks_v2", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)" -) }} diff --git a/models/streamline/bronze/core/bronze__streamline_confirmed_blocks.sql b/models/streamline/bronze/core/bronze__streamline_confirmed_blocks.sql deleted file mode 100644 index b25f1ea9..00000000 --- a/models/streamline/bronze/core/bronze__streamline_confirmed_blocks.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_query( - model = "confirm_blocks_v2", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)" -) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__streamline_receipts.sql b/models/streamline/bronze/core/bronze__streamline_receipts.sql deleted file mode 100644 index 6e86ac4c..00000000 --- a/models/streamline/bronze/core/bronze__streamline_receipts.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_query( - model = "receipts_v2", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)" -) }} diff --git a/models/streamline/bronze/core/bronze__streamline_traces.sql b/models/streamline/bronze/core/bronze__streamline_traces.sql deleted file mode 100644 index 01365fad..00000000 --- a/models/streamline/bronze/core/bronze__streamline_traces.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_query( - model = "traces_v2", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)" -) }} diff --git a/models/streamline/bronze/core/bronze__streamline_transactions.sql b/models/streamline/bronze/core/bronze__streamline_transactions.sql deleted file mode 100644 index 05123157..00000000 --- a/models/streamline/bronze/core/bronze__streamline_transactions.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_query( - model = "transactions_v2", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)" -) }} diff --git a/models/streamline/bronze/core/bronze__traces.sql b/models/streamline/bronze/core/bronze__traces.sql new file mode 100644 index 00000000..f8661762 --- /dev/null +++ b/models/streamline/bronze/core/bronze__traces.sql @@ -0,0 +1,39 @@ +{# Set variables #} +{% set source_name = 'TRACES' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = '' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = default_vars['partition_function'] %} +{% set partition_join_key = default_vars['partition_join_key'] %} +{% set balances = default_vars['balances'] %} +{% set block_number = default_vars['block_number'] %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__traces_fr.sql b/models/streamline/bronze/core/bronze__traces_fr.sql new file mode 100644 index 00000000..b150c368 --- /dev/null +++ b/models/streamline/bronze/core/bronze__traces_fr.sql @@ -0,0 +1,30 @@ +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +SELECT + partition_key, + block_number, + array_index, + VALUE, + DATA, + metadata, + file_name, + _inserted_timestamp +FROM + {{ ref('bronze__traces_fr_v2') }} +{% if var('GLOBAL_USES_STREAMLINE_V1', false) %} +UNION ALL +SELECT + _partition_by_block_id AS partition_key, + block_number, + VALUE :"array_index" :: INT AS array_index, + VALUE, + DATA, + metadata, + file_name, + _inserted_timestamp +FROM + {{ ref('bronze__traces_fr_v1') }} +{% endif %} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__traces_fr_v1.sql b/models/streamline/bronze/core/bronze__traces_fr_v1.sql new file mode 100644 index 00000000..3905c1b8 --- /dev/null +++ b/models/streamline/bronze/core/bronze__traces_fr_v1.sql @@ -0,0 +1,40 @@ +{# Set variables #} +{% set source_name = 'DEBUG_TRACEBLOCKBYNUMBER' if var('GLOBAL_USES_SINGLE_FLIGHT_METHOD',false) else 'TRACES' %} +{% set source_version = '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)" %} +{% set partition_join_key = '_partition_by_block_id' %} +{% set balances = default_vars['balances'] %} +{% set block_number = false %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core_streamline_v1'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_fr( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + partition_join_key = partition_join_key, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} diff --git a/models/streamline/bronze/core/bronze__traces_fr_v2.sql b/models/streamline/bronze/core/bronze__traces_fr_v2.sql new file mode 100644 index 00000000..04158254 --- /dev/null +++ b/models/streamline/bronze/core/bronze__traces_fr_v2.sql @@ -0,0 +1,40 @@ +{# Set variables #} +{% set source_name = 'TRACES' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = default_vars['partition_function'] %} +{% set partition_join_key = default_vars['partition_join_key'] %} +{% set balances = default_vars['balances'] %} +{% set block_number = default_vars['block_number'] %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_fr( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + partition_join_key = partition_join_key, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__transactions.sql b/models/streamline/bronze/core/bronze__transactions.sql new file mode 100644 index 00000000..69cf41d9 --- /dev/null +++ b/models/streamline/bronze/core/bronze__transactions.sql @@ -0,0 +1,39 @@ +{# Set variables #} +{% set source_name = 'TRANSACTIONS' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = '' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = default_vars['partition_function'] %} +{% set partition_join_key = default_vars['partition_join_key'] %} +{% set balances = default_vars['balances'] %} +{% set block_number = default_vars['block_number'] %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__transactions_fr.sql b/models/streamline/bronze/core/bronze__transactions_fr.sql new file mode 100644 index 00000000..6bca77b6 --- /dev/null +++ b/models/streamline/bronze/core/bronze__transactions_fr.sql @@ -0,0 +1,28 @@ +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +SELECT + partition_key, + block_number, + VALUE, + DATA, + metadata, + file_name, + _inserted_timestamp +FROM + {{ ref('bronze__transactions_fr_v2') }} +{% if var('GLOBAL_USES_STREAMLINE_V1', false) %} +UNION ALL +SELECT + _partition_by_block_id AS partition_key, + block_number, + VALUE, + DATA, + metadata, + file_name, + _inserted_timestamp +FROM + {{ ref('bronze__transactions_fr_v1') }} +{% endif %} \ No newline at end of file diff --git a/models/streamline/bronze/core/bronze__transactions_fr_v1.sql b/models/streamline/bronze/core/bronze__transactions_fr_v1.sql new file mode 100644 index 00000000..41179ac7 --- /dev/null +++ b/models/streamline/bronze/core/bronze__transactions_fr_v1.sql @@ -0,0 +1,40 @@ +{# Set variables #} +{% set source_name = 'TRANSACTIONS' %} +{% set source_version = '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)" %} +{% set partition_join_key = '_partition_by_block_id' %} +{% set balances = default_vars['balances'] %} +{% set block_number = false %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core_streamline_v1'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_fr( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + partition_join_key = partition_join_key, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} diff --git a/models/streamline/bronze/core/bronze__transactions_fr_v2.sql b/models/streamline/bronze/core/bronze__transactions_fr_v2.sql new file mode 100644 index 00000000..7907074d --- /dev/null +++ b/models/streamline/bronze/core/bronze__transactions_fr_v2.sql @@ -0,0 +1,40 @@ +{# Set variables #} +{% set source_name = 'TRANSACTIONS' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{% set partition_function = default_vars['partition_function'] %} +{% set partition_join_key = default_vars['partition_join_key'] %} +{% set balances = default_vars['balances'] %} +{% set block_number = default_vars['block_number'] %} +{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %} + +{# Log configuration details #} +{{ log_bronze_details( + source_name = source_name, + source_version = source_version, + model_type = model_type, + partition_function = partition_function, + partition_join_key = partition_join_key, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_core'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_fr( + source_name = source_name.lower(), + source_version = source_version.lower(), + partition_function = partition_function, + partition_join_key = partition_join_key, + balances = balances, + block_number = block_number, + uses_receipts_by_hash = uses_receipts_by_hash +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_blocks.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_blocks.sql deleted file mode 100644 index c54ce415..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_blocks.sql +++ /dev/null @@ -1,6 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_union_query( - model = "blocks" -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_blocks_v1.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_blocks_v1.sql deleted file mode 100644 index b4fc8426..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_blocks_v1.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query( - model = "blocks", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )", - partition_join_key = "_partition_by_block_id", - block_number = false -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_blocks_v2.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_blocks_v2.sql deleted file mode 100644 index 40b7eb09..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_blocks_v2.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query( - model = "blocks_v2", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks.sql deleted file mode 100644 index 7b742f0c..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks.sql +++ /dev/null @@ -1,6 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_union_query( - model = "confirmed_blocks" -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks_v1.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks_v1.sql deleted file mode 100644 index f6c19d44..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks_v1.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query( - model = "confirm_blocks", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )", - partition_join_key = "_partition_by_block_id", - block_number = false -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks_v2.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks_v2.sql deleted file mode 100644 index e4135c70..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_confirmed_blocks_v2.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query( - model = "confirm_blocks_v2", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" -) }} - diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_receipts.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_receipts.sql deleted file mode 100644 index 5285d9b6..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_receipts.sql +++ /dev/null @@ -1,6 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_union_query( - model = "receipts" -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_receipts_v1.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_receipts_v1.sql deleted file mode 100644 index 43ac3384..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_receipts_v1.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query( - model = "receipts", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )", - partition_join_key = "_partition_by_block_id", - block_number = false -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_receipts_v2.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_receipts_v2.sql deleted file mode 100644 index 45be9b2d..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_receipts_v2.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query( - model = "receipts_v2", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_traces.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_traces.sql deleted file mode 100644 index 7b77c312..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_traces.sql +++ /dev/null @@ -1,6 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_union_query( - model = "traces" -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_traces_v1.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_traces_v1.sql deleted file mode 100644 index b73afe31..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_traces_v1.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query( - model = "traces", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )", - partition_join_key = "_partition_by_block_id", - block_number = false -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_traces_v2.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_traces_v2.sql deleted file mode 100644 index 791a9c2c..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_traces_v2.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query( - model = "traces_v2", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" -) }} - diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_transactions.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_transactions.sql deleted file mode 100644 index 681d010e..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_transactions.sql +++ /dev/null @@ -1,6 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_union_query( - model = "transactions" -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_transactions_v1.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_transactions_v1.sql deleted file mode 100644 index f5cb11af..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_transactions_v1.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query( - model = "transactions", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )", - partition_join_key = "_partition_by_block_id", - block_number = false -) }} diff --git a/models/streamline/bronze/core/fr/bronze__streamline_fr_transactions_v2.sql b/models/streamline/bronze/core/fr/bronze__streamline_fr_transactions_v2.sql deleted file mode 100644 index bfe69f33..00000000 --- a/models/streamline/bronze/core/fr/bronze__streamline_fr_transactions_v2.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query( - model = "transactions_v2", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" -) }} - diff --git a/models/streamline/bronze/decoder/bronze__streamline_decoded_logs.sql b/models/streamline/bronze/decoder/bronze__streamline_decoded_logs.sql deleted file mode 100644 index bda391ae..00000000 --- a/models/streamline/bronze/decoder/bronze__streamline_decoded_logs.sql +++ /dev/null @@ -1,6 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_query_decoder( - model = "decoded_logs_v2" -) }} diff --git a/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs.sql b/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs.sql deleted file mode 100644 index ed3fa55f..00000000 --- a/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config ( - materialized = 'view' -) }} - -SELECT - * -FROM - {{ ref('bronze__streamline_fr_decoded_logs_v2') }} -UNION ALL -SELECT - * -FROM - {{ ref('bronze__streamline_fr_decoded_logs_v1') }} diff --git a/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs_v1.sql b/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs_v1.sql deleted file mode 100644 index 5066c8b7..00000000 --- a/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs_v1.sql +++ /dev/null @@ -1,6 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query_decoder( - model = "decoded_logs" -) }} diff --git a/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs_v2.sql b/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs_v2.sql deleted file mode 100644 index 98e22ce4..00000000 --- a/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_logs_v2.sql +++ /dev/null @@ -1,6 +0,0 @@ -{{ config ( - materialized = 'view' -) }} -{{ fsc_evm.streamline_external_table_fr_query_decoder( - model = "decoded_logs_v2" -) }} diff --git a/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs.sql b/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs.sql new file mode 100644 index 00000000..8a339c61 --- /dev/null +++ b/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs.sql @@ -0,0 +1,23 @@ +{# Set variables #} +{% set source_name = 'DECODED_LOGS' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = '' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{# Log configuration details #} +{{ log_model_details( + vars = default_vars +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_decoded_logs'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_decoder( + source_name = source_name.lower(), + source_version = source_version.lower() +) }} \ No newline at end of file diff --git a/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr.sql b/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr.sql new file mode 100644 index 00000000..bbe59557 --- /dev/null +++ b/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr.sql @@ -0,0 +1,20 @@ +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_decoded_logs'] +) }} + +SELECT + * +FROM + {{ ref('bronze__decoded_logs_fr_v2') }} +{% if var('GLOBAL_USES_STREAMLINE_V1', false) %} +UNION ALL +SELECT + * +FROM + {{ ref('bronze__decoded_logs_fr_v1') }} +{% endif %} diff --git a/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr_v1.sql b/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr_v1.sql new file mode 100644 index 00000000..8c122fa9 --- /dev/null +++ b/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr_v1.sql @@ -0,0 +1,23 @@ +{# Set variables #} +{% set source_name = 'DECODED_LOGS' %} +{% set source_version = '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{# Log configuration details #} +{{ log_model_details( + vars = default_vars +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_decoded_logs_streamline_v1'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_decoder_fr( + source_name = source_name.lower(), + source_version = source_version.lower() +) }} \ No newline at end of file diff --git a/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr_v2.sql b/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr_v2.sql new file mode 100644 index 00000000..2bd430a1 --- /dev/null +++ b/models/streamline/bronze/decoder/decoded_logs/bronze__decoded_logs_fr_v2.sql @@ -0,0 +1,23 @@ +{# Set variables #} +{% set source_name = 'DECODED_LOGS' %} +{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %} +{% set model_type = 'FR' %} + +{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%} + +{# Log configuration details #} +{{ log_model_details( + vars = default_vars +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze_decoded_logs'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_decoder_fr( + source_name = source_name.lower(), + source_version = source_version.lower() +) }} \ No newline at end of file diff --git a/models/streamline/bronze/decoder/bronze__streamline_decoded_traces.sql b/models/streamline/bronze/decoder/decoded_traces/bronze__streamline_decoded_traces.sql similarity index 60% rename from models/streamline/bronze/decoder/bronze__streamline_decoded_traces.sql rename to models/streamline/bronze/decoder/decoded_traces/bronze__streamline_decoded_traces.sql index 0042e430..7a4abe9c 100644 --- a/models/streamline/bronze/decoder/bronze__streamline_decoded_traces.sql +++ b/models/streamline/bronze/decoder/decoded_traces/bronze__streamline_decoded_traces.sql @@ -1,6 +1,6 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_query_decoder( +{{ v0_streamline_external_table_query_decoder( model = "decoded_traces_v2" ) }} diff --git a/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_traces.sql b/models/streamline/bronze/decoder/decoded_traces/bronze__streamline_fr_decoded_traces.sql similarity index 100% rename from models/streamline/bronze/decoder/bronze__streamline_fr_decoded_traces.sql rename to models/streamline/bronze/decoder/decoded_traces/bronze__streamline_fr_decoded_traces.sql diff --git a/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_traces_v1.sql b/models/streamline/bronze/decoder/decoded_traces/bronze__streamline_fr_decoded_traces_v1.sql similarity index 58% rename from models/streamline/bronze/decoder/bronze__streamline_fr_decoded_traces_v1.sql rename to models/streamline/bronze/decoder/decoded_traces/bronze__streamline_fr_decoded_traces_v1.sql index 1ed69e6b..0264eeda 100644 --- a/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_traces_v1.sql +++ b/models/streamline/bronze/decoder/decoded_traces/bronze__streamline_fr_decoded_traces_v1.sql @@ -1,6 +1,6 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query_decoder( +{{ v0_streamline_external_table_fr_query_decoder( model = "decoded_traces" ) }} diff --git a/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_traces_v2.sql b/models/streamline/bronze/decoder/decoded_traces/bronze__streamline_fr_decoded_traces_v2.sql similarity index 59% rename from models/streamline/bronze/decoder/bronze__streamline_fr_decoded_traces_v2.sql rename to models/streamline/bronze/decoder/decoded_traces/bronze__streamline_fr_decoded_traces_v2.sql index b54c6657..9d5ebbc7 100644 --- a/models/streamline/bronze/decoder/bronze__streamline_fr_decoded_traces_v2.sql +++ b/models/streamline/bronze/decoder/decoded_traces/bronze__streamline_fr_decoded_traces_v2.sql @@ -1,6 +1,6 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query_decoder( +{{ v0_streamline_external_table_fr_query_decoder( model = "decoded_traces_v2" ) }} diff --git a/models/streamline/bronze/reads/bronze__streamline_fr_reads_v1.sql b/models/streamline/bronze/reads/bronze__streamline_fr_reads_v1.sql index e94a0ad9..2dec3eef 100644 --- a/models/streamline/bronze/reads/bronze__streamline_fr_reads_v1.sql +++ b/models/streamline/bronze/reads/bronze__streamline_fr_reads_v1.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "reads", partition_function = "TO_DATE(concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)))", partition_join_key = "_partition_by_modified_date", diff --git a/models/streamline/bronze/reads/bronze__streamline_fr_reads_v2.sql b/models/streamline/bronze/reads/bronze__streamline_fr_reads_v2.sql index c1679d7a..a94cb298 100644 --- a/models/streamline/bronze/reads/bronze__streamline_fr_reads_v2.sql +++ b/models/streamline/bronze/reads/bronze__streamline_fr_reads_v2.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_fr_query( +{{ v0_streamline_external_table_fr_query( model = "reads_v2", partition_function = "TO_DATE(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1))" ) }} diff --git a/models/streamline/bronze/reads/bronze__streamline_reads.sql b/models/streamline/bronze/reads/bronze__streamline_reads.sql index d62e788a..17b8f979 100644 --- a/models/streamline/bronze/reads/bronze__streamline_reads.sql +++ b/models/streamline/bronze/reads/bronze__streamline_reads.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ fsc_evm.streamline_external_table_query( +{{ v0_streamline_external_table_query( model = "reads_v2", partition_function = "TO_DATE(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1))" ) }} diff --git a/models/streamline/silver/_24_hour_lookback.sql b/models/streamline/silver/_24_hour_lookback.sql index cc44e860..e9d060f8 100644 --- a/models/streamline/silver/_24_hour_lookback.sql +++ b/models/streamline/silver/_24_hour_lookback.sql @@ -1,4 +1,26 @@ {{ config ( materialized = "ephemeral" ) }} -{{ fsc_evm.block_lookback_24_hour() }} + +WITH max_time AS ( + + SELECT + MAX(block_timestamp) AS max_timestamp + FROM + {{ ref("silver__blocks") }} +) +SELECT + MIN(block_number) AS block_number +FROM + {{ ref("silver__blocks") }} + JOIN max_time + ON block_timestamp BETWEEN DATEADD( + 'hour', + -25, + max_timestamp + ) + AND DATEADD( + 'hour', + -24, + max_timestamp + ) diff --git a/models/streamline/silver/_block_lookback.sql b/models/streamline/silver/_block_lookback.sql index 450bba2e..f37bcf61 100644 --- a/models/streamline/silver/_block_lookback.sql +++ b/models/streamline/silver/_block_lookback.sql @@ -1,4 +1,11 @@ {{ config ( materialized = "ephemeral" ) }} -{{ fsc_evm.block_lookback_72_hour() }} + +SELECT + MIN(block_number) AS block_number +FROM + {{ ref("silver__blocks") }} +WHERE + block_timestamp >= DATEADD('hour', -72, TRUNCATE(SYSDATE(), 'HOUR')) + AND block_timestamp < DATEADD('hour', -71, TRUNCATE(SYSDATE(), 'HOUR')) diff --git a/models/streamline/silver/_block_ranges.sql b/models/streamline/silver/_block_ranges.sql index 411dcc96..0f5f0b68 100644 --- a/models/streamline/silver/_block_ranges.sql +++ b/models/streamline/silver/_block_ranges.sql @@ -2,4 +2,54 @@ materialized = "ephemeral", unique_key = "block_number", ) }} -{{ fsc_evm.block_ranges() }} + +SELECT + block_number, + CASE + WHEN RIGHT( + block_number, + 1 + ) = 0 THEN block_number + END AS block_number_10, + CASE + WHEN RIGHT( + block_number, + 2 + ) IN ( + 00, + 25, + 50, + 75 + ) THEN block_number + END AS block_number_25, + CASE + WHEN RIGHT( + block_number, + 2 + ) IN ( + 00, + 50 + ) THEN block_number + END AS block_number_50, + CASE + WHEN RIGHT( + block_number, + 2 + ) IN (00) THEN block_number + END AS block_number_100, + CASE + WHEN RIGHT( + block_number, + 3 + ) IN (000) THEN block_number + END AS block_number_1000, + CASE + WHEN RIGHT( + block_number, + 4 + ) IN (0000) THEN block_number + END AS block_number_10000, + block_timestamp, + TO_TIMESTAMP_NTZ(_inserted_timestamp) AS _inserted_timestamp +FROM + {{ ref("silver__blocks") }} diff --git a/models/streamline/silver/_max_block_by_date.sql b/models/streamline/silver/_max_block_by_date.sql index d7730aa2..a56cf82d 100644 --- a/models/streamline/silver/_max_block_by_date.sql +++ b/models/streamline/silver/_max_block_by_date.sql @@ -2,4 +2,26 @@ materialized = "ephemeral", unique_key = "block_number", ) }} -{{ fsc_evm.max_block_by_date() }} + +WITH base AS ( + + SELECT + block_timestamp :: DATE AS block_date, + MAX(block_number) block_number + FROM + {{ ref("silver__blocks") }} + GROUP BY + block_timestamp :: DATE +) +SELECT + block_date, + block_number +FROM + base +WHERE + block_date <> ( + SELECT + MAX(block_date) + FROM + base + ) \ No newline at end of file diff --git a/models/streamline/silver/_max_block_by_hour.sql b/models/streamline/silver/_max_block_by_hour.sql index 7193cc68..80e3d972 100644 --- a/models/streamline/silver/_max_block_by_hour.sql +++ b/models/streamline/silver/_max_block_by_hour.sql @@ -1,4 +1,37 @@ {{ config ( materialized = "ephemeral" ) }} -{{ fsc_evm.max_block_by_hour() }} + +WITH base AS ( + + SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_hour, + MAX(block_number) block_number + FROM + {{ ref("silver__blocks") }} + WHERE + block_timestamp > DATEADD( + 'day', + -5, + CURRENT_DATE + ) + GROUP BY + 1 +) +SELECT + block_hour, + block_number +FROM + base +WHERE + block_hour <> ( + SELECT + MAX( + block_hour + ) + FROM + base + ) diff --git a/models/streamline/silver/core/complete/streamline__blocks_complete.sql b/models/streamline/silver/core/complete/streamline__blocks_complete.sql new file mode 100644 index 00000000..4c8977d8 --- /dev/null +++ b/models/streamline/silver/core/complete/streamline__blocks_complete.sql @@ -0,0 +1,50 @@ +{# Set variables #} +{%- set source_name = 'BLOCKS' -%} +{%- set model_type = 'COMPLETE' -%} + +{%- set full_refresh_type = var((source_name ~ '_complete_full_refresh').upper(), false) -%} + +{% set post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)" %} + +{# Log configuration details #} +{{ log_complete_details( + post_hook = post_hook, + full_refresh_type = full_refresh_type +) }} + +{# Set up dbt configuration #} +-- depends_on: {{ ref('bronze__' ~ source_name.lower()) }} + +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = post_hook, + full_refresh = full_refresh_type, + tags = ['streamline_core_complete'] +) }} + +{# Main query starts here #} +SELECT + block_number, + file_name, + {{ dbt_utils.generate_surrogate_key(['block_number']) }} AS complete_{{ source_name.lower() }}_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {% if is_incremental() %} + {{ ref('bronze__' ~ source_name.lower()) }} + WHERE + _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1970-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM + {{ this }} + ) + {% else %} + {{ ref('bronze__' ~ source_name.lower() ~ '_fr') }} + {% endif %} + +QUALIFY (ROW_NUMBER() OVER (PARTITION BY block_number ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/core/complete/streamline__complete_blocks.sql b/models/streamline/silver/core/complete/streamline__complete_blocks.sql deleted file mode 100644 index d1f30200..00000000 --- a/models/streamline/silver/core/complete/streamline__complete_blocks.sql +++ /dev/null @@ -1,11 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_blocks') }} -{{ config ( - materialized = "incremental", - unique_key = "block_number", - cluster_by = "ROUND(block_number, -3)", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", - tags = ['streamline_core_complete'] -) }} -{{ fsc_evm.streamline_core_complete( - model = 'blocks' -) }} diff --git a/models/streamline/silver/core/complete/streamline__complete_confirmed_blocks.sql b/models/streamline/silver/core/complete/streamline__complete_confirmed_blocks.sql deleted file mode 100644 index dc85023b..00000000 --- a/models/streamline/silver/core/complete/streamline__complete_confirmed_blocks.sql +++ /dev/null @@ -1,10 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_confirmed_blocks') }} -{{ config ( - materialized = "incremental", - unique_key = "block_number", - cluster_by = "ROUND(block_number, -3)", - tags = ['streamline_core_complete'] -) }} -{{ fsc_evm.streamline_core_complete( - model = 'confirmed_blocks' -) }} diff --git a/models/streamline/silver/core/complete/streamline__complete_receipts.sql b/models/streamline/silver/core/complete/streamline__complete_receipts.sql deleted file mode 100644 index cb8bcf70..00000000 --- a/models/streamline/silver/core/complete/streamline__complete_receipts.sql +++ /dev/null @@ -1,11 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_receipts') }} -{{ config ( - materialized = "incremental", - unique_key = "block_number", - cluster_by = "ROUND(block_number, -3)", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", - tags = ['streamline_core_complete'] -) }} -{{ fsc_evm.streamline_core_complete( - model = 'receipts' -) }} diff --git a/models/streamline/silver/core/complete/streamline__complete_traces.sql b/models/streamline/silver/core/complete/streamline__complete_traces.sql deleted file mode 100644 index b1b1ee40..00000000 --- a/models/streamline/silver/core/complete/streamline__complete_traces.sql +++ /dev/null @@ -1,11 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_traces') }} -{{ config ( - materialized = "incremental", - unique_key = "block_number", - cluster_by = "ROUND(block_number, -3)", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", - tags = ['streamline_core_complete'] -) }} -{{ fsc_evm.streamline_core_complete( - model = 'traces' -) }} \ No newline at end of file diff --git a/models/streamline/silver/core/complete/streamline__complete_transactions.sql b/models/streamline/silver/core/complete/streamline__complete_transactions.sql deleted file mode 100644 index d55c63c3..00000000 --- a/models/streamline/silver/core/complete/streamline__complete_transactions.sql +++ /dev/null @@ -1,11 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_transactions') }} -{{ config ( - materialized = "incremental", - unique_key = "block_number", - cluster_by = "ROUND(block_number, -3)", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", - tags = ['streamline_core_complete'] -) }} -{{ fsc_evm.streamline_core_complete( - model = 'transactions' -) }} \ No newline at end of file diff --git a/models/streamline/silver/core/complete/streamline__confirm_blocks_complete.sql b/models/streamline/silver/core/complete/streamline__confirm_blocks_complete.sql new file mode 100644 index 00000000..fe73ef27 --- /dev/null +++ b/models/streamline/silver/core/complete/streamline__confirm_blocks_complete.sql @@ -0,0 +1,50 @@ +{# Set variables #} +{%- set source_name = 'CONFIRM_BLOCKS' -%} +{%- set model_type = 'COMPLETE' -%} + +{%- set full_refresh_type = var((source_name ~ '_complete_full_refresh').upper(), false) -%} + +{% set post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)" %} + +{# Log configuration details #} +{{ log_complete_details( + post_hook = post_hook, + full_refresh_type = full_refresh_type +) }} + +{# Set up dbt configuration #} +-- depends_on: {{ ref('bronze__' ~ source_name.lower()) }} + +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = post_hook, + full_refresh = full_refresh_type, + tags = ['streamline_core_complete_confirm_blocks'] +) }} + +{# Main query starts here #} +SELECT + block_number, + file_name, + {{ dbt_utils.generate_surrogate_key(['block_number']) }} AS complete_{{ source_name.lower() }}_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {% if is_incremental() %} + {{ ref('bronze__' ~ source_name.lower()) }} + WHERE + _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1970-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM + {{ this }} + ) + {% else %} + {{ ref('bronze__' ~ source_name.lower() ~ '_fr') }} + {% endif %} + +QUALIFY (ROW_NUMBER() OVER (PARTITION BY block_number ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/core/complete/streamline__receipts_complete.sql b/models/streamline/silver/core/complete/streamline__receipts_complete.sql new file mode 100644 index 00000000..b77138da --- /dev/null +++ b/models/streamline/silver/core/complete/streamline__receipts_complete.sql @@ -0,0 +1,50 @@ +{# Set variables #} +{%- set source_name = 'RECEIPTS' -%} +{%- set model_type = 'COMPLETE' -%} + +{%- set full_refresh_type = var((source_name ~ '_complete_full_refresh').upper(), false) -%} + +{% set post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)" %} + +{# Log configuration details #} +{{ log_complete_details( + post_hook = post_hook, + full_refresh_type = full_refresh_type +) }} + +{# Set up dbt configuration #} +-- depends_on: {{ ref('bronze__' ~ source_name.lower()) }} + +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = post_hook, + full_refresh = full_refresh_type, + tags = ['streamline_core_complete_receipts'] +) }} + +{# Main query starts here #} +SELECT + block_number, + file_name, + {{ dbt_utils.generate_surrogate_key(['block_number']) }} AS complete_{{ source_name.lower() }}_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {% if is_incremental() %} + {{ ref('bronze__' ~ source_name.lower()) }} + WHERE + _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1970-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM + {{ this }} + ) + {% else %} + {{ ref('bronze__' ~ source_name.lower() ~ '_fr') }} + {% endif %} + +QUALIFY (ROW_NUMBER() OVER (PARTITION BY block_number ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/core/complete/streamline__traces_complete.sql b/models/streamline/silver/core/complete/streamline__traces_complete.sql new file mode 100644 index 00000000..e7158769 --- /dev/null +++ b/models/streamline/silver/core/complete/streamline__traces_complete.sql @@ -0,0 +1,50 @@ +{# Set variables #} +{%- set source_name = 'TRACES' -%} +{%- set model_type = 'COMPLETE' -%} + +{%- set full_refresh_type = var((source_name ~ '_complete_full_refresh').upper(), false) -%} + +{% set post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)" %} + +{# Log configuration details #} +{{ log_complete_details( + post_hook = post_hook, + full_refresh_type = full_refresh_type +) }} + +{# Set up dbt configuration #} +-- depends_on: {{ ref('bronze__' ~ source_name.lower()) }} + +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = post_hook, + full_refresh = full_refresh_type, + tags = ['streamline_core_complete'] +) }} + +{# Main query starts here #} +SELECT + block_number, + file_name, + {{ dbt_utils.generate_surrogate_key(['block_number']) }} AS complete_{{ source_name.lower() }}_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {% if is_incremental() %} + {{ ref('bronze__' ~ source_name.lower()) }} + WHERE + _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1970-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM + {{ this }} + ) + {% else %} + {{ ref('bronze__' ~ source_name.lower() ~ '_fr') }} + {% endif %} + +QUALIFY (ROW_NUMBER() OVER (PARTITION BY block_number ORDER BY _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/streamline/silver/core/complete/streamline__transactions_complete.sql b/models/streamline/silver/core/complete/streamline__transactions_complete.sql new file mode 100644 index 00000000..06ec7f76 --- /dev/null +++ b/models/streamline/silver/core/complete/streamline__transactions_complete.sql @@ -0,0 +1,50 @@ +{# Set variables #} +{%- set source_name = 'TRANSACTIONS' -%} +{%- set model_type = 'COMPLETE' -%} + +{%- set full_refresh_type = var((source_name ~ '_complete_full_refresh').upper(), false) -%} + +{% set post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)"%} + +{# Log configuration details #} +{{ log_complete_details( + post_hook = post_hook, + full_refresh_type = full_refresh_type +) }} + +{# Set up dbt configuration #} +-- depends_on: {{ ref('bronze__' ~ source_name.lower()) }} + +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = post_hook, + full_refresh = full_refresh_type, + tags = ['streamline_core_complete'] +) }} + +{# Main query starts here #} +SELECT + block_number, + file_name, + {{ dbt_utils.generate_surrogate_key(['block_number']) }} AS complete_{{ source_name.lower() }}_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {% if is_incremental() %} + {{ ref('bronze__' ~ source_name.lower()) }} + WHERE + _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1970-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM + {{ this }} + ) + {% else %} + {{ ref('bronze__' ~ source_name.lower() ~ '_fr') }} + {% endif %} + +QUALIFY (ROW_NUMBER() OVER (PARTITION BY block_number ORDER BY _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/streamline/silver/core/history/streamline__blocks_transactions_history.sql b/models/streamline/silver/core/history/streamline__blocks_transactions_history.sql index 333332e8..baed1d72 100644 --- a/models/streamline/silver/core/history/streamline__blocks_transactions_history.sql +++ b/models/streamline/silver/core/history/streamline__blocks_transactions_history.sql @@ -1,20 +1,112 @@ +{# Set variables #} +{%- set model_name = 'BLOCKS_TRANSACTIONS' -%} +{%- set model_type = 'HISTORY' -%} + +{%- set default_vars = set_default_variables_streamline(model_name, model_type) -%} + +{# Set up parameters for the streamline process. These will come from the vars set in dbt_project.yml #} + +{%- set streamline_params = set_streamline_parameters( + model_name=model_name, + model_type=model_type +) -%} + +{%- set node_url = default_vars['node_url'] -%} +{%- set node_secret_path = default_vars['node_secret_path'] -%} +{%- set model_quantum_state = default_vars['model_quantum_state'] -%} +{%- set sql_limit = streamline_params['sql_limit'] -%} +{%- set testing_limit = default_vars['testing_limit'] -%} +{%- set order_by_clause = default_vars['order_by_clause'] -%} +{%- set new_build = default_vars['new_build'] -%} +{%- set method_params = streamline_params['method_params'] -%} +{%- set method = streamline_params['method'] -%} + +{# Log configuration details #} +{{ log_streamline_details( + model_name=model_name, + model_type=model_type, + node_url=node_url, + model_quantum_state=model_quantum_state, + sql_limit=sql_limit, + testing_limit=testing_limit, + order_by_clause=order_by_clause, + new_build=new_build, + streamline_params=streamline_params, + method_params=method_params, + method=method +) }} + +{# Set up dbt configuration #} {{ config ( materialized = "view", post_hook = fsc_utils.if_data_call_function_v2( func = 'streamline.udf_bulk_rest_api_v2', target = "{{this.schema}}.{{this.identifier}}", - params ={ "external_table" :"blocks_v2", - "sql_limit" :"50000", - "producer_batch_size" :"1000", - "worker_batch_size" :"100", - "sql_source" :"{{this.identifier}}", - "exploded_key": tojson(["data", "result.transactions"]) } + params = streamline_params ), tags = ['streamline_core_history'] ) }} -{{ fsc_evm.streamline_core_requests( - model_type = 'history', - model = 'blocks_transactions', - quantum_state = 'streamline', - vault_secret_path = "vault/prod/ethereum/quicknode/mainnet" -) }} + +{# Main query starts here #} +WITH +{% if not new_build %} + last_3_days AS ( + SELECT block_number + FROM {{ ref("_block_lookback") }} + ), +{% endif %} + +{# Identify blocks that need processing #} +to_do AS ( + SELECT block_number + FROM {{ ref("streamline__blocks") }} + WHERE + block_number IS NOT NULL + {% if not new_build %} + AND block_number <= (SELECT block_number FROM last_3_days) + {% endif %} + + EXCEPT + + SELECT block_number + FROM {{ ref("streamline__blocks_complete") }} b + INNER JOIN {{ ref("streamline__transactions_complete") }} t USING(block_number) + WHERE 1=1 + {% if not new_build %} + AND block_number <= (SELECT block_number FROM last_3_days) + {% endif %} +), +ready_blocks AS ( + SELECT block_number + FROM to_do + + {% if testing_limit is not none %} + LIMIT {{ testing_limit }} + {% endif %} +) + +{# Generate API requests for each block #} +SELECT + block_number, + ROUND(block_number, -3) AS partition_key, + live.udf_api( + 'POST', + '{{ node_url }}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', '{{ model_quantum_state }}' + ), + OBJECT_CONSTRUCT( + 'id', block_number, + 'jsonrpc', '2.0', + 'method', '{{ method }}', + 'params', {{ method_params }} + ), + '{{ node_secret_path }}' + ) AS request +FROM + ready_blocks + +{{ order_by_clause }} + +LIMIT {{ sql_limit }} \ No newline at end of file diff --git a/models/streamline/silver/core/history/streamline__confirm_blocks_history.sql b/models/streamline/silver/core/history/streamline__confirm_blocks_history.sql new file mode 100644 index 00000000..d715b311 --- /dev/null +++ b/models/streamline/silver/core/history/streamline__confirm_blocks_history.sql @@ -0,0 +1,133 @@ +{# Set variables #} +{%- set model_name = 'CONFIRM_BLOCKS' -%} +{%- set model_type = 'HISTORY' -%} + +{%- set default_vars = set_default_variables_streamline(model_name, model_type) -%} + +{# Set up parameters for the streamline process. These will come from the vars set in dbt_project.yml #} +{%- set streamline_params = set_streamline_parameters( + model_name=model_name, + model_type=model_type +) -%} + +{%- set node_url = default_vars['node_url'] -%} +{%- set node_secret_path = default_vars['node_secret_path'] -%} +{%- set model_quantum_state = default_vars['model_quantum_state'] -%} +{%- set sql_limit = streamline_params['sql_limit'] -%} +{%- set testing_limit = default_vars['testing_limit'] -%} +{%- set order_by_clause = default_vars['order_by_clause'] -%} +{%- set new_build = default_vars['new_build'] -%} +{%- set method_params = streamline_params['method_params'] -%} +{%- set method = streamline_params['method'] -%} + +{# Log configuration details #} +{{ log_streamline_details( + model_name=model_name, + model_type=model_type, + node_url=node_url, + model_quantum_state=model_quantum_state, + sql_limit=sql_limit, + testing_limit=testing_limit, + order_by_clause=order_by_clause, + new_build=new_build, + streamline_params=streamline_params, + method_params=method_params, + method=method +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params = streamline_params + ), + tags = ['streamline_core_history_confirm_blocks'] +) }} + +{# Main query starts here #} +WITH +{% if not new_build %} + last_3_days AS ( + SELECT block_number + FROM {{ ref("_block_lookback") }} + ), +{% endif %} + +{# Delay blocks #} +look_back AS ( + SELECT + block_number + FROM + {{ ref("_max_block_by_hour") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 6 + ), + +{# Identify blocks that need processing #} +to_do AS ( + SELECT block_number + FROM {{ ref("streamline__blocks") }} + WHERE + block_number IS NOT NULL + AND block_number <= (SELECT block_number FROM look_back) + {% if not new_build %} + AND block_number <= (SELECT block_number FROM last_3_days) + {% endif %} + + EXCEPT + + {# Exclude blocks that have already been processed #} + SELECT block_number + FROM {{ ref('streamline__' ~ model_name.lower() ~ '_complete') }} + WHERE 1=1 + AND block_number IS NOT NULL + AND block_number <= (SELECT block_number FROM look_back) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) + {% if not new_build %} + AND block_number <= (SELECT block_number FROM last_3_days) + {% endif %} +) + +{# Prepare the final list of blocks to process #} +,ready_blocks AS ( + SELECT block_number + FROM to_do + + {% if testing_limit is not none %} + LIMIT {{ testing_limit }} + {% endif %} +) + +{# Generate API requests for each block #} +SELECT + block_number, + ROUND(block_number, -3) AS partition_key, + live.udf_api( + 'POST', + '{{ node_url }}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', '{{ model_quantum_state }}' + ), + OBJECT_CONSTRUCT( + 'id', block_number, + 'jsonrpc', '2.0', + 'method', '{{ method }}', + 'params', {{ method_params }} + ), + '{{ node_secret_path }}' + ) AS request +FROM + ready_blocks + +{{ order_by_clause }} + +LIMIT {{ sql_limit }} \ No newline at end of file diff --git a/models/streamline/silver/core/history/streamline__receipts_history.sql b/models/streamline/silver/core/history/streamline__receipts_history.sql index b269accc..4cd5823e 100644 --- a/models/streamline/silver/core/history/streamline__receipts_history.sql +++ b/models/streamline/silver/core/history/streamline__receipts_history.sql @@ -1,20 +1,113 @@ +{# Set variables #} +{%- set model_name = 'RECEIPTS' -%} +{%- set model_type = 'HISTORY' -%} + +{%- set default_vars = set_default_variables_streamline(model_name, model_type) -%} + +{# Set up parameters for the streamline process. These will come from the vars set in dbt_project.yml #} +{%- set streamline_params = set_streamline_parameters( + model_name=model_name, + model_type=model_type +) -%} + +{%- set node_url = default_vars['node_url'] -%} +{%- set node_secret_path = default_vars['node_secret_path'] -%} +{%- set model_quantum_state = default_vars['model_quantum_state'] -%} +{%- set sql_limit = streamline_params['sql_limit'] -%} +{%- set testing_limit = default_vars['testing_limit'] -%} +{%- set order_by_clause = default_vars['order_by_clause'] -%} +{%- set new_build = default_vars['new_build'] -%} +{%- set method_params = streamline_params['method_params'] -%} +{%- set method = streamline_params['method'] -%} + +{# Log configuration details #} +{{ log_streamline_details( + model_name=model_name, + model_type=model_type, + node_url=node_url, + model_quantum_state=model_quantum_state, + sql_limit=sql_limit, + testing_limit=testing_limit, + order_by_clause=order_by_clause, + new_build=new_build, + streamline_params=streamline_params, + method_params=method_params, + method=method +) }} + +{# Set up dbt configuration #} {{ config ( materialized = "view", post_hook = fsc_utils.if_data_call_function_v2( func = 'streamline.udf_bulk_rest_api_v2', target = "{{this.schema}}.{{this.identifier}}", - params ={ "external_table" :"receipts_v2", - "sql_limit" :"100000", - "producer_batch_size" :"100", - "worker_batch_size" :"10", - "sql_source" :"{{this.identifier}}", - "exploded_key": tojson(["result"]) } + params = streamline_params ), - tags = ['streamline_core_history'] -) }} -{{ fsc_evm.streamline_core_requests( - model_type = 'history', - model = 'receipts', - quantum_state = 'streamline', - vault_secret_path = "vault/prod/ethereum/quicknode/mainnet" + tags = ['streamline_core_history_receipts'] ) }} + +{# Main query starts here #} +WITH +{% if not new_build %} + last_3_days AS ( + SELECT block_number + FROM {{ ref("_block_lookback") }} + ), +{% endif %} + +{# Identify blocks that need processing #} +to_do AS ( + SELECT block_number + FROM {{ ref("streamline__blocks") }} + WHERE + block_number IS NOT NULL + {% if not new_build %} + AND block_number <= (SELECT block_number FROM last_3_days) + {% endif %} + + EXCEPT + + {# Exclude blocks that have already been processed #} + SELECT block_number + FROM {{ ref('streamline__' ~ model_name.lower() ~ '_complete') }} + WHERE 1=1 + {% if not new_build %} + AND block_number <= (SELECT block_number FROM last_3_days) + {% endif %} +) + +{# Prepare the final list of blocks to process #} +,ready_blocks AS ( + SELECT block_number + FROM to_do + + {% if testing_limit is not none %} + LIMIT {{ testing_limit }} + {% endif %} +) + +{# Generate API requests for each block #} +SELECT + block_number, + ROUND(block_number, -3) AS partition_key, + live.udf_api( + 'POST', + '{{ node_url }}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', '{{ model_quantum_state }}' + ), + OBJECT_CONSTRUCT( + 'id', block_number, + 'jsonrpc', '2.0', + 'method', '{{ method }}', + 'params', {{ method_params }} + ), + '{{ node_secret_path }}' + ) AS request +FROM + ready_blocks + +{{ order_by_clause }} + +LIMIT {{ sql_limit }} \ No newline at end of file diff --git a/models/streamline/silver/core/history/streamline__traces_history.sql b/models/streamline/silver/core/history/streamline__traces_history.sql index 35ffab91..0cf5cc27 100644 --- a/models/streamline/silver/core/history/streamline__traces_history.sql +++ b/models/streamline/silver/core/history/streamline__traces_history.sql @@ -1,20 +1,113 @@ +{# Set variables #} +{%- set model_name = 'TRACES' -%} +{%- set model_type = 'HISTORY' -%} + +{%- set default_vars = set_default_variables_streamline(model_name, model_type) -%} + +{# Set up parameters for the streamline process. These will come from the vars set in dbt_project.yml #} +{%- set streamline_params = set_streamline_parameters( + model_name=model_name, + model_type=model_type +) -%} + +{%- set node_url = default_vars['node_url'] -%} +{%- set node_secret_path = default_vars['node_secret_path'] -%} +{%- set model_quantum_state = default_vars['model_quantum_state'] -%} +{%- set sql_limit = streamline_params['sql_limit'] -%} +{%- set testing_limit = default_vars['testing_limit'] -%} +{%- set order_by_clause = default_vars['order_by_clause'] -%} +{%- set new_build = default_vars['new_build'] -%} +{%- set method_params = streamline_params['method_params'] -%} +{%- set method = streamline_params['method'] -%} + +{# Log configuration details #} +{{ log_streamline_details( + model_name=model_name, + model_type=model_type, + node_url=node_url, + model_quantum_state=model_quantum_state, + sql_limit=sql_limit, + testing_limit=testing_limit, + order_by_clause=order_by_clause, + new_build=new_build, + streamline_params=streamline_params, + method_params=method_params, + method=method +) }} + +{# Set up dbt configuration #} {{ config ( materialized = "view", post_hook = fsc_utils.if_data_call_function_v2( func = 'streamline.udf_bulk_rest_api_v2', target = "{{this.schema}}.{{this.identifier}}", - params ={ "external_table" :"traces_v2", - "sql_limit" :"100000", - "producer_batch_size" :"100", - "worker_batch_size" :"10", - "sql_source" :"{{this.identifier}}", - "exploded_key": tojson(["result"]) } + params = streamline_params ), tags = ['streamline_core_history'] ) }} -{{ fsc_evm.streamline_core_requests( - model_type = 'history', - model = 'traces', - quantum_state = 'streamline', - vault_secret_path = "vault/prod/ethereum/quicknode/mainnet" -) }} + +{# Main query starts here #} +WITH +{% if not new_build %} + last_3_days AS ( + SELECT block_number + FROM {{ ref("_block_lookback") }} + ), +{% endif %} + +{# Identify blocks that need processing #} +to_do AS ( + SELECT block_number + FROM {{ ref("streamline__blocks") }} + WHERE + block_number IS NOT NULL + {% if not new_build %} + AND block_number <= (SELECT block_number FROM last_3_days) + {% endif %} + + EXCEPT + + {# Exclude blocks that have already been processed #} + SELECT block_number + FROM {{ ref('streamline__' ~ model_name.lower() ~ '_complete') }} + WHERE 1=1 + {% if not new_build %} + AND block_number <= (SELECT block_number FROM last_3_days) + {% endif %} +) + +{# Prepare the final list of blocks to process #} +,ready_blocks AS ( + SELECT block_number + FROM to_do + + {% if testing_limit is not none %} + LIMIT {{ testing_limit }} + {% endif %} +) + +{# Generate API requests for each block #} +SELECT + block_number, + ROUND(block_number, -3) AS partition_key, + live.udf_api( + 'POST', + '{{ node_url }}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', '{{ model_quantum_state }}' + ), + OBJECT_CONSTRUCT( + 'id', block_number, + 'jsonrpc', '2.0', + 'method', '{{ method }}', + 'params', {{ method_params }} + ), + '{{ node_secret_path }}' + ) AS request +FROM + ready_blocks + +{{ order_by_clause }} + +LIMIT {{ sql_limit }} \ No newline at end of file diff --git a/models/streamline/silver/core/realtime/streamline__blocks_transactions_realtime.sql b/models/streamline/silver/core/realtime/streamline__blocks_transactions_realtime.sql index 39840366..0bd0a93f 100644 --- a/models/streamline/silver/core/realtime/streamline__blocks_transactions_realtime.sql +++ b/models/streamline/silver/core/realtime/streamline__blocks_transactions_realtime.sql @@ -1,21 +1,126 @@ -{{ config( +{# Set variables #} +{%- set model_name = 'BLOCKS_TRANSACTIONS' -%} +{%- set model_type = 'REALTIME' -%} +{%- set min_block = var('GLOBAL_START_UP_BLOCK', none) -%} + +{%- set default_vars = set_default_variables_streamline(model_name, model_type) -%} + +{# Set up parameters for the streamline process. These will come from the vars set in dbt_project.yml #} +{%- set streamline_params = set_streamline_parameters( + model_name=model_name, + model_type=model_type +) -%} + +{%- set node_url = default_vars['node_url'] -%} +{%- set node_secret_path = default_vars['node_secret_path'] -%} +{%- set model_quantum_state = default_vars['model_quantum_state'] -%} +{%- set sql_limit = streamline_params['sql_limit'] -%} +{%- set testing_limit = default_vars['testing_limit'] -%} +{%- set order_by_clause = default_vars['order_by_clause'] -%} +{%- set new_build = default_vars['new_build'] -%} +{%- set method_params = streamline_params['method_params'] -%} +{%- set method = streamline_params['method'] -%} + +{# Log configuration details #} +{{ log_streamline_details( + model_name=model_name, + model_type=model_type, + node_url=node_url, + model_quantum_state=model_quantum_state, + sql_limit=sql_limit, + testing_limit=testing_limit, + order_by_clause=order_by_clause, + new_build=new_build, + streamline_params=streamline_params, + method_params=method_params, + method=method, + min_block=min_block +) }} + +{# Set up dbt configuration #} +{{ config ( materialized = "view", post_hook = fsc_utils.if_data_call_function_v2( - func = "streamline.udf_bulk_rest_api_v2", - target = "{{ this.schema }}.{{ this.identifier }}", - params ={ "external_table": "blocks_v2", - "sql_limit": "50000", - "producer_batch_size": "1000", - "worker_batch_size": "100", - "sql_source": "{{ this.identifier }}", - "exploded_key": tojson(["data", "result.transactions"]) } + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params = streamline_params ), tags = ['streamline_core_realtime'] ) }} -{{ fsc_evm.streamline_core_requests( - model_type = 'realtime', - model = 'blocks_transactions', - quantum_state = 'streamline', - vault_secret_path = "vault/prod/ethereum/quicknode/mainnet", - query_limit = 300 -) }} + +{# Main query starts here #} +WITH +{% if not new_build %} + last_3_days AS ( + SELECT block_number + FROM {{ ref("_block_lookback") }} + ), +{% endif %} + +{# Identify blocks that need processing #} +to_do AS ( + SELECT block_number + FROM {{ ref("streamline__blocks") }} + WHERE + block_number IS NOT NULL + {% if not new_build %} + AND block_number >= (SELECT block_number FROM last_3_days) + {% endif %} + + {% if min_block is not none %} + AND block_number >= {{ min_block }} + {% endif %} + + EXCEPT + + SELECT block_number + FROM {{ ref("streamline__blocks_complete") }} b + INNER JOIN {{ ref("streamline__transactions_complete") }} t USING(block_number) + WHERE 1=1 + {% if not new_build %} + AND block_number >= (SELECT block_number FROM last_3_days) + {% endif %} +), +ready_blocks AS ( + SELECT block_number + FROM to_do + + {% if not new_build %} + UNION + SELECT block_number + FROM {{ ref("_unconfirmed_blocks") }} + UNION + SELECT block_number + FROM {{ ref("_missing_txs") }} + {% endif %} + + {% if testing_limit is not none %} + LIMIT {{ testing_limit }} + {% endif %} +) + +{# Generate API requests for each block #} +SELECT + block_number, + ROUND(block_number, -3) AS partition_key, + live.udf_api( + 'POST', + '{{ node_url }}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', '{{ model_quantum_state }}' + ), + OBJECT_CONSTRUCT( + 'id', block_number, + 'jsonrpc', '2.0', + 'method', '{{ method }}', + 'params', {{ method_params }} + ), + '{{ node_secret_path }}' + ) AS request +FROM + ready_blocks + +{{ order_by_clause }} + +LIMIT {{ sql_limit }} \ No newline at end of file diff --git a/models/streamline/silver/core/realtime/streamline__confirm_blocks_realtime.sql b/models/streamline/silver/core/realtime/streamline__confirm_blocks_realtime.sql new file mode 100644 index 00000000..6135a3fc --- /dev/null +++ b/models/streamline/silver/core/realtime/streamline__confirm_blocks_realtime.sql @@ -0,0 +1,138 @@ +{# Set variables #} +{%- set model_name = 'CONFIRM_BLOCKS' -%} +{%- set model_type = 'REALTIME' -%} +{%- set min_block = var('GLOBAL_START_UP_BLOCK', none) -%} + +{%- set default_vars = set_default_variables_streamline(model_name, model_type) -%} + +{# Set up parameters for the streamline process. These will come from the vars set in dbt_project.yml #} +{%- set streamline_params = set_streamline_parameters( + model_name=model_name, + model_type=model_type +) -%} + +{%- set node_url = default_vars['node_url'] -%} +{%- set node_secret_path = default_vars['node_secret_path'] -%} +{%- set model_quantum_state = default_vars['model_quantum_state'] -%} +{%- set sql_limit = streamline_params['sql_limit'] -%} +{%- set testing_limit = default_vars['testing_limit'] -%} +{%- set order_by_clause = default_vars['order_by_clause'] -%} +{%- set new_build = default_vars['new_build'] -%} +{%- set method_params = streamline_params['method_params'] -%} +{%- set method = streamline_params['method'] -%} + +{# Log configuration details #} +{{ log_streamline_details( + model_name=model_name, + model_type=model_type, + node_url=node_url, + model_quantum_state=model_quantum_state, + sql_limit=sql_limit, + testing_limit=testing_limit, + order_by_clause=order_by_clause, + new_build=new_build, + streamline_params=streamline_params, + method_params=method_params, + method=method, + min_block=min_block +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params = streamline_params + ), + tags = ['streamline_core_realtime_confirm_blocks'] +) }} + +{# Main query starts here #} +WITH +{% if not new_build %} + last_3_days AS ( + SELECT block_number + FROM {{ ref("_block_lookback") }} + ), +{% endif %} + +{# Delay blocks #} +look_back AS ( + SELECT + block_number + FROM + {{ ref("_max_block_by_hour") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 6 + ), + +{# Identify blocks that need processing #} +to_do AS ( + SELECT block_number + FROM {{ ref("streamline__blocks") }} + WHERE + block_number IS NOT NULL + AND block_number <= (SELECT block_number FROM look_back) + {% if not new_build %} + AND block_number >= (SELECT block_number FROM last_3_days) + {% endif %} + {% if min_block is not none %} + AND block_number >= {{ min_block }} + {% endif %} + + EXCEPT + + {# Exclude blocks that have already been processed #} + SELECT block_number + FROM {{ ref('streamline__' ~ model_name.lower() ~ '_complete') }} + WHERE 1=1 + AND block_number IS NOT NULL + AND block_number <= (SELECT block_number FROM look_back) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) + {% if not new_build %} + AND block_number >= (SELECT block_number FROM last_3_days) + {% endif %} +) + +{# Prepare the final list of blocks to process #} +,ready_blocks AS ( + SELECT block_number + FROM to_do + + {% if testing_limit is not none %} + LIMIT {{ testing_limit }} + {% endif %} +) + +{# Generate API requests for each block #} +SELECT + block_number, + ROUND(block_number, -3) AS partition_key, + live.udf_api( + 'POST', + '{{ node_url }}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', '{{ model_quantum_state }}' + ), + OBJECT_CONSTRUCT( + 'id', block_number, + 'jsonrpc', '2.0', + 'method', '{{ method }}', + 'params', {{ method_params }} + ), + '{{ node_secret_path }}' + ) AS request +FROM + ready_blocks + +{{ order_by_clause }} + +LIMIT {{ sql_limit }} \ No newline at end of file diff --git a/models/streamline/silver/core/realtime/streamline__confirmed_blocks_realtime.sql b/models/streamline/silver/core/realtime/streamline__confirmed_blocks_realtime.sql deleted file mode 100644 index c1a46600..00000000 --- a/models/streamline/silver/core/realtime/streamline__confirmed_blocks_realtime.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = fsc_utils.if_data_call_function_v2( - func = 'streamline.udf_bulk_rest_api_v2', - target = "{{this.schema}}.{{this.identifier}}", - params ={ "external_table" :"confirm_blocks_v2", - "sql_limit" :"5000000", - "producer_batch_size" :"5000", - "worker_batch_size" :"500", - "sql_source" :"{{this.identifier}}" } - ), - tags = ['streamline_core_realtime'] -) }} -{{ fsc_evm.streamline_core_requests( - model_type = 'realtime', - model = 'confirmed_blocks', - quantum_state = 'streamline', - vault_secret_path = "vault/prod/ethereum/quicknode/mainnet", - query_limit = 600 -) }} diff --git a/models/streamline/silver/core/realtime/streamline__receipts_realtime.sql b/models/streamline/silver/core/realtime/streamline__receipts_realtime.sql index 7d5c81c6..6ec5c850 100644 --- a/models/streamline/silver/core/realtime/streamline__receipts_realtime.sql +++ b/models/streamline/silver/core/realtime/streamline__receipts_realtime.sql @@ -1,21 +1,130 @@ +{# Set variables #} +{%- set model_name = 'RECEIPTS' -%} +{%- set model_type = 'REALTIME' -%} +{%- set min_block = var('GLOBAL_START_UP_BLOCK', none) -%} + +{%- set default_vars = set_default_variables_streamline(model_name, model_type) -%} + +{# Set up parameters for the streamline process. These will come from the vars set in dbt_project.yml #} +{%- set streamline_params = set_streamline_parameters( + model_name=model_name, + model_type=model_type +) -%} + +{%- set node_url = default_vars['node_url'] -%} +{%- set node_secret_path = default_vars['node_secret_path'] -%} +{%- set model_quantum_state = default_vars['model_quantum_state'] -%} +{%- set sql_limit = streamline_params['sql_limit'] -%} +{%- set testing_limit = default_vars['testing_limit'] -%} +{%- set order_by_clause = default_vars['order_by_clause'] -%} +{%- set new_build = default_vars['new_build'] -%} +{%- set method_params = streamline_params['method_params'] -%} +{%- set method = streamline_params['method'] -%} + +{# Log configuration details #} +{{ log_streamline_details( + model_name=model_name, + model_type=model_type, + node_url=node_url, + model_quantum_state=model_quantum_state, + sql_limit=sql_limit, + testing_limit=testing_limit, + order_by_clause=order_by_clause, + new_build=new_build, + streamline_params=streamline_params, + method_params=method_params, + method=method, + min_block=min_block +) }} + +{# Set up dbt configuration #} {{ config ( materialized = "view", post_hook = fsc_utils.if_data_call_function_v2( func = 'streamline.udf_bulk_rest_api_v2', target = "{{this.schema}}.{{this.identifier}}", - params ={ "external_table" :"receipts_v2", - "sql_limit" :"100000", - "producer_batch_size" :"100", - "worker_batch_size" :"10", - "sql_source" :"{{this.identifier}}", - "exploded_key": tojson(["result"]) } + params = streamline_params ), - tags = ['streamline_core_realtime'] -) }} -{{ fsc_evm.streamline_core_requests( - model_type = 'realtime', - model = 'receipts', - quantum_state = 'streamline', - vault_secret_path = "vault/prod/ethereum/quicknode/mainnet", - query_limit = 300 + tags = ['streamline_core_realtime_receipts'] ) }} + +{# Main query starts here #} +WITH +{% if not new_build %} + last_3_days AS ( + SELECT block_number + FROM {{ ref("_block_lookback") }} + ), +{% endif %} + +{# Identify blocks that need processing #} +to_do AS ( + SELECT block_number + FROM {{ ref("streamline__blocks") }} + WHERE + block_number IS NOT NULL + {% if not new_build %} + AND block_number >= (SELECT block_number FROM last_3_days) + {% endif %} + {% if min_block is not none %} + AND block_number >= {{ min_block }} + {% endif %} + + EXCEPT + + {# Exclude blocks that have already been processed #} + SELECT block_number + FROM {{ ref('streamline__' ~ model_name.lower() ~ '_complete') }} + WHERE 1=1 + {% if not new_build %} + AND block_number >= (SELECT block_number FROM last_3_days) + {% endif %} +) + +{# Prepare the final list of blocks to process #} +,ready_blocks AS ( + SELECT block_number + FROM to_do + + {% if not new_build %} + UNION + SELECT block_number + FROM {{ ref("_unconfirmed_blocks") }} + UNION + SELECT block_number + FROM {{ ref("_missing_txs") }} + UNION + SELECT block_number + FROM {{ ref("_missing_receipts") }} + {% endif %} + + {% if testing_limit is not none %} + LIMIT {{ testing_limit }} + {% endif %} +) + +{# Generate API requests for each block #} +SELECT + block_number, + ROUND(block_number, -3) AS partition_key, + live.udf_api( + 'POST', + '{{ node_url }}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', '{{ model_quantum_state }}' + ), + OBJECT_CONSTRUCT( + 'id', block_number, + 'jsonrpc', '2.0', + 'method', '{{ method }}', + 'params', {{ method_params }} + ), + '{{ node_secret_path }}' + ) AS request +FROM + ready_blocks + +{{ order_by_clause }} + +LIMIT {{ sql_limit }} \ No newline at end of file diff --git a/models/streamline/silver/core/realtime/streamline__traces_realtime.sql b/models/streamline/silver/core/realtime/streamline__traces_realtime.sql index cce16528..34f7f984 100644 --- a/models/streamline/silver/core/realtime/streamline__traces_realtime.sql +++ b/models/streamline/silver/core/realtime/streamline__traces_realtime.sql @@ -1,21 +1,127 @@ +{# Set variables #} +{%- set model_name = 'TRACES' -%} +{%- set model_type = 'REALTIME' -%} +{%- set min_block = var('GLOBAL_START_UP_BLOCK', none) -%} + +{%- set default_vars = set_default_variables_streamline(model_name, model_type) -%} + +{# Set up parameters for the streamline process. These will come from the vars set in dbt_project.yml #} +{%- set streamline_params = set_streamline_parameters( + model_name=model_name, + model_type=model_type +) -%} + +{%- set node_url = default_vars['node_url'] -%} +{%- set node_secret_path = default_vars['node_secret_path'] -%} +{%- set model_quantum_state = default_vars['model_quantum_state'] -%} +{%- set sql_limit = streamline_params['sql_limit'] -%} +{%- set testing_limit = default_vars['testing_limit'] -%} +{%- set order_by_clause = default_vars['order_by_clause'] -%} +{%- set new_build = default_vars['new_build'] -%} +{%- set method_params = streamline_params['method_params'] -%} +{%- set method = streamline_params['method'] -%} + +{# Log configuration details #} +{{ log_streamline_details( + model_name=model_name, + model_type=model_type, + node_url=node_url, + model_quantum_state=model_quantum_state, + sql_limit=sql_limit, + testing_limit=testing_limit, + order_by_clause=order_by_clause, + new_build=new_build, + streamline_params=streamline_params, + method_params=method_params, + method=method, + min_block=min_block +) }} + +{# Set up dbt configuration #} {{ config ( materialized = "view", post_hook = fsc_utils.if_data_call_function_v2( func = 'streamline.udf_bulk_rest_api_v2', target = "{{this.schema}}.{{this.identifier}}", - params ={ "external_table" :"traces_v2", - "sql_limit" :"100000", - "producer_batch_size" :"100", - "worker_batch_size" :"10", - "sql_source" :"{{this.identifier}}", - "exploded_key": tojson(["result"]) } + params = streamline_params ), tags = ['streamline_core_realtime'] ) }} -{{ fsc_evm.streamline_core_requests( - model_type = 'realtime', - model = 'traces', - quantum_state = 'streamline', - vault_secret_path = "vault/prod/ethereum/quicknode/mainnet", - query_limit = 300 -) }} + +{# Main query starts here #} +WITH +{% if not new_build %} + last_3_days AS ( + SELECT block_number + FROM {{ ref("_block_lookback") }} + ), +{% endif %} + +{# Identify blocks that need processing #} +to_do AS ( + SELECT block_number + FROM {{ ref("streamline__blocks") }} + WHERE + block_number IS NOT NULL + {% if not new_build %} + AND block_number >= (SELECT block_number FROM last_3_days) + {% endif %} + {% if min_block is not none %} + AND block_number >= {{ min_block }} + {% endif %} + + EXCEPT + + {# Exclude blocks that have already been processed #} + SELECT block_number + FROM {{ ref('streamline__' ~ model_name.lower() ~ '_complete') }} + WHERE 1=1 + {% if not new_build %} + AND block_number >= (SELECT block_number FROM last_3_days) + {% endif %} +) + +{# Prepare the final list of blocks to process #} +,ready_blocks AS ( + SELECT block_number + FROM to_do + + {% if not new_build %} + UNION + SELECT block_number + FROM {{ ref("_unconfirmed_blocks") }} + UNION + SELECT block_number + FROM {{ ref("_missing_traces") }} + {% endif %} + + {% if testing_limit is not none %} + LIMIT {{ testing_limit }} + {% endif %} +) + +{# Generate API requests for each block #} +SELECT + block_number, + ROUND(block_number, -3) AS partition_key, + live.udf_api( + 'POST', + '{{ node_url }}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', '{{ model_quantum_state }}' + ), + OBJECT_CONSTRUCT( + 'id', block_number, + 'jsonrpc', '2.0', + 'method', '{{ method }}', + 'params', {{ method_params }} + ), + '{{ node_secret_path }}' + ) AS request +FROM + ready_blocks + +{{ order_by_clause }} + +LIMIT {{ sql_limit }} \ No newline at end of file diff --git a/models/streamline/silver/core/retry/_missing_receipts.sql b/models/streamline/silver/core/retry/_missing_receipts.sql index e4042caf..fb055b60 100644 --- a/models/streamline/silver/core/retry/_missing_receipts.sql +++ b/models/streamline/silver/core/retry/_missing_receipts.sql @@ -1,4 +1,34 @@ {{ config ( materialized = "ephemeral" ) }} -{{ fsc_evm.retry_missing_receipts() }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +) +SELECT + DISTINCT t.block_number AS block_number +FROM + {{ ref("silver__transactions") }} + t + LEFT JOIN {{ ref("silver__receipts") }} + r USING ( + block_number, + block_hash, + tx_hash + ) +WHERE + r.tx_hash IS NULL + AND t.block_number >= ( + SELECT + block_number + FROM + lookback + ) + AND t.block_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND ( + r._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + OR r._inserted_timestamp IS NULL) diff --git a/models/streamline/silver/core/retry/_missing_traces.sql b/models/streamline/silver/core/retry/_missing_traces.sql index f8bb0338..346f556c 100644 --- a/models/streamline/silver/core/retry/_missing_traces.sql +++ b/models/streamline/silver/core/retry/_missing_traces.sql @@ -14,4 +14,4 @@ FROM ) WHERE tr.tx_hash IS NULL - AND tx.block_timestamp > DATEADD('day', -5, SYSDATE()) \ No newline at end of file + AND tx.block_timestamp > DATEADD('day', -5, SYSDATE()) diff --git a/models/streamline/silver/core/retry/_missing_txs.sql b/models/streamline/silver/core/retry/_missing_txs.sql index 0f47532c..7718e970 100644 --- a/models/streamline/silver/core/retry/_missing_txs.sql +++ b/models/streamline/silver/core/retry/_missing_txs.sql @@ -1,4 +1,40 @@ {{ config ( materialized = "ephemeral" ) }} -{{ fsc_evm.retry_missing_txs() }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +), +transactions AS ( + SELECT + block_number, + POSITION, + LAG( + POSITION, + 1 + ) over ( + PARTITION BY block_number + ORDER BY + POSITION ASC + ) AS prev_POSITION + FROM + {{ ref("silver__transactions") }} + WHERE + block_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND block_number >= ( + SELECT + block_number + FROM + lookback + ) +) +SELECT + DISTINCT block_number AS block_number +FROM + transactions +WHERE + POSITION - prev_POSITION <> 1 diff --git a/models/streamline/silver/core/retry/_unconfirmed_blocks.sql b/models/streamline/silver/core/retry/_unconfirmed_blocks.sql index d0cdaf99..8e2507cd 100644 --- a/models/streamline/silver/core/retry/_unconfirmed_blocks.sql +++ b/models/streamline/silver/core/retry/_unconfirmed_blocks.sql @@ -1,4 +1,34 @@ {{ config ( materialized = "ephemeral" ) }} -{{ fsc_evm.retry_unconfirmed_blocks() }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +) +SELECT + DISTINCT cb.block_number AS block_number +FROM + {{ ref("silver__confirmed_blocks") }} + cb + LEFT JOIN {{ ref("silver__transactions") }} + txs USING ( + block_number, + block_hash, + tx_hash + ) +WHERE + txs.tx_hash IS NULL + AND cb.block_number >= ( + SELECT + block_number + FROM + lookback + ) + AND cb._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND ( + txs._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + OR txs._inserted_timestamp IS NULL) diff --git a/models/streamline/silver/core/streamline__blocks.sql b/models/streamline/silver/core/streamline__blocks.sql index 2cdc3189..9e56c764 100644 --- a/models/streamline/silver/core/streamline__blocks.sql +++ b/models/streamline/silver/core/streamline__blocks.sql @@ -1,5 +1,36 @@ +{%- if flags.WHICH == 'compile' and execute -%} + + {% set config_log = '\n' %} + {% set config_log = config_log ~ '\n=== DBT Model Config ===\n'%} + {% set config_log = config_log ~ '\n{{ config (\n' %} + {% set config_log = config_log ~ ' materialized = "' ~ config.get('materialized') ~ '",\n' %} + {% set config_log = config_log ~ ' tags = ' ~ config.get('tags') | tojson ~ '\n' %} + {% set config_log = config_log ~ ') }}\n' %} + {{ log(config_log, info=True) }} + {{ log("", info=True) }} +{%- endif -%} + {{ config ( materialized = "view", tags = ['streamline_core_complete'] ) }} -{{ fsc_evm.block_sequence() }} + +SELECT + _id, + ( + ({{ var('GLOBAL_BLOCKS_PER_HOUR',0) }} / 60) * {{ var('GLOBAL_CHAINHEAD_DELAY',3) }} + ) :: INT AS block_number_delay, --minute-based block delay + (_id - block_number_delay) :: INT AS block_number, + utils.udf_int_to_hex(block_number) AS block_number_hex +FROM + {{ ref('silver__number_sequence') }} +WHERE + _id <= ( + SELECT + COALESCE( + block_number, + 0 + ) + FROM + {{ ref("streamline__get_chainhead") }} + ) \ No newline at end of file diff --git a/models/streamline/silver/core/streamline__get_chainhead.sql b/models/streamline/silver/core/streamline__get_chainhead.sql index b4acd721..9e98ae7b 100644 --- a/models/streamline/silver/core/streamline__get_chainhead.sql +++ b/models/streamline/silver/core/streamline__get_chainhead.sql @@ -1,7 +1,54 @@ +{%- set model_quantum_state = var('CHAINHEAD_QUANTUM_STATE', 'livequery') -%} + +{%- set node_url = var('GLOBAL_NODE_URL', '{Service}/{Authentication}') -%} + +{%- if flags.WHICH == 'compile' and execute -%} + + {{ log("=== Current Variable Settings ===", info=True) }} + {{ log("CHAINHEAD_QUANTUM_STATE: " ~ model_quantum_state, info=True) }} + {{ log("", info=True) }} + + {{ log("=== API Details ===", info=True) }} + {{ log("NODE_URL: " ~ node_url, info=True) }} + {{ log("NODE_SECRET_PATH: " ~ var('GLOBAL_NODE_SECRET_PATH'), info=True) }} + {{ log("", info=True) }} + + {% set config_log = '\n' %} + {% set config_log = config_log ~ '\n=== DBT Model Config ===\n'%} + {% set config_log = config_log ~ '\n{{ config (\n' %} + {% set config_log = config_log ~ ' materialized = "' ~ config.get('materialized') ~ '",\n' %} + {% set config_log = config_log ~ ' tags = ' ~ config.get('tags') | tojson ~ '\n' %} + {% set config_log = config_log ~ ') }}\n' %} + {{ log(config_log, info=True) }} + {{ log("", info=True) }} + +{%- endif -%} + {{ config ( materialized = 'table', - tags = ['streamline_core_complete'] -) }} -{{ fsc_evm.streamline_core_chainhead( - vault_secret_path = 'vault/prod/ethereum/quicknode/mainnet' + tags = ['streamline_core_complete','chainhead'] ) }} + +SELECT + live.udf_api( + 'POST', + '{{ node_url }}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', '{{ model_quantum_state }}' + ), + OBJECT_CONSTRUCT( + 'id', + 0, + 'jsonrpc', + '2.0', + 'method', + 'eth_blockNumber', + 'params', + [] + ), + '{{ var('GLOBAL_NODE_SECRET_PATH') }}' + ) AS resp, + utils.udf_hex_to_int( + resp :data :result :: STRING + ) AS block_number \ No newline at end of file diff --git a/models/streamline/silver/core/streamline__get_chainhead.yml b/models/streamline/silver/core/streamline__get_chainhead.yml new file mode 100644 index 00000000..e318e4db --- /dev/null +++ b/models/streamline/silver/core/streamline__get_chainhead.yml @@ -0,0 +1,9 @@ +version: 2 +models: + - name: streamline__get_chainhead + description: "This model is used to get the chainhead from the blockchain." + + columns: + - name: BLOCK_NUMBER + tests: + - not_null \ No newline at end of file diff --git a/models/streamline/silver/decoder/complete/streamline__complete_decoded_logs.sql b/models/streamline/silver/decoder/complete/streamline__complete_decoded_logs.sql deleted file mode 100644 index 21ca68c5..00000000 --- a/models/streamline/silver/decoder/complete/streamline__complete_decoded_logs.sql +++ /dev/null @@ -1,13 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_decoded_logs') }} -{{ config ( - materialized = "incremental", - unique_key = "_log_id", - cluster_by = "ROUND(block_number, -3)", - incremental_predicates = ["dynamic_range", "block_number"], - merge_update_columns = ["_log_id"], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(_log_id)", - tags = ['streamline_decoded_logs_complete'] -) }} -{{ fsc_evm.streamline_decoded_complete( - model = 'decoded_logs' -) }} diff --git a/models/streamline/silver/decoder/decoded_logs/complete/streamline__decoded_logs_complete.sql b/models/streamline/silver/decoder/decoded_logs/complete/streamline__decoded_logs_complete.sql new file mode 100644 index 00000000..3e805898 --- /dev/null +++ b/models/streamline/silver/decoder/decoded_logs/complete/streamline__decoded_logs_complete.sql @@ -0,0 +1,50 @@ +{# Set variables #} +{%- set source_name = 'DECODED_LOGS' -%} +{%- set model_type = 'COMPLETE' -%} + +{%- set full_refresh_type = var((source_name ~ '_complete_full_refresh').upper(), false) -%} + +{% set post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(_log_id)" %} + +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +-- depends_on: {{ ref('bronze__' ~ source_name.lower()) }} + +{{ config ( + materialized = "incremental", + unique_key = "_log_id", + cluster_by = "ROUND(block_number, -3)", + incremental_predicates = ["dynamic_range", "block_number"], + merge_update_columns = ["_log_id"], + post_hook = post_hook, + full_refresh = full_refresh_type, + tags = ['streamline_decoded_logs_complete'] +) }} + +{# Main query starts here #} +SELECT + block_number, + file_name, + id AS _log_id, + {{ dbt_utils.generate_surrogate_key(['id']) }} AS complete_{{ source_name.lower() }}_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {% if is_incremental() %} + {{ ref('bronze__' ~ source_name.lower()) }} + WHERE + _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1970-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM + {{ this }} + ) + {% else %} + {{ ref('bronze__' ~ source_name.lower() ~ '_fr') }} + {% endif %} + +QUALIFY (ROW_NUMBER() OVER (PARTITION BY id ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/decoder/decoded_logs/realtime/streamline__decoded_logs_realtime.sql b/models/streamline/silver/decoder/decoded_logs/realtime/streamline__decoded_logs_realtime.sql new file mode 100644 index 00000000..a0ff21b9 --- /dev/null +++ b/models/streamline/silver/decoder/decoded_logs/realtime/streamline__decoded_logs_realtime.sql @@ -0,0 +1,110 @@ +{%- set testing_limit = var('DECODED_LOGS_REALTIME_TESTING_LIMIT', none) -%} + +{%- set streamline_params = { + "external_table": var("DECODED_LOGS_REALTIME_EXTERNAL_TABLE", "decoded_logs"), + "sql_limit": var("DECODED_LOGS_REALTIME_SQL_LIMIT", 10000000), + "producer_batch_size": var("DECODED_LOGS_REALTIME_PRODUCER_BATCH_SIZE", 400000), + "worker_batch_size": var("DECODED_LOGS_REALTIME_WORKER_BATCH_SIZE", 200000), + "sql_source": "decoded_logs_realtime" +} -%} + +{# Log configuration details #} +{{ log_model_details( + params = streamline_params +) }} + +{# Set up dbt configuration #} +{{ config ( + materialized = "view", + post_hook = [fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_decode_logs_v2', + target = "{{this.schema}}.{{this.identifier}}", + params = { + "external_table": streamline_params['external_table'], + "sql_limit": streamline_params['sql_limit'], + "producer_batch_size": streamline_params['producer_batch_size'], + "worker_batch_size": streamline_params['worker_batch_size'], + "sql_source": streamline_params['sql_source'] + } + ), + fsc_utils.if_data_call_wait()], + tags = ['streamline_decoded_logs_realtime'] +) }} + +WITH target_blocks AS ( + SELECT + block_number + FROM + {{ ref('core__fact_blocks') }} + WHERE + block_number >= ( + SELECT + block_number + FROM + {{ ref('_24_hour_lookback') }} + ) +), +existing_logs_to_exclude AS ( + SELECT + _log_id + FROM + {{ ref('streamline__decoded_logs_complete') }} + l + INNER JOIN target_blocks b USING (block_number) + WHERE + l.inserted_timestamp :: DATE >= DATEADD('day', -2, SYSDATE()) +), +candidate_logs AS ( + SELECT + l.block_number, + l.tx_hash, + l.event_index, + l.contract_address, + l.topics, + l.data, + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id + FROM + target_blocks b + INNER JOIN {{ ref('core__fact_event_logs') }} + l USING (block_number) + WHERE + l.tx_succeeded + AND l.inserted_timestamp :: DATE >= DATEADD('day', -2, SYSDATE()) +) +SELECT + l.block_number, + l._log_id, + A.abi, + OBJECT_CONSTRUCT( + 'topics', + l.topics, + 'data', + l.data, + 'address', + l.contract_address + ) AS DATA +FROM + candidate_logs l + INNER JOIN {{ ref('silver__complete_event_abis') }} A + ON A.parent_contract_address = l.contract_address + AND A.event_signature = l.topics [0] :: STRING + AND l.block_number BETWEEN A.start_block + AND A.end_block +WHERE + NOT EXISTS ( + SELECT + 1 + FROM + existing_logs_to_exclude e + WHERE + e._log_id = l._log_id + ) + +{% if testing_limit is not none %} + LIMIT + {{ testing_limit }} +{% endif %} \ No newline at end of file diff --git a/models/streamline/silver/decoder/complete/streamline__complete_decoded_traces.sql b/models/streamline/silver/decoder/decoded_traces/complete/streamline__complete_decoded_traces.sql similarity index 92% rename from models/streamline/silver/decoder/complete/streamline__complete_decoded_traces.sql rename to models/streamline/silver/decoder/decoded_traces/complete/streamline__complete_decoded_traces.sql index c8ee41c4..1e1f6a17 100644 --- a/models/streamline/silver/decoder/complete/streamline__complete_decoded_traces.sql +++ b/models/streamline/silver/decoder/decoded_traces/complete/streamline__complete_decoded_traces.sql @@ -8,6 +8,6 @@ post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(_call_id)", tags = ['streamline_decoded_traces_complete'] ) }} -{{ fsc_evm.streamline_decoded_complete( +{{ v0_streamline_decoded_complete( model = 'decoded_traces' ) }} diff --git a/models/streamline/silver/decoder/realtime/streamline__decode_traces_realtime.sql b/models/streamline/silver/decoder/decoded_traces/realtime/streamline__decode_traces_realtime.sql similarity index 100% rename from models/streamline/silver/decoder/realtime/streamline__decode_traces_realtime.sql rename to models/streamline/silver/decoder/decoded_traces/realtime/streamline__decode_traces_realtime.sql diff --git a/models/streamline/silver/decoder/realtime/streamline__decode_logs_realtime.sql b/models/streamline/silver/decoder/realtime/streamline__decode_logs_realtime.sql deleted file mode 100644 index 921b61a6..00000000 --- a/models/streamline/silver/decoder/realtime/streamline__decode_logs_realtime.sql +++ /dev/null @@ -1,89 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = [fsc_utils.if_data_call_function_v2( - func = 'streamline.udf_bulk_decode_logs_v2', - target = "{{this.schema}}.{{this.identifier}}", - params ={ "external_table" :"DECODED_LOGS", - "sql_limit" :"20000000", - "producer_batch_size" :"20000000", - "worker_batch_size" :"200000", - "sql_source" :"{{this.identifier}}" } - ), - fsc_utils.if_data_call_wait()], - tags = ['streamline_decoded_logs_realtime'] -) }} - -WITH target_blocks AS ( - - SELECT - block_number - FROM - {{ ref('core__fact_blocks') }} - WHERE - block_number >= ( - SELECT - block_number - FROM - {{ ref("_block_lookback") }} - ) -), -existing_logs_to_exclude AS ( - SELECT - _log_id - FROM - {{ ref('streamline__complete_decoded_logs') }} - l - INNER JOIN target_blocks b USING (block_number) - WHERE - l._inserted_timestamp :: DATE >= DATEADD('day', -5, SYSDATE()) -), -candidate_logs AS ( - SELECT - l.block_number, - l.tx_hash, - l.event_index, - l.contract_address, - l.topics, - l.data, - CONCAT( - l.tx_hash :: STRING, - '-', - l.event_index :: STRING - ) AS _log_id - FROM - target_blocks b - INNER JOIN {{ ref('core__fact_event_logs') }} - l USING (block_number) - WHERE - l.tx_succeeded - AND l.inserted_timestamp :: DATE >= DATEADD('day', -5, SYSDATE()) -) -SELECT - l.block_number, - l._log_id, - A.abi AS abi, - OBJECT_CONSTRUCT( - 'topics', - l.topics, - 'data', - l.data, - 'address', - l.contract_address - ) AS DATA -FROM - candidate_logs l - INNER JOIN {{ ref('silver__complete_event_abis') }} A - ON A.parent_contract_address = l.contract_address - AND A.event_signature = l.topics [0] :: STRING - AND l.block_number BETWEEN A.start_block - AND A.end_block -WHERE - NOT EXISTS ( - SELECT - 1 - FROM - existing_logs_to_exclude e - WHERE - e._log_id = l._log_id - ) -limit 7500000 \ No newline at end of file diff --git a/package-lock.yml b/package-lock.yml index 6345f9fb..d5a9564f 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -6,7 +6,7 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: 8c99db499671ff6f514bd0695f7b1f20bce8d80d + revision: c3ab97e8e06d31e8c6f63819714e0a2d45c45e82 - git: https://github.com/FlipsideCrypto/fsc-evm.git revision: 2a41facb272eba867486813d29dad177b913e59f - package: get-select/dbt_snowflake_query_tags @@ -15,4 +15,4 @@ packages: version: 0.7.2 - git: https://github.com/FlipsideCrypto/livequery-models.git revision: b024188be4e9c6bc00ed77797ebdc92d351d620e -sha1_hash: 0a7662309787040f0824f4e98f018e024d01fca1 +sha1_hash: 3683c230b43d3e3d45bf55b57becf0ea43cad172 diff --git a/packages.yml b/packages.yml index 19f9153b..44d638c8 100644 --- a/packages.yml +++ b/packages.yml @@ -6,7 +6,7 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: "v1.30.0" + revision: "v1.31.0" - git: https://github.com/FlipsideCrypto/fsc-evm.git revision: "v1.6.0" - package: get-select/dbt_snowflake_query_tags