From 88df737fa29d1d283e6ddf9550bc7d3b587f9083 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 5 Sep 2024 20:43:33 -0600 Subject: [PATCH] AN-5203/Deploy flowEVM (#343) * flow evm init * flow evm testnet - blocks * deploy dev udfs, update typo * send block height as hex * dev limit * use stg external table * some links * add query to side doc * upd external tbl to stg & testnet * silver and evm txs * testnet silver models pt1 * testnet events final, decode hash udf * udf get evm chainhead * upd resource list * new udf. receipts pipeline. upd testnet model cols. expand readme * receipts silver * traces * vault * reorg to evm dir * rm testnet, blocks lookback * lookbacks, move qualify * macro and align naming * del testnet, reset namespace to gen * silver_evm - receipts and txs * core_evm fact_blocks & fact_transactions * fix vault path in get blocks. Logs v1 (need sample) * core_evm fact_logs * CR updates * del readme and add workflow * upd vault path, batch size * incr logic on modified to core_evm * use local utils.udf, add blockNumber col to complete blocks check * rm col * correct _invocation_id * upd nv csv * incr batch limit due to late start * add evm tag to model run --- ...dbt_run_streamline_collections_testnet.yml | 46 --- ...ml => dbt_run_streamline_evm_realtime.yml} | 12 +- ...streamline_transaction_results_testnet.yml | 46 --- ...bt_run_streamline_transactions_testnet.yml | 46 --- data/seeds__network_version.csv | 3 +- macros/evm/schema.yml | 20 ++ macros/evm/udf_decode_hash_array.sql | 31 ++ macros/evm/udf_get_evm_chainhead.sql | 30 ++ macros/streamline/models.sql | 74 +++- models/evm/bronze/bronze_evm__FR_blocks.sql | 8 + models/evm/bronze/bronze_evm__FR_receipts.sql | 8 + models/evm/bronze/bronze_evm__FR_traces.sql | 8 + models/evm/bronze/bronze_evm__blocks.sql | 8 + models/evm/bronze/bronze_evm__receipts.sql | 8 + models/evm/bronze/bronze_evm__traces.sql | 8 + .../descriptions/blocks/block_header_json.md | 5 + models/evm/descriptions/blocks/blockchain.md | 5 + models/evm/descriptions/blocks/blocks_hash.md | 5 + .../evm/descriptions/blocks/blocks_nonce.md | 5 + .../descriptions/blocks/blocks_table_doc.md | 5 + models/evm/descriptions/blocks/difficulty.md | 5 + models/evm/descriptions/blocks/extra_data.md | 5 + models/evm/descriptions/blocks/gas_limit.md | 5 + models/evm/descriptions/blocks/gas_used.md | 5 + models/evm/descriptions/blocks/miner.md | 5 + models/evm/descriptions/blocks/network.md | 5 + models/evm/descriptions/blocks/parent_hash.md | 5 + .../evm/descriptions/blocks/receipts_root.md | 5 + models/evm/descriptions/blocks/sha3_uncles.md | 5 + models/evm/descriptions/blocks/size.md | 5 + .../descriptions/blocks/total_difficulty.md | 5 + models/evm/descriptions/blocks/tx_count.md | 5 + .../evm/descriptions/blocks/uncle_blocks.md | 5 + .../descriptions/event_logs/event_index.md | 5 + .../descriptions/event_logs/event_inputs.md | 5 + .../evm/descriptions/event_logs/event_name.md | 5 + .../descriptions/event_logs/event_removed.md | 5 + .../evm/descriptions/event_logs/event_sig.md | 5 + .../descriptions/event_logs/log_id_events.md | 5 + .../event_logs/logs_contract_address.md | 5 + .../event_logs/logs_contract_name.md | 5 + .../evm/descriptions/event_logs/logs_data.md | 5 + .../descriptions/event_logs/logs_table_doc.md | 5 + .../descriptions/event_logs/logs_tx_hash.md | 5 + .../descriptions/event_logs/origin_from.md | 5 + .../evm/descriptions/event_logs/origin_to.md | 5 + models/evm/descriptions/event_logs/topics.md | 5 + .../evm/descriptions/general/block_number.md | 5 + .../descriptions/general/block_timestamp.md | 5 + models/evm/descriptions/general/decimals.md | 5 + .../evm/descriptions/general/deprecation.md | 11 + .../evm/descriptions/general/from_address.md | 5 + .../evm/descriptions/general/ingested_at.md | 5 + .../descriptions/general/precise_amounts.md | 17 + models/evm/descriptions/general/symbol.md | 5 + models/evm/descriptions/general/to_address.md | 5 + .../transactions/cumulative_gas_used.md | 5 + .../transactions/tx_block_hash.md | 5 + .../evm/descriptions/transactions/tx_fee.md | 5 + .../descriptions/transactions/tx_gas_limit.md | 5 + .../descriptions/transactions/tx_gas_price.md | 5 + .../descriptions/transactions/tx_gas_used.md | 5 + .../evm/descriptions/transactions/tx_hash.md | 5 + .../transactions/tx_input_data.md | 5 + .../evm/descriptions/transactions/tx_json.md | 5 + .../evm/descriptions/transactions/tx_nonce.md | 5 + .../transactions/tx_origin_sig.md | 5 + .../descriptions/transactions/tx_position.md | 5 + .../descriptions/transactions/tx_status.md | 5 + .../descriptions/transactions/tx_table_doc.md | 5 + models/evm/descriptions/transactions/value.md | 5 + .../evm/gold/core/core_evm__fact_blocks.sql | 84 +++++ .../evm/gold/core/core_evm__fact_blocks.yml | 46 +++ .../gold/core/core_evm__fact_event_logs.sql | 41 +++ .../gold/core/core_evm__fact_event_logs.yml | 38 ++ .../gold/core/core_evm__fact_transactions.sql | 50 +++ .../gold/core/core_evm__fact_transactions.yml | 64 ++++ models/evm/silver/core/silver_evm__blocks.sql | 82 +++++ models/evm/silver/core/silver_evm__logs.sql | 194 ++++++++++ .../evm/silver/core/silver_evm__receipts.sql | 122 +++++++ models/evm/silver/core/silver_evm__traces.sql | 76 ++++ .../silver/core/silver_evm__transactions.sql | 333 ++++++++++++++++++ .../silver/streamline/_evm_block_lookback.sql | 11 + .../streamline__complete_get_evm_blocks.sql | 43 +++ .../streamline__complete_get_evm_receipts.sql | 42 +++ .../streamline__complete_get_evm_traces.sql | 44 +++ .../streamline__get_evm_blocks_realtime.sql | 89 +++++ .../streamline__get_evm_receipts_realtime.sql | 99 ++++++ .../streamline__get_evm_traces_realtime.sql | 104 ++++++ .../streamline/retry/_missing_receipts.sql | 34 ++ .../streamline/retry/_missing_traces.sql | 31 ++ .../streamline/streamline__evm_blocks.sql | 21 ++ models/sources.yml | 4 + 93 files changed, 2159 insertions(+), 147 deletions(-) delete mode 100644 .github/workflows/dbt_run_streamline_collections_testnet.yml rename .github/workflows/{dbt_run_streamline_blocks_testnet.yml => dbt_run_streamline_evm_realtime.yml} (70%) delete mode 100644 .github/workflows/dbt_run_streamline_transaction_results_testnet.yml delete mode 100644 .github/workflows/dbt_run_streamline_transactions_testnet.yml create mode 100644 macros/evm/schema.yml create mode 100644 macros/evm/udf_decode_hash_array.sql create mode 100644 macros/evm/udf_get_evm_chainhead.sql create mode 100644 models/evm/bronze/bronze_evm__FR_blocks.sql create mode 100644 models/evm/bronze/bronze_evm__FR_receipts.sql create mode 100644 models/evm/bronze/bronze_evm__FR_traces.sql create mode 100644 models/evm/bronze/bronze_evm__blocks.sql create mode 100644 models/evm/bronze/bronze_evm__receipts.sql create mode 100644 models/evm/bronze/bronze_evm__traces.sql create mode 100644 models/evm/descriptions/blocks/block_header_json.md create mode 100644 models/evm/descriptions/blocks/blockchain.md create mode 100644 models/evm/descriptions/blocks/blocks_hash.md create mode 100644 models/evm/descriptions/blocks/blocks_nonce.md create mode 100644 models/evm/descriptions/blocks/blocks_table_doc.md create mode 100644 models/evm/descriptions/blocks/difficulty.md create mode 100644 models/evm/descriptions/blocks/extra_data.md create mode 100644 models/evm/descriptions/blocks/gas_limit.md create mode 100644 models/evm/descriptions/blocks/gas_used.md create mode 100644 models/evm/descriptions/blocks/miner.md create mode 100644 models/evm/descriptions/blocks/network.md create mode 100644 models/evm/descriptions/blocks/parent_hash.md create mode 100644 models/evm/descriptions/blocks/receipts_root.md create mode 100644 models/evm/descriptions/blocks/sha3_uncles.md create mode 100644 models/evm/descriptions/blocks/size.md create mode 100644 models/evm/descriptions/blocks/total_difficulty.md create mode 100644 models/evm/descriptions/blocks/tx_count.md create mode 100644 models/evm/descriptions/blocks/uncle_blocks.md create mode 100644 models/evm/descriptions/event_logs/event_index.md create mode 100644 models/evm/descriptions/event_logs/event_inputs.md create mode 100644 models/evm/descriptions/event_logs/event_name.md create mode 100644 models/evm/descriptions/event_logs/event_removed.md create mode 100644 models/evm/descriptions/event_logs/event_sig.md create mode 100644 models/evm/descriptions/event_logs/log_id_events.md create mode 100644 models/evm/descriptions/event_logs/logs_contract_address.md create mode 100644 models/evm/descriptions/event_logs/logs_contract_name.md create mode 100644 models/evm/descriptions/event_logs/logs_data.md create mode 100644 models/evm/descriptions/event_logs/logs_table_doc.md create mode 100644 models/evm/descriptions/event_logs/logs_tx_hash.md create mode 100644 models/evm/descriptions/event_logs/origin_from.md create mode 100644 models/evm/descriptions/event_logs/origin_to.md create mode 100644 models/evm/descriptions/event_logs/topics.md create mode 100644 models/evm/descriptions/general/block_number.md create mode 100644 models/evm/descriptions/general/block_timestamp.md create mode 100644 models/evm/descriptions/general/decimals.md create mode 100644 models/evm/descriptions/general/deprecation.md create mode 100644 models/evm/descriptions/general/from_address.md create mode 100644 models/evm/descriptions/general/ingested_at.md create mode 100644 models/evm/descriptions/general/precise_amounts.md create mode 100644 models/evm/descriptions/general/symbol.md create mode 100644 models/evm/descriptions/general/to_address.md create mode 100644 models/evm/descriptions/transactions/cumulative_gas_used.md create mode 100644 models/evm/descriptions/transactions/tx_block_hash.md create mode 100644 models/evm/descriptions/transactions/tx_fee.md create mode 100644 models/evm/descriptions/transactions/tx_gas_limit.md create mode 100644 models/evm/descriptions/transactions/tx_gas_price.md create mode 100644 models/evm/descriptions/transactions/tx_gas_used.md create mode 100644 models/evm/descriptions/transactions/tx_hash.md create mode 100644 models/evm/descriptions/transactions/tx_input_data.md create mode 100644 models/evm/descriptions/transactions/tx_json.md create mode 100644 models/evm/descriptions/transactions/tx_nonce.md create mode 100644 models/evm/descriptions/transactions/tx_origin_sig.md create mode 100644 models/evm/descriptions/transactions/tx_position.md create mode 100644 models/evm/descriptions/transactions/tx_status.md create mode 100644 models/evm/descriptions/transactions/tx_table_doc.md create mode 100644 models/evm/descriptions/transactions/value.md create mode 100644 models/evm/gold/core/core_evm__fact_blocks.sql create mode 100644 models/evm/gold/core/core_evm__fact_blocks.yml create mode 100644 models/evm/gold/core/core_evm__fact_event_logs.sql create mode 100644 models/evm/gold/core/core_evm__fact_event_logs.yml create mode 100644 models/evm/gold/core/core_evm__fact_transactions.sql create mode 100644 models/evm/gold/core/core_evm__fact_transactions.yml create mode 100644 models/evm/silver/core/silver_evm__blocks.sql create mode 100644 models/evm/silver/core/silver_evm__logs.sql create mode 100644 models/evm/silver/core/silver_evm__receipts.sql create mode 100644 models/evm/silver/core/silver_evm__traces.sql create mode 100644 models/evm/silver/core/silver_evm__transactions.sql create mode 100644 models/evm/silver/streamline/_evm_block_lookback.sql create mode 100644 models/evm/silver/streamline/complete/streamline__complete_get_evm_blocks.sql create mode 100644 models/evm/silver/streamline/complete/streamline__complete_get_evm_receipts.sql create mode 100644 models/evm/silver/streamline/complete/streamline__complete_get_evm_traces.sql create mode 100644 models/evm/silver/streamline/realtime/streamline__get_evm_blocks_realtime.sql create mode 100644 models/evm/silver/streamline/realtime/streamline__get_evm_receipts_realtime.sql create mode 100644 models/evm/silver/streamline/realtime/streamline__get_evm_traces_realtime.sql create mode 100644 models/evm/silver/streamline/retry/_missing_receipts.sql create mode 100644 models/evm/silver/streamline/retry/_missing_traces.sql create mode 100644 models/evm/silver/streamline/streamline__evm_blocks.sql diff --git a/.github/workflows/dbt_run_streamline_collections_testnet.yml b/.github/workflows/dbt_run_streamline_collections_testnet.yml deleted file mode 100644 index bc166fd..0000000 --- a/.github/workflows/dbt_run_streamline_collections_testnet.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: dbt_run_streamline_collections_testnet -run-name: dbt_run_streamline_collections_testnet - -on: - workflow_dispatch: - schedule: - # 1x/hour schedule = At hour 8, 21, 36, 51 every day (see https://crontab.guru) - - cron: "11,31,51 10,20 * * *" - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run DBT Realtime - run: | - dbt run -s 2+streamline__get_testnet_collections_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' diff --git a/.github/workflows/dbt_run_streamline_blocks_testnet.yml b/.github/workflows/dbt_run_streamline_evm_realtime.yml similarity index 70% rename from .github/workflows/dbt_run_streamline_blocks_testnet.yml rename to .github/workflows/dbt_run_streamline_evm_realtime.yml index 2c97938..1f1b0ce 100644 --- a/.github/workflows/dbt_run_streamline_blocks_testnet.yml +++ b/.github/workflows/dbt_run_streamline_evm_realtime.yml @@ -1,11 +1,11 @@ -name: dbt_run_streamline_blocks_testnet -run-name: dbt_run_streamline_blocks_testnet +name: dbt_run_streamline_evm_realtime +run-name: dbt_run_streamline_evm_realtime on: workflow_dispatch: schedule: - # 1x/hour schedule = At hour 8, 21, 36, 51 every day (see https://crontab.guru) - - cron: "7,27,47 10,20 * * *" + # Every hour (see https://crontab.guru) + - cron: "0 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" @@ -41,6 +41,6 @@ jobs: pip install -r requirements.txt dbt deps - - name: Run DBT Realtime + - name: Run DBT Jobs run: | - dbt run -s 2+streamline__get_testnet_blocks_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -s 2+tag:streamline_realtime_evm tag:evm diff --git a/.github/workflows/dbt_run_streamline_transaction_results_testnet.yml b/.github/workflows/dbt_run_streamline_transaction_results_testnet.yml deleted file mode 100644 index 95db09a..0000000 --- a/.github/workflows/dbt_run_streamline_transaction_results_testnet.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: dbt_run_streamline_transaction_results_testnet -run-name: dbt_run_streamline_transaction_results_testnet - -on: - workflow_dispatch: - schedule: - # 1x/hour schedule = At hour 8, 21, 36, 51 every day (see https://crontab.guru) - - cron: "15,35,55 10,20 * * *" - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run DBT Realtime - run: | - dbt run -s 1+streamline__get_testnet_transaction_results_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' diff --git a/.github/workflows/dbt_run_streamline_transactions_testnet.yml b/.github/workflows/dbt_run_streamline_transactions_testnet.yml deleted file mode 100644 index 37ab93e..0000000 --- a/.github/workflows/dbt_run_streamline_transactions_testnet.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: dbt_run_streamline_transactions_testnet -run-name: dbt_run_streamline_transactions_testnet - -on: - workflow_dispatch: - schedule: - # 1x/hour schedule = At hour 8, 21, 36, 51 every day (see https://crontab.guru) - - cron: "15,35,55 10,20 * * *" - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run DBT Realtime - run: | - dbt run -s 1+streamline__get_testnet_transactions_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' diff --git a/data/seeds__network_version.csv b/data/seeds__network_version.csv index 0a24b8c..ea8a5e2 100644 --- a/data/seeds__network_version.csv +++ b/data/seeds__network_version.csv @@ -28,4 +28,5 @@ root_height,network_version,node_url,end_height 44950207,mainnet-21,access-001.mainnet21.nodes.onflow.org:9000,47169686 47169687,mainnet-22,access-001.mainnet22.nodes.onflow.org:9000,55114466 55114467,mainnet-23,access-001.mainnet23.nodes.onflow.org:9000,65264618 -65264619,mainnet-24,access-001.mainnet24.nodes.onflow.org:9000,100000000 +65264619,mainnet-24,access-001.mainnet24.nodes.onflow.org:9000,85981134 +85981135,mainnet-25,access-001.mainnet25.nodes.onflow.org:9000,1000000000 diff --git a/macros/evm/schema.yml b/macros/evm/schema.yml new file mode 100644 index 0000000..1475513 --- /dev/null +++ b/macros/evm/schema.yml @@ -0,0 +1,20 @@ +version: 2 + +macros: + - name: run_create_udf_decode_hash_array + description: | + This macro creates a UDF that takes in a bytearray, parses out the value, and returns a decoded hash. + arguments: + - name: raw_array + description: The bytearray to be decoded. + type: array + return_type: string + + - name: run_create_udf_get_evm_chainhead + description: | + This macro creates a UDF that returns chainhead for the Flow EVM execution environment. This takes an optional argument NETWORK and accepts either "testnet" or "mainnet". + arguments: + - name: network + description: The network to query for chainhead. + type: string + return_type: string diff --git a/macros/evm/udf_decode_hash_array.sql b/macros/evm/udf_decode_hash_array.sql new file mode 100644 index 0000000..ab834f0 --- /dev/null +++ b/macros/evm/udf_decode_hash_array.sql @@ -0,0 +1,31 @@ +{% macro run_create_udf_decode_hash_array() %} + {% set sql %} + +CREATE +OR REPLACE FUNCTION {{ target.database }}.streamline.udf_decode_hash_array(raw_array ARRAY) +RETURNS STRING +LANGUAGE PYTHON +RUNTIME_VERSION = '3.8' +HANDLER = 'decode_hash_array' +AS +$$ +def decode_hash_array(raw_array): + try: + # Parse the JSON array + data = raw_array + + # Extract and convert values + hex_values = [format(int(item['value']), '02x') for item in data] + + # Concatenate and add prefix + result = '0x' + ''.join(hex_values) + + return result.lower() + except Exception as e: + return f"Error: {str(e)}" +$$ +; + + {% endset %} + {% do run_query(sql) %} +{% endmacro %} diff --git a/macros/evm/udf_get_evm_chainhead.sql b/macros/evm/udf_get_evm_chainhead.sql new file mode 100644 index 0000000..dd9dec3 --- /dev/null +++ b/macros/evm/udf_get_evm_chainhead.sql @@ -0,0 +1,30 @@ +{% macro run_create_udf_get_evm_chainhead() %} + {% set sql %} + + CREATE + OR REPLACE FUNCTION {{ target.database }}.streamline.udf_get_evm_chainhead( + network STRING DEFAULT 'mainnet' + ) + RETURNS INTEGER + AS + $$ + SELECT + utils.udf_hex_to_int( + live.udf_api( + 'POST', + '{Service}', + {}, + OBJECT_CONSTRUCT( + 'method', 'eth_blockNumber', + 'id', 1, + 'jsonrpc', '2.0', + 'params', [] + ), + 'Vault/{{ target.name }}/flow/evm/' || lower(network) + ):data:result + ) :: INTEGER AS block_number + $$ + + {% endset %} + {% do run_query(sql) %} +{% endmacro %} diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql index 7d5514e..94eda3a 100644 --- a/macros/streamline/models.sql +++ b/macros/streamline/models.sql @@ -84,8 +84,7 @@ b.{{ partition_name }} = s.{{ partition_name }} {% endmacro %} - - {% macro streamline_multiple_external_table_query( +{% macro streamline_multiple_external_table_query( table_names, partition_function, partition_name, @@ -150,3 +149,74 @@ FINAL {% endmacro %} + +{# Added v2 external table query for flowEVM Deployment Sept 2024 #} + +{% macro streamline_external_table_query_v2( + model, + partition_function + ) %} + WITH meta AS ( + SELECT + job_created_time AS _inserted_timestamp, + file_name, + {{ partition_function }} AS partition_key + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, SYSDATE()), + table_name => '{{ source( "bronze_streamline", model) }}') + ) A + ) + SELECT + s.*, + b.file_name, + _inserted_timestamp + FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.partition_key = s.partition_key + WHERE + b.partition_key = s.partition_key + AND DATA :error IS NULL + +{% endmacro %} + +{% macro streamline_external_table_FR_query_v2( + model, + partition_function + ) %} + WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + {{ partition_function }} AS partition_key + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", model) }}' + ) + ) A + ) +SELECT + s.*, + b.file_name, + _inserted_timestamp +FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.partition_key = s.partition_key +WHERE + b.partition_key = s.partition_key + AND DATA :error IS NULL +{% endmacro %} diff --git a/models/evm/bronze/bronze_evm__FR_blocks.sql b/models/evm/bronze/bronze_evm__FR_blocks.sql new file mode 100644 index 0000000..6457a02 --- /dev/null +++ b/models/evm/bronze/bronze_evm__FR_blocks.sql @@ -0,0 +1,8 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_FR_query_v2( + model = "evm_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/bronze/bronze_evm__FR_receipts.sql b/models/evm/bronze/bronze_evm__FR_receipts.sql new file mode 100644 index 0000000..37eefad --- /dev/null +++ b/models/evm/bronze/bronze_evm__FR_receipts.sql @@ -0,0 +1,8 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_FR_query_v2( + model = "evm_receipts", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/bronze/bronze_evm__FR_traces.sql b/models/evm/bronze/bronze_evm__FR_traces.sql new file mode 100644 index 0000000..02a0d18 --- /dev/null +++ b/models/evm/bronze/bronze_evm__FR_traces.sql @@ -0,0 +1,8 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_FR_query_v2( + model = "evm_traces", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/bronze/bronze_evm__blocks.sql b/models/evm/bronze/bronze_evm__blocks.sql new file mode 100644 index 0000000..781174b --- /dev/null +++ b/models/evm/bronze/bronze_evm__blocks.sql @@ -0,0 +1,8 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_query_v2( + model = "evm_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/bronze/bronze_evm__receipts.sql b/models/evm/bronze/bronze_evm__receipts.sql new file mode 100644 index 0000000..e57b882 --- /dev/null +++ b/models/evm/bronze/bronze_evm__receipts.sql @@ -0,0 +1,8 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_query_v2( + model = "evm_receipts", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/bronze/bronze_evm__traces.sql b/models/evm/bronze/bronze_evm__traces.sql new file mode 100644 index 0000000..c299004 --- /dev/null +++ b/models/evm/bronze/bronze_evm__traces.sql @@ -0,0 +1,8 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_query_v2( + model = "evm_traces", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/descriptions/blocks/block_header_json.md b/models/evm/descriptions/blocks/block_header_json.md new file mode 100644 index 0000000..cad7117 --- /dev/null +++ b/models/evm/descriptions/blocks/block_header_json.md @@ -0,0 +1,5 @@ +{% docs flowevm_block_header_json %} + +This JSON column contains the block header details. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/blockchain.md b/models/evm/descriptions/blocks/blockchain.md new file mode 100644 index 0000000..0b9bee2 --- /dev/null +++ b/models/evm/descriptions/blocks/blockchain.md @@ -0,0 +1,5 @@ +{% docs flowevm_blockchain %} + +The blockchain on which transactions are being confirmed. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/blocks_hash.md b/models/evm/descriptions/blocks/blocks_hash.md new file mode 100644 index 0000000..b6269e3 --- /dev/null +++ b/models/evm/descriptions/blocks/blocks_hash.md @@ -0,0 +1,5 @@ +{% docs flowevm_blocks_hash %} + +The hash of the block header for a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/blocks_nonce.md b/models/evm/descriptions/blocks/blocks_nonce.md new file mode 100644 index 0000000..58958fc --- /dev/null +++ b/models/evm/descriptions/blocks/blocks_nonce.md @@ -0,0 +1,5 @@ +{% docs flowevm_blocks_nonce %} + +Block nonce is a value used during mining to demonstrate proof of work for a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/blocks_table_doc.md b/models/evm/descriptions/blocks/blocks_table_doc.md new file mode 100644 index 0000000..e145e90 --- /dev/null +++ b/models/evm/descriptions/blocks/blocks_table_doc.md @@ -0,0 +1,5 @@ +{% docs flowevm_blocks_table_doc %} + +This table contains block level data for the flowEVM Blockchain. This table can be used to analyze trends at a block level, for example gas fees vs. total transactions over time. For more information on EVM transactions, please see [Etherscan Resources](https://etherscan.io/directory/Learning_Resources/Ethereum) or [The Ethereum Organization](https://ethereum.org/en/developers/docs/blocks/) + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/difficulty.md b/models/evm/descriptions/blocks/difficulty.md new file mode 100644 index 0000000..8b688bc --- /dev/null +++ b/models/evm/descriptions/blocks/difficulty.md @@ -0,0 +1,5 @@ +{% docs flowevm_difficulty %} + +The effort required to mine the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/extra_data.md b/models/evm/descriptions/blocks/extra_data.md new file mode 100644 index 0000000..9a365dc --- /dev/null +++ b/models/evm/descriptions/blocks/extra_data.md @@ -0,0 +1,5 @@ +{% docs flowevm_extra_data %} + +Any data included by the validator for a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/gas_limit.md b/models/evm/descriptions/blocks/gas_limit.md new file mode 100644 index 0000000..feea70a --- /dev/null +++ b/models/evm/descriptions/blocks/gas_limit.md @@ -0,0 +1,5 @@ +{% docs flowevm_gas_limit %} + +Total gas limit provided by all transactions in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/gas_used.md b/models/evm/descriptions/blocks/gas_used.md new file mode 100644 index 0000000..9970c19 --- /dev/null +++ b/models/evm/descriptions/blocks/gas_used.md @@ -0,0 +1,5 @@ +{% docs flowevm_gas_used %} + +Total gas used in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/miner.md b/models/evm/descriptions/blocks/miner.md new file mode 100644 index 0000000..115c282 --- /dev/null +++ b/models/evm/descriptions/blocks/miner.md @@ -0,0 +1,5 @@ +{% docs flowevm_miner %} + +Miner who successfully added a given block to the blockchain. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/network.md b/models/evm/descriptions/blocks/network.md new file mode 100644 index 0000000..4286d44 --- /dev/null +++ b/models/evm/descriptions/blocks/network.md @@ -0,0 +1,5 @@ +{% docs flowevm_network %} + +The network on the blockchain used by a transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/parent_hash.md b/models/evm/descriptions/blocks/parent_hash.md new file mode 100644 index 0000000..a9967b6 --- /dev/null +++ b/models/evm/descriptions/blocks/parent_hash.md @@ -0,0 +1,5 @@ +{% docs flowevm_parent_hash %} + +The hash of the block from which a given block is generated. Also known as the parent block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/receipts_root.md b/models/evm/descriptions/blocks/receipts_root.md new file mode 100644 index 0000000..70516ad --- /dev/null +++ b/models/evm/descriptions/blocks/receipts_root.md @@ -0,0 +1,5 @@ +{% docs flowevm_receipts_root %} + +The root of the state trie. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/sha3_uncles.md b/models/evm/descriptions/blocks/sha3_uncles.md new file mode 100644 index 0000000..eaf652c --- /dev/null +++ b/models/evm/descriptions/blocks/sha3_uncles.md @@ -0,0 +1,5 @@ +{% docs flowevm_sha3_uncles %} + +The mechanism which Ethereum Javascript RLP encodes an empty string. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/size.md b/models/evm/descriptions/blocks/size.md new file mode 100644 index 0000000..486548e --- /dev/null +++ b/models/evm/descriptions/blocks/size.md @@ -0,0 +1,5 @@ +{% docs flowevm_size %} + +Block size, which is determined by a given block's gas limit. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/total_difficulty.md b/models/evm/descriptions/blocks/total_difficulty.md new file mode 100644 index 0000000..dab4ee5 --- /dev/null +++ b/models/evm/descriptions/blocks/total_difficulty.md @@ -0,0 +1,5 @@ +{% docs flowevm_total_difficulty %} + +Total difficulty of the chain at a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/tx_count.md b/models/evm/descriptions/blocks/tx_count.md new file mode 100644 index 0000000..500ec34 --- /dev/null +++ b/models/evm/descriptions/blocks/tx_count.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_count %} + +Total number of transactions within a block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/uncle_blocks.md b/models/evm/descriptions/blocks/uncle_blocks.md new file mode 100644 index 0000000..cdb2ca6 --- /dev/null +++ b/models/evm/descriptions/blocks/uncle_blocks.md @@ -0,0 +1,5 @@ +{% docs flowevm_uncle_blocks %} + +Uncle blocks occur when two blocks are mined and broadcasted at the same time, with the same block number. The block validated across the most nodes will be added to the primary chain, and the other one becomes an uncle block. Miners do receive rewards for uncle blocks. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/event_index.md b/models/evm/descriptions/event_logs/event_index.md new file mode 100644 index 0000000..03583f2 --- /dev/null +++ b/models/evm/descriptions/event_logs/event_index.md @@ -0,0 +1,5 @@ +{% docs flowevm_event_index %} + +Event number within a transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/event_inputs.md b/models/evm/descriptions/event_logs/event_inputs.md new file mode 100644 index 0000000..be19f04 --- /dev/null +++ b/models/evm/descriptions/event_logs/event_inputs.md @@ -0,0 +1,5 @@ +{% docs flowevm_event_inputs %} + +The decoded event inputs for a given event. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/event_name.md b/models/evm/descriptions/event_logs/event_name.md new file mode 100644 index 0000000..3bd263c --- /dev/null +++ b/models/evm/descriptions/event_logs/event_name.md @@ -0,0 +1,5 @@ +{% docs flowevm_event_name %} + +The decoded event name for a given event. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/event_removed.md b/models/evm/descriptions/event_logs/event_removed.md new file mode 100644 index 0000000..6821437 --- /dev/null +++ b/models/evm/descriptions/event_logs/event_removed.md @@ -0,0 +1,5 @@ +{% docs flowevm_event_removed %} + +Whether the event has been removed from the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/event_sig.md b/models/evm/descriptions/event_logs/event_sig.md new file mode 100644 index 0000000..6ab88be --- /dev/null +++ b/models/evm/descriptions/event_logs/event_sig.md @@ -0,0 +1,5 @@ +{% docs flowevm_origin_sig %} + +The function signature of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/log_id_events.md b/models/evm/descriptions/event_logs/log_id_events.md new file mode 100644 index 0000000..154e7ec --- /dev/null +++ b/models/evm/descriptions/event_logs/log_id_events.md @@ -0,0 +1,5 @@ +{% docs flowevm_log_id_events %} + +This is the primary key for this table. This is a concatenation of the transaction hash and the event index at which the event occurred. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/logs_contract_address.md b/models/evm/descriptions/event_logs/logs_contract_address.md new file mode 100644 index 0000000..5c36229 --- /dev/null +++ b/models/evm/descriptions/event_logs/logs_contract_address.md @@ -0,0 +1,5 @@ +{% docs flowevm_logs_contract_address %} + +The address interacted with for a given event. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/logs_contract_name.md b/models/evm/descriptions/event_logs/logs_contract_name.md new file mode 100644 index 0000000..9f228d3 --- /dev/null +++ b/models/evm/descriptions/event_logs/logs_contract_name.md @@ -0,0 +1,5 @@ +{% docs flowevm_logs_contract_name %} + +The name of the contract or token, where possible. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/logs_data.md b/models/evm/descriptions/event_logs/logs_data.md new file mode 100644 index 0000000..2828d5e --- /dev/null +++ b/models/evm/descriptions/event_logs/logs_data.md @@ -0,0 +1,5 @@ +{% docs flowevm_logs_data %} + +The un-decoded event data. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/logs_table_doc.md b/models/evm/descriptions/event_logs/logs_table_doc.md new file mode 100644 index 0000000..4adf6c0 --- /dev/null +++ b/models/evm/descriptions/event_logs/logs_table_doc.md @@ -0,0 +1,5 @@ +{% docs flowevm_logs_table_doc %} + +This table contains flattened event logs from transactions on the flowEVM Blockchain. Transactions may have multiple events, which are denoted by the event index for a transaction hash. Therefore, this table is unique on the combination of transaction hash and event index. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/logs_tx_hash.md b/models/evm/descriptions/event_logs/logs_tx_hash.md new file mode 100644 index 0000000..d6876fd --- /dev/null +++ b/models/evm/descriptions/event_logs/logs_tx_hash.md @@ -0,0 +1,5 @@ +{% docs flowevm_logs_tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. This field will not be unique in this table, as a given transaction can include multiple events. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/origin_from.md b/models/evm/descriptions/event_logs/origin_from.md new file mode 100644 index 0000000..3099c62 --- /dev/null +++ b/models/evm/descriptions/event_logs/origin_from.md @@ -0,0 +1,5 @@ +{% docs flowevm_origin_from %} + +The from address of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/origin_to.md b/models/evm/descriptions/event_logs/origin_to.md new file mode 100644 index 0000000..fbefe39 --- /dev/null +++ b/models/evm/descriptions/event_logs/origin_to.md @@ -0,0 +1,5 @@ +{% docs flowevm_origin_to %} + +The to address of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/topics.md b/models/evm/descriptions/event_logs/topics.md new file mode 100644 index 0000000..d3d13f1 --- /dev/null +++ b/models/evm/descriptions/event_logs/topics.md @@ -0,0 +1,5 @@ +{% docs flowevm_topics %} + +The un-decoded event input topics. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/block_number.md b/models/evm/descriptions/general/block_number.md new file mode 100644 index 0000000..3c665d4 --- /dev/null +++ b/models/evm/descriptions/general/block_number.md @@ -0,0 +1,5 @@ +{% docs flowevm_block_number %} + +Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. + +{% enddocs %} diff --git a/models/evm/descriptions/general/block_timestamp.md b/models/evm/descriptions/general/block_timestamp.md new file mode 100644 index 0000000..4c47d1f --- /dev/null +++ b/models/evm/descriptions/general/block_timestamp.md @@ -0,0 +1,5 @@ +{% docs flowevm_block_timestamp %} + +The date and time at which the block was produced. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/decimals.md b/models/evm/descriptions/general/decimals.md new file mode 100644 index 0000000..a828460 --- /dev/null +++ b/models/evm/descriptions/general/decimals.md @@ -0,0 +1,5 @@ +{% docs flowevm_decimals %} + +The number of decimal places this contract needs adjusted where token values exist. For example, use the decimal field to correctly transform raw amounts in ```fact_token_transfers```. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/deprecation.md b/models/evm/descriptions/general/deprecation.md new file mode 100644 index 0000000..004e578 --- /dev/null +++ b/models/evm/descriptions/general/deprecation.md @@ -0,0 +1,11 @@ +{% docs internal_column %} + +Deprecated. This column is no longer used. Please remove from your query by Jan. 31 2024. + +{% enddocs %} + +{% docs amount_deprecation %} + +This column is being deprecated for standardization purposes on Jan. 31 2024. Please use the equivalent column without the native asset prefix. For example, use `amount` instead of `eth_amount`. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/from_address.md b/models/evm/descriptions/general/from_address.md new file mode 100644 index 0000000..1016bda --- /dev/null +++ b/models/evm/descriptions/general/from_address.md @@ -0,0 +1,5 @@ +{% docs flowevm_from_address %} + +The sending address of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/ingested_at.md b/models/evm/descriptions/general/ingested_at.md new file mode 100644 index 0000000..a17f341 --- /dev/null +++ b/models/evm/descriptions/general/ingested_at.md @@ -0,0 +1,5 @@ +{% docs flowevm_ingested_at %} + +Internal column. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/precise_amounts.md b/models/evm/descriptions/general/precise_amounts.md new file mode 100644 index 0000000..96a815c --- /dev/null +++ b/models/evm/descriptions/general/precise_amounts.md @@ -0,0 +1,17 @@ +{% docs precise_amount_unadjusted %} + +The precise, unadjusted amount of the transaction. This is returned as a string to avoid precision loss. + +{% enddocs %} + +{% docs precise_amount_adjusted %} + +The precise, adjusted amount of the transaction. This is returned as a string to avoid precision loss. + +{% enddocs %} + +{% docs tx_fee_precise %} + +The precise amount of the transaction fee. This is returned as a string to avoid precision loss. + +{% enddocs %} diff --git a/models/evm/descriptions/general/symbol.md b/models/evm/descriptions/general/symbol.md new file mode 100644 index 0000000..22fee0e --- /dev/null +++ b/models/evm/descriptions/general/symbol.md @@ -0,0 +1,5 @@ +{% docs flowevm_symbol %} + +The symbol belonging to the address of the token + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/to_address.md b/models/evm/descriptions/general/to_address.md new file mode 100644 index 0000000..87bf89a --- /dev/null +++ b/models/evm/descriptions/general/to_address.md @@ -0,0 +1,5 @@ +{% docs flowevm_to_address %} + +The receiving address of this transaction. This can be a contract address. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/cumulative_gas_used.md b/models/evm/descriptions/transactions/cumulative_gas_used.md new file mode 100644 index 0000000..f29c1e9 --- /dev/null +++ b/models/evm/descriptions/transactions/cumulative_gas_used.md @@ -0,0 +1,5 @@ +{% docs flowevm_cumulative_gas_used %} + +The total amount of gas used when this transaction was executed in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_block_hash.md b/models/evm/descriptions/transactions/tx_block_hash.md new file mode 100644 index 0000000..850d9f3 --- /dev/null +++ b/models/evm/descriptions/transactions/tx_block_hash.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_block_hash %} + +Block hash is a unique 66-character identifier that is generate when a block is produced. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_fee.md b/models/evm/descriptions/transactions/tx_fee.md new file mode 100644 index 0000000..74f46ae --- /dev/null +++ b/models/evm/descriptions/transactions/tx_fee.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_fee %} + +Amount paid to validate the transaction in ETH. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_gas_limit.md b/models/evm/descriptions/transactions/tx_gas_limit.md new file mode 100644 index 0000000..7069642 --- /dev/null +++ b/models/evm/descriptions/transactions/tx_gas_limit.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_gas_limit %} + +Maximum amount of gas allocated for the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_gas_price.md b/models/evm/descriptions/transactions/tx_gas_price.md new file mode 100644 index 0000000..bea902e --- /dev/null +++ b/models/evm/descriptions/transactions/tx_gas_price.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_gas_price %} + +Cost per unit of gas in Gwei. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_gas_used.md b/models/evm/descriptions/transactions/tx_gas_used.md new file mode 100644 index 0000000..e530435 --- /dev/null +++ b/models/evm/descriptions/transactions/tx_gas_used.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_gas_used %} + +Gas used by transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_hash.md b/models/evm/descriptions/transactions/tx_hash.md new file mode 100644 index 0000000..66f3ba7 --- /dev/null +++ b/models/evm/descriptions/transactions/tx_hash.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_input_data.md b/models/evm/descriptions/transactions/tx_input_data.md new file mode 100644 index 0000000..37f4aa6 --- /dev/null +++ b/models/evm/descriptions/transactions/tx_input_data.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_input_data %} + +This column contains additional data for this transaction, and is commonly used as part of a contract interaction or as a message to the recipient. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_json.md b/models/evm/descriptions/transactions/tx_json.md new file mode 100644 index 0000000..7190ddb --- /dev/null +++ b/models/evm/descriptions/transactions/tx_json.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_json %} + +This JSON column contains the transaction details, including event logs. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_nonce.md b/models/evm/descriptions/transactions/tx_nonce.md new file mode 100644 index 0000000..9b97341 --- /dev/null +++ b/models/evm/descriptions/transactions/tx_nonce.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_nonce %} + +The number of transactions sent from a given address. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_origin_sig.md b/models/evm/descriptions/transactions/tx_origin_sig.md new file mode 100644 index 0000000..57ae164 --- /dev/null +++ b/models/evm/descriptions/transactions/tx_origin_sig.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_origin_sig %} + +The function signature of the contract call. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_position.md b/models/evm/descriptions/transactions/tx_position.md new file mode 100644 index 0000000..15aabdc --- /dev/null +++ b/models/evm/descriptions/transactions/tx_position.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_position %} + +The position of the transaction within the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_status.md b/models/evm/descriptions/transactions/tx_status.md new file mode 100644 index 0000000..982674a --- /dev/null +++ b/models/evm/descriptions/transactions/tx_status.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_status %} + +Status of the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_table_doc.md b/models/evm/descriptions/transactions/tx_table_doc.md new file mode 100644 index 0000000..5f12009 --- /dev/null +++ b/models/evm/descriptions/transactions/tx_table_doc.md @@ -0,0 +1,5 @@ +{% docs flowevm_tx_table_doc %} + +This table contains transaction level data for the flowEVM Blockchain. Each transaction will have a unique transaction hash, along with transactions fees and a ETH value transferred when applicable. Transactions may be native ETH transfers or interactions with contract addresses. For more information, please see [The Ethereum Organization - Transactions](https://ethereum.org/en/developers/docs/transactions/) + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/value.md b/models/evm/descriptions/transactions/value.md new file mode 100644 index 0000000..fcdd1d1 --- /dev/null +++ b/models/evm/descriptions/transactions/value.md @@ -0,0 +1,5 @@ +{% docs flowevm_value %} + +The value transacted in ETH. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/gold/core/core_evm__fact_blocks.sql b/models/evm/gold/core/core_evm__fact_blocks.sql new file mode 100644 index 0000000..23b8073 --- /dev/null +++ b/models/evm/gold/core/core_evm__fact_blocks.sql @@ -0,0 +1,84 @@ +{{ config( + materialized = 'incremental', + unique_key = "fact_blocks_id", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['inserted_timestamp :: DATE', 'ROUND(block_number, -3)'], + persist_docs ={ "relation": true, + "columns": true }, + tags = ['evm'] +) }} + +SELECT + block_number, + block_timestamp, + 'mainnet' AS network, + 'flow' AS blockchain, + transaction_count AS tx_count, + difficulty, + total_difficulty, + extra_data, + gas_limit, + gas_used, + block_hash, + parent_hash, + receipts_root, + sha3_uncles, + SIZE, + uncles AS uncle_blocks, + OBJECT_CONSTRUCT( + 'baseFeePerGas', + base_fee_per_gas, + 'difficulty', + difficulty, + 'extraData', + extra_data, + 'gasLimit', + gas_limit, + 'gasUsed', + gas_used, + 'hash', + block_hash, + 'logsBloom', + logs_bloom, + 'miner', + miner, + 'nonce', + nonce, + 'number', + block_number, + 'parentHash', + parent_hash, + 'receiptsRoot', + receipts_root, + 'sha3Uncles', + sha3_uncles, + 'size', + SIZE, + 'stateRoot', + state_root, + 'timestamp', + block_timestamp, + 'totalDifficulty', + total_difficulty, + 'transactionsRoot', + transactions_root, + 'uncles', + uncles + ) AS block_header_json, + evm_blocks_id AS fact_blocks_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver_evm__blocks') }} + +{% if is_incremental() %} +{{ ref('bronze_evm__blocks') }} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/evm/gold/core/core_evm__fact_blocks.yml b/models/evm/gold/core/core_evm__fact_blocks.yml new file mode 100644 index 0000000..a3c5d1d --- /dev/null +++ b/models/evm/gold/core/core_evm__fact_blocks.yml @@ -0,0 +1,46 @@ +version: 2 +models: + - name: core_evm__fact_blocks + description: '{{ doc("flowevm_blocks_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("flowevm_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("flowevm_block_timestamp") }}' + - name: NETWORK + description: '{{ doc("flowevm_network") }}' + - name: BLOCKCHAIN + description: '{{ doc("flowevm_blockchain") }}' + - name: TX_COUNT + description: '{{ doc("flowevm_tx_count") }}' + - name: DIFFICULTY + description: '{{ doc("flowevm_difficulty") }}' + - name: TOTAL_DIFFICULTY + description: '{{ doc("flowevm_total_difficulty") }}' + - name: EXTRA_DATA + description: '{{ doc("flowevm_extra_data") }}' + - name: GAS_LIMIT + description: '{{ doc("flowevm_gas_limit") }}' + - name: GAS_USED + description: '{{ doc("flowevm_gas_used") }}' + - name: HASH + description: '{{ doc("flowevm_blocks_hash") }}' + - name: PARENT_HASH + description: '{{ doc("flowevm_parent_hash") }}' + - name: RECEIPTS_ROOT + description: '{{ doc("flowevm_receipts_root") }}' + - name: SHA3_UNCLES + description: '{{ doc("flowevm_sha3_uncles") }}' + - name: SIZE + description: '{{ doc("flowevm_size") }}' + - name: UNCLE_BLOCKS + description: '{{ doc("flowevm_uncle_blocks") }}' + - name: BLOCK_HEADER_JSON + description: '{{ doc("flowevm_block_header_json") }}' + - name: FACT_BLOCKS_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/evm/gold/core/core_evm__fact_event_logs.sql b/models/evm/gold/core/core_evm__fact_event_logs.sql new file mode 100644 index 0000000..717b5b3 --- /dev/null +++ b/models/evm/gold/core/core_evm__fact_event_logs.sql @@ -0,0 +1,41 @@ +{{ config( + materialized = 'incremental', + unique_key = "fact_event_logs_id", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['inserted_timestamp :: DATE', 'ROUND(block_number, -3)'], + persist_docs ={ "relation": true, + "columns": true }, + tags = ['evm'] +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + event_index, + contract_address, + topics, + DATA, + event_removed, + tx_status, + _log_id, + evm_logs_id AS fact_event_logs_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver_evm__logs') }} + +{% if is_incremental() %} +{{ ref('bronze_evm__blocks') }} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/evm/gold/core/core_evm__fact_event_logs.yml b/models/evm/gold/core/core_evm__fact_event_logs.yml new file mode 100644 index 0000000..bd77340 --- /dev/null +++ b/models/evm/gold/core/core_evm__fact_event_logs.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: core_evm__fact_event_logs + description: '{{ doc("flowevm_logs_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("flowevm_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("flowevm_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("flowevm_logs_tx_hash") }}' + - name: EVENT_INDEX + description: '{{ doc("flowevm_event_index") }}' + - name: CONTRACT_ADDRESS + description: '{{ doc("flowevm_logs_contract_address") }}' + - name: TOPICS + description: '{{ doc("flowevm_topics") }}' + - name: DATA + description: '{{ doc("flowevm_logs_data") }}' + - name: EVENT_REMOVED + description: '{{ doc("flowevm_event_removed") }}' + - name: _LOG_ID + description: '{{ doc("internal_column") }}' + - name: TX_STATUS + description: '{{ doc("flowevm_tx_status") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("flowevm_origin_sig") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("flowevm_origin_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("flowevm_origin_to") }}' + - name: FACT_EVENT_LOGS_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/evm/gold/core/core_evm__fact_transactions.sql b/models/evm/gold/core/core_evm__fact_transactions.sql new file mode 100644 index 0000000..fe0909e --- /dev/null +++ b/models/evm/gold/core/core_evm__fact_transactions.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'incremental', + unique_key = "fact_transactions_id", + incremental_strategy = 'delete+instert', + cluster_by = ['inserted_timestamp :: DATE', 'ROUND(block_number, -3)'], + persist_docs ={ "relation": true, + "columns": true }, + tags = ['evm'] +) }} +SELECT + block_number, + block_timestamp, + block_hash, + tx_hash, + nonce, + POSITION, + origin_function_signature, + from_address, + to_address, + VALUE, + value_precise_unadj AS value_precise_raw, + value_precise_adj AS value_precise, + tx_fee, + tx_fee_precise, + gas_price_adj AS gas_price, + effective_gas_price, + gas AS gas_limit, + gas_used, + cumulative_gas_used, + input_data, + tx_status AS status, + r, + s, + v, + evm_txs_id AS fact_transactions_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver_evm__transactions') }} + +{% if is_incremental() %} +{{ ref('bronze_evm__blocks') }} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/evm/gold/core/core_evm__fact_transactions.yml b/models/evm/gold/core/core_evm__fact_transactions.yml new file mode 100644 index 0000000..640a48f --- /dev/null +++ b/models/evm/gold/core/core_evm__fact_transactions.yml @@ -0,0 +1,64 @@ +version: 2 +models: + - name: core_evm__fact_transactions + description: '{{ doc("flowevm_tx_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("flowevm_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("flowevm_block_timestamp") }}' + - name: BLOCK_HASH + description: '{{ doc("flowevm_tx_block_hash") }}' + - name: TX_HASH + description: '{{ doc("flowevm_tx_hash") }}' + - name: NONCE + description: '{{ doc("flowevm_tx_nonce") }}' + - name: POSITION + description: '{{ doc("flowevm_tx_position") }}' + - name: FROM_ADDRESS + description: '{{ doc("flowevm_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("flowevm_to_address") }}' + - name: VALUE + description: '{{ doc("flowevm_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: VALUE_PRECISE + description: '{{ doc("precise_amount_adjusted") }}' + - name: TX_FEE + description: '{{ doc("flowevm_tx_fee") }}' + - name: TX_FEE_PRECISE + description: '{{ doc("tx_fee_precise") }}' + - name: GAS_PRICE + description: '{{ doc("flowevm_tx_gas_price") }}' + - name: EFFECTIVE_GAS_PRICE + description: The total base charge plus tip paid for each unit of gas, in Gwei. + - name: GAS_LIMIT + description: '{{ doc("flowevm_tx_gas_limit") }}' + - name: GAS_USED + description: '{{ doc("flowevm_tx_gas_used") }}' + - name: CUMULATIVE_GAS_USED + description: '{{ doc("flowevm_cumulative_gas_used") }}' + - name: MAX_FEE_PER_GAS + description: The maximum fee per gas of the transaction, in Gwei. + - name: MAX_PRIORITY_FEE_PER_GAS + description: The maximum priority fee per gas of the transaction, in Gwei. + - name: STATUS + description: '{{ doc("flowevm_tx_status") }}' + - name: INPUT_DATA + description: '{{ doc("flowevm_tx_input_data") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("flowevm_tx_origin_sig") }}' + - name: R + description: The r value of the transaction signature. + - name: S + description: The s value of the transaction signature. + - name: V + description: The v value of the transaction signature. + - name: FACT_TRANSACTIONS_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__blocks.sql b/models/evm/silver/core/silver_evm__blocks.sql new file mode 100644 index 0000000..a3860de --- /dev/null +++ b/models/evm/silver/core/silver_evm__blocks.sql @@ -0,0 +1,82 @@ +-- depends_on: {{ ref('bronze_evm__blocks') }} +-- depends_on: {{ ref('bronze_evm__FR_blocks') }} +{{ config( + materialized = 'incremental', + unique_key = "evm_blocks_id", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['_inserted_timestamp :: DATE', '_partition_by_block_id'], + tags = ['evm'] +) }} + +SELECT + block_number, + DATA :result :hash :: STRING AS block_hash, + TO_TIMESTAMP( + utils.udf_hex_to_int( + DATA :result :timestamp :: STRING + ) + ) AS block_timestamp, + ARRAY_SIZE( + DATA :result :transactions :: ARRAY + ) AS transaction_count, + utils.udf_hex_to_int( + DATA :result :baseFeePerGas :: STRING + ) AS base_fee_per_gas, + utils.udf_hex_to_int( + DATA :result :difficulty :: STRING + ) AS difficulty, + DATA :result :extraData :: STRING AS extra_data, + utils.udf_hex_to_int( + DATA :result :gasLimit :: STRING + ) AS gas_limit, + utils.udf_hex_to_int( + DATA :result :gasUsed :: STRING + ) AS gas_used, + DATA :result :logsBloom :: STRING AS logs_bloom, + DATA :result :miner :: STRING AS miner, + DATA :result :mixHash :: STRING AS mix_hash, + utils.udf_hex_to_int( + DATA :result :nonce :: STRING + ) AS nonce, + DATA :result :parentHash :: STRING AS parent_hash, + DATA :result :receiptsRoot :: STRING AS receipts_root, + DATA :result :sha3Uncles :: STRING AS sha3_uncles, + utils.udf_hex_to_int( + DATA :result :size :: STRING + ) AS SIZE, + DATA :result :stateRoot :: STRING AS state_root, + ZEROIFNULL( + utils.udf_hex_to_int( + DATA :result :totalDifficulty :: STRING + ) + ) AS total_difficulty, + DATA :result :transactions :: ARRAY AS transactions, + DATA :result :transactionsRoot :: STRING AS transactions_root, + DATA :result :uncles :: ARRAY AS uncles, + partition_key AS _partition_by_block_id, + {{ dbt_utils.generate_surrogate_key( + ['data:result:hash::STRING'] + ) }} AS evm_blocks_id, + _inserted_timestamp, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__blocks') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze_evm__FR_blocks') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY evm_blocks_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/evm/silver/core/silver_evm__logs.sql b/models/evm/silver/core/silver_evm__logs.sql new file mode 100644 index 0000000..294d97a --- /dev/null +++ b/models/evm/silver/core/silver_evm__logs.sql @@ -0,0 +1,194 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + unique_key = "evm_logs_id", + cluster_by = ['_inserted_timestamp :: DATE', '_partition_by_block_id'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['evm'] +) }} + +WITH base AS ( + + SELECT + block_number, + tx_hash, + from_address AS origin_from_address, + to_address AS origin_to_address, + tx_status, + logs, + _inserted_timestamp, + _partition_by_block_id + FROM + {{ ref('silver_evm__receipts') }} + WHERE + ARRAY_SIZE(logs) > 0 + +{% if is_incremental() %} +AND _INSERTED_TIMESTAMP >= ( + SELECT + MAX(_INSERTED_TIMESTAMP) _INSERTED_TIMESTAMP + FROM + {{ this }} +) +{% endif %} +), +flat_logs AS ( + SELECT + block_number, + tx_hash, + origin_from_address, + origin_to_address, + tx_status, + VALUE :address :: STRING AS contract_address, + VALUE :blockHash :: STRING AS block_hash, + VALUE :data :: STRING AS DATA, + utils.udf_hex_to_int( + VALUE :logIndex :: STRING + ) :: INT AS event_index, + VALUE :removed :: BOOLEAN AS event_removed, + VALUE :topics AS topics, + _inserted_timestamp, + _partition_by_block_id + FROM + base, + LATERAL FLATTEN( + input => logs + ) +), +new_records AS ( + SELECT + l.block_number, + txs.block_timestamp, + l.tx_hash, + l.origin_from_address, + l.origin_to_address, + txs.origin_function_signature, + l.tx_status, + l.contract_address, + l.block_hash, + l.data, + l.event_index, + l.event_removed, + l.topics, + l._inserted_timestamp, + CASE + WHEN txs.block_timestamp IS NULL + OR txs.origin_function_signature IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l._partition_by_block_id + FROM + flat_logs l + LEFT OUTER JOIN {{ ref('silver_evm__transactions') }} + txs + ON l.block_number = txs.block_number + AND l.tx_hash = txs.tx_hash + +{% if is_incremental() %} +AND txs._INSERTED_TIMESTAMP >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + txs.block_timestamp, + t.tx_hash, + t.origin_from_address, + t.origin_to_address, + txs.origin_function_signature, + t.tx_status, + t.contract_address, + t.block_hash, + t.data, + t.event_index, + t.event_removed, + t.topics, + GREATEST( + t._inserted_timestamp, + txs._inserted_timestamp + ) AS _inserted_timestamp, + _log_id, + FALSE AS is_pending, + t._partition_by_block_id + FROM + {{ this }} + t + INNER JOIN {{ ref('silver_evm__transactions') }} + txs USING ( + block_number, + tx_hash + ) + WHERE + t.is_pending +) +{% endif %}, +FINAL AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_status, + contract_address, + block_hash, + DATA, + event_index, + event_removed, + topics, + _inserted_timestamp, + _log_id, + is_pending, + _partition_by_block_id + FROM + new_records + +{% if is_incremental() %} +UNION +SELECT + block_number, + block_timestamp, + tx_hash, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_status, + contract_address, + block_hash, + DATA, + event_index, + event_removed, + topics, + _inserted_timestamp, + _log_id, + is_pending, + _partition_by_block_id +FROM + missing_data +{% endif %} +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS evm_logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_hash, event_index +ORDER BY + _inserted_timestamp DESC, is_pending ASC)) = 1 diff --git a/models/evm/silver/core/silver_evm__receipts.sql b/models/evm/silver/core/silver_evm__receipts.sql new file mode 100644 index 0000000..4ef0416 --- /dev/null +++ b/models/evm/silver/core/silver_evm__receipts.sql @@ -0,0 +1,122 @@ +-- depends_on: {{ ref('bronze_evm__receipts') }} +-- depends_on: {{ ref('bronze_evm__FR_receipts') }} +{{ config( + materialized = 'incremental', + unique_key = "evm_receipts_id", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['_inserted_timestamp :: DATE', '_partition_by_block_id'], + tags = ['evm'] +) }} + +WITH receipts AS ( + + SELECT + block_number, + DATA, + partition_key AS _partition_by_block_id, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__receipts') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze_evm__FR_receipts') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 +), +FINAL AS ( +SELECT + block_number, + COALESCE( + VALUE :PrecompiledCalls :: STRING, + VALUE :precompiledCalls :: STRING + ) AS precompiled_calls, + VALUE :blobGasPrice :: INT AS blob_gas_price, + VALUE :blockHash :: STRING AS block_hash, + VALUE :blockNumber :: INT AS blockNumber, + VALUE :contractAddress :: STRING AS contract_address, + VALUE :cumulativeGasUsed :: INT AS cumulative_gas_used, + VALUE :effectiveGasPrice :: INT AS effective_gas_price_unadj, + VALUE :from :: STRING AS from_address, + VALUE :effectiveGasPrice :: INT / pow( + 10, + 9 + ) AS effective_gas_price_adj, + ZEROIFNULL( + VALUE :gasUsed :: INT + ) AS gas_used, + VALUE :logs :: ARRAY AS logs, + VALUE :logsBloom :: STRING AS logs_bloom, + VALUE :revertReason :: STRING AS revert_reason, + VALUE :root :: STRING AS root, + VALUE :status :: INT AS status, + VALUE :status :: INT = 1 AS tx_succeeded, + IFF( + VALUE :status :: INT = 1, + 'SUCCESS', + 'FAIL' + ) AS tx_status, + VALUE :transactionHash :: STRING AS tx_hash, + VALUE :transactionIndex :: INT AS tx_index, + CASE + WHEN block_number <> blockNumber THEN NULL + ELSE VALUE :transactionIndex :: INT + END AS POSITION, + VALUE :type :: STRING AS receipt_type, + VALUE :to :: STRING AS to_address, + _partition_by_block_id, + _inserted_timestamp +FROM + receipts, + LATERAL FLATTEN ( + DATA :result :: variant + ) +) +SELECT + block_number, + precompiled_calls, + blob_gas_price, + block_hash, + blockNumber, + contract_address, + cumulative_gas_used, + effective_gas_price_unadj, + effective_gas_price_adj, + from_address, + gas_used, + logs, + logs_bloom, + revert_reason, + root, + status, + tx_succeeded, + tx_status, + tx_hash, + tx_index, + POSITION, + receipt_type, + to_address, + _partition_by_block_id, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number', 'tx_hash'] + ) }} AS evm_receipts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL +WHERE + tx_hash IS NOT NULL + AND POSITION IS NOT NULL diff --git a/models/evm/silver/core/silver_evm__traces.sql b/models/evm/silver/core/silver_evm__traces.sql new file mode 100644 index 0000000..c6c6cfc --- /dev/null +++ b/models/evm/silver/core/silver_evm__traces.sql @@ -0,0 +1,76 @@ +-- depends_on: {{ ref('bronze_evm__traces') }} +-- depends_on: {{ ref('bronze_evm__FR_traces') }} +{{ config( + materialized = 'incremental', + unique_key = "evm_traces_id", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['_inserted_timestamp :: DATE', '_partition_by_block_id'], + tags = ['evm'] +) }} + +WITH traces AS ( + + SELECT + block_number, + DATA, + partition_key AS _partition_by_block_id, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__traces') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze_evm__FR_traces') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 +), +flatten_traces AS ( + SELECT + block_number, + INDEX AS array_index, + VALUE :: variant AS trace_response, + _partition_by_block_id, + _inserted_timestamp + FROM + traces, + LATERAL FLATTEN ( + DATA :result :: variant + ) +) +SELECT + block_number, + array_index, + trace_response :from :: STRING AS from_address, + utils.udf_hex_to_int( + trace_response :gas :: STRING + ) AS gas, + utils.udf_hex_to_int( + trace_response :gasUsed :: STRING + ) AS gas_used, + trace_response :input :: STRING AS input, + trace_response :to :: STRING AS to_address, + trace_response :type :: STRING AS trace_type, + utils.udf_hex_to_int( + trace_response :value :: STRING + ) AS VALUE, + {{ dbt_utils.generate_surrogate_key( + ['block_number', 'array_index'] + ) }} AS evm_traces_id, + _partition_by_block_id, + _inserted_timestamp, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + flatten_traces diff --git a/models/evm/silver/core/silver_evm__transactions.sql b/models/evm/silver/core/silver_evm__transactions.sql new file mode 100644 index 0000000..9a7308c --- /dev/null +++ b/models/evm/silver/core/silver_evm__transactions.sql @@ -0,0 +1,333 @@ +{{ config( + materialized = 'incremental', + unique_key = "evm_txs_id", + incremental_strategy = 'delete+insert', + cluster_by = ['block_number'], + tags = ['evm'] +) }} + +WITH tx_array AS ( + + SELECT + block_number, + block_hash, + block_timestamp, + transactions, + _partition_by_block_id, + _inserted_timestamp + FROM + {{ ref('silver_evm__blocks') }} + WHERE + transaction_count > 0 + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) modified_timestamp + FROM + {{ this }} +) +{% endif %} +), +flatten_txs AS ( + SELECT + block_number, + block_hash, + block_timestamp, + INDEX AS array_index, + VALUE :: variant AS tx_response, + _partition_by_block_id, + _inserted_timestamp + FROM + tx_array, + LATERAL FLATTEN (transactions) +), +base_tx AS ( + SELECT + block_number, + block_hash, + block_timestamp, + utils.udf_hex_to_int( + tx_response :blockNumber :: STRING + ) AS blockNumber, + utils.udf_hex_to_int( + tx_response :chainId :: STRING + ) AS chain_id, + tx_response :from :: STRING AS from_address, + utils.udf_hex_to_int( + tx_response :gas :: STRING + ) AS gas, + utils.udf_hex_to_int( + tx_response :gasPrice :: STRING + ) AS gas_price_unadj, + ZEROIFNULL(gas_price_unadj / pow(10, 9)) AS gas_price_adj, + tx_response :hash :: STRING AS tx_hash, + tx_response :input :: STRING AS input_data, + SUBSTR( + input_data, + 1, + 10 + ) AS origin_function_signature, + -- note, no maxFeePerGas or maxPriorityFeePerGas + utils.udf_hex_to_int( + tx_response :nonce :: STRING + ) AS nonce, + utils.udf_hex_to_int( + tx_response :r :: STRING + ) AS r, + utils.udf_hex_to_int( + tx_response :s :: STRING + ) AS s, + -- note, no sourceHash + tx_response :to :: STRING AS to_address, + utils.udf_hex_to_int( + tx_response :transactionIndex :: STRING + ) AS POSITION, + utils.udf_hex_to_int( + tx_response :type :: STRING + ) AS tx_type, + utils.udf_hex_to_int( + tx_response :v :: STRING + ) AS v, + utils.udf_hex_to_int( + tx_response :value :: STRING + ) AS value_precise_unadj, + value_precise_unadj / pow( + 10, + 18 + ) AS value_precise_adj, + value_precise_adj :: FLOAT AS VALUE, + -- note, no yParity + _partition_by_block_id, + _inserted_timestamp + FROM + flatten_txs +), +new_records AS ( + SELECT + t.block_number, + t.block_hash, + t.chain_id, + t.from_address, + t.gas, + t.gas_price_unadj, + t.gas_price_adj, + t.tx_hash, + t.input_data, + t.origin_function_signature, + t.nonce, + t.r, + t.s, + t.to_address, + t.position, + t.tx_type, + t.v, + t.value_precise_unadj, + t.value_precise_adj, + t.value, + t.block_timestamp, + r.tx_status IS NULL AS is_pending, + r.gas_used, + r.gas_used * t.gas_price_unadj :: bigint / pow( + 10, + 18 + ) AS tx_fee_precise, + ZEROIFNULL( + tx_fee_precise :: FLOAT + ) AS tx_fee, + r.tx_succeeded, + r.tx_status, + r.cumulative_gas_used, + r.effective_gas_price_adj AS effective_gas_price, + r.receipt_type, + t._inserted_timestamp, + t._partition_by_block_id + FROM + base_tx t + LEFT OUTER JOIN {{ ref('silver_evm__receipts') }} + r + ON t.block_number = r.block_number + AND t.tx_hash = r.tx_hash + +{% if is_incremental() %} +AND r._INSERTED_TIMESTAMP >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +), + +{% if is_incremental() %} +missing_data AS ( + SELECT + t.block_number, + t.block_hash, + t.chain_id, + t.from_address, + t.gas, + t.gas_price_adj, + t.gas_price_unadj, + t.tx_hash, + t.input_data, + t.origin_function_signature, + t.nonce, + t.r, + t.s, + t.to_address, + t.position, + t.tx_type, + t.v, + t.value_precise_unadj, + t.value_precise_adj, + t.value, + t.block_timestamp, + FALSE AS is_pending, + r.gas_used, + r.tx_succeeded, + r.tx_status, + r.cumulative_gas_used, + r.effective_gas_price_adj AS effective_gas_price, + r.gas_used * t.gas_price_unadj :: bigint / pow( + 10, + 18 + ) AS tx_fee_precise_heal, + ZEROIFNULL( + tx_fee_precise_heal :: FLOAT + ) AS tx_fee, + r.receipt_type, + GREATEST( + t._inserted_timestamp, + r._inserted_timestamp + ) AS _inserted_timestamp, + t._partition_by_block_id + FROM + {{ this }} + t + INNER JOIN {{ ref('silver_evm__receipts') }} + r + ON t.tx_hash = r.tx_hash + AND t.block_number = r.block_number + WHERE + t.is_pending +), +{% endif %} + +FINAL AS ( + SELECT + block_number, + block_hash, + chain_id, + from_address, + gas, + gas_price_adj, + gas_price_unadj, + tx_hash, + input_data, + origin_function_signature, + nonce, + r, + s, + to_address, + POSITION, + tx_type, + v, + VALUE, + value_precise_unadj, + value_precise_adj, + block_timestamp, + is_pending, + gas_used, + tx_succeeded, + tx_status, + cumulative_gas_used, + effective_gas_price, + tx_fee, + tx_fee_precise, + receipt_type, + _inserted_timestamp, + _partition_by_block_id + FROM + new_records + +{% if is_incremental() %} +UNION +SELECT + block_number, + block_hash, + chain_id, + from_address, + gas, + gas_price_adj, + gas_price_unadj, + tx_hash, + input_data, + origin_function_signature, + nonce, + r, + s, + to_address, + POSITION, + tx_type, + v, + VALUE, + value_precise_unadj, + value_precise_adj, + block_timestamp, + is_pending, + gas_used, + tx_succeeded, + tx_status, + cumulative_gas_used, + effective_gas_price, + tx_fee, + tx_fee_precise_heal AS tx_fee_precise, + receipt_type, + _inserted_timestamp, + _partition_by_block_id +FROM + missing_data +{% endif %} +) +SELECT + block_number, + block_hash, + chain_id, + from_address, + gas, + gas_price_adj, + gas_price_unadj, + tx_hash, + input_data, + origin_function_signature, + nonce, + r, + s, + to_address, + POSITION, + tx_type AS TYPE, + v, + VALUE, + value_precise_unadj, + value_precise_adj, + block_timestamp, + is_pending, + gas_used, + tx_succeeded, + tx_status, + cumulative_gas_used, + effective_gas_price, + tx_fee, + tx_fee_precise, + receipt_type AS tx_type, + _inserted_timestamp, + _partition_by_block_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS evm_txs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL diff --git a/models/evm/silver/streamline/_evm_block_lookback.sql b/models/evm/silver/streamline/_evm_block_lookback.sql new file mode 100644 index 0000000..1ee4149 --- /dev/null +++ b/models/evm/silver/streamline/_evm_block_lookback.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = "ephemeral" +) }} + +SELECT + MIN(block_number) AS block_number +FROM + {{ ref("silver_evm__blocks") }} +WHERE + block_timestamp >= DATEADD('hour', -72, TRUNCATE(SYSDATE(), 'HOUR')) + AND block_timestamp < DATEADD('hour', -71, TRUNCATE(SYSDATE(), 'HOUR')) diff --git a/models/evm/silver/streamline/complete/streamline__complete_get_evm_blocks.sql b/models/evm/silver/streamline/complete/streamline__complete_get_evm_blocks.sql new file mode 100644 index 0000000..2d4ea66 --- /dev/null +++ b/models/evm/silver/streamline/complete/streamline__complete_get_evm_blocks.sql @@ -0,0 +1,43 @@ +-- depends_on: {{ ref('bronze_evm__blocks') }} +-- depends_on: {{ ref('bronze_evm__FR_blocks') }} +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_complete_evm'] +) }} + +SELECT + block_number, + utils.udf_hex_to_int(DATA :result :number :: STRING) as blockNumber, + partition_key, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number::STRING'] + ) }} AS complete_evm_blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__blocks') }} +WHERE + _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01' :: timestamp_ntz + ) +{% else %} + {{ ref('bronze_evm__FR_blocks') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/evm/silver/streamline/complete/streamline__complete_get_evm_receipts.sql b/models/evm/silver/streamline/complete/streamline__complete_get_evm_receipts.sql new file mode 100644 index 0000000..07d101e --- /dev/null +++ b/models/evm/silver/streamline/complete/streamline__complete_get_evm_receipts.sql @@ -0,0 +1,42 @@ +-- depends_on: {{ ref('bronze_evm__receipts') }} +-- depends_on: {{ ref('bronze_evm__FR_receipts') }} +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_complete_evm'] +) }} + +SELECT + block_number, + partition_key, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number::STRING'] + ) }} AS complete_evm_receipts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__receipts') }} +WHERE + _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01' :: timestamp_ntz + ) +{% else %} + {{ ref('bronze_evm__FR_receipts') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/evm/silver/streamline/complete/streamline__complete_get_evm_traces.sql b/models/evm/silver/streamline/complete/streamline__complete_get_evm_traces.sql new file mode 100644 index 0000000..12ff13c --- /dev/null +++ b/models/evm/silver/streamline/complete/streamline__complete_get_evm_traces.sql @@ -0,0 +1,44 @@ +-- depends_on: {{ ref('bronze_evm__traces') }} +-- depends_on: {{ ref('bronze_evm__FR_traces') }} +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_complete_evm'] +) }} + +SELECT + block_number, + partition_key, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number::STRING'] + ) }} AS complete_evm_traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__traces') }} +WHERE + _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01' :: timestamp_ntz + ) +{% else %} + {{ ref('bronze_evm__FR_traces') }} +WHERE + TRUE +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/evm/silver/streamline/realtime/streamline__get_evm_blocks_realtime.sql b/models/evm/silver/streamline/realtime/streamline__get_evm_blocks_realtime.sql new file mode 100644 index 0000000..17ad7db --- /dev/null +++ b/models/evm/silver/streamline/realtime/streamline__get_evm_blocks_realtime.sql @@ -0,0 +1,89 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_blocks", + "sql_limit" :"25000", + "producer_batch_size" :"5000", + "worker_batch_size" :"1000", + "sql_source" :"{{this.identifier}}" } + ), + tags = ['streamline_realtime_evm'] +) }} + +WITH last_3_days AS ( + + SELECT + ZEROIFNULL(block_number) AS block_number + FROM + {{ ref("_evm_block_lookback") }} +), +tbl AS ( + SELECT + block_number + FROM + {{ ref('streamline__evm_blocks') }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref('streamline__complete_get_evm_blocks') }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) + AND blockNumber IS NOT NULL +) +SELECT + block_number, + DATE_PART(epoch_second, SYSDATE()) :: STRING AS request_timestamp, + '{{ invocation_id }}' AS _invocation_id, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'id', + block_number, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT( + utils.udf_int_to_hex(block_number), + TRUE -- Include transactions + ) + ), + 'Vault/{{ target.name }}/flow/evm/mainnet' + ) AS request +FROM + tbl +ORDER BY + block_number DESC diff --git a/models/evm/silver/streamline/realtime/streamline__get_evm_receipts_realtime.sql b/models/evm/silver/streamline/realtime/streamline__get_evm_receipts_realtime.sql new file mode 100644 index 0000000..7c1ff2a --- /dev/null +++ b/models/evm/silver/streamline/realtime/streamline__get_evm_receipts_realtime.sql @@ -0,0 +1,99 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_receipts", + "sql_limit" :"25000", + "producer_batch_size" :"5000", + "worker_batch_size" :"1000", + "sql_source" :"{{this.identifier}}" } + ), + tags = ['streamline_realtime_evm'] +) }} + +WITH last_3_days AS ( + + SELECT + ZEROIFNULL(block_number) AS block_number + FROM + {{ ref("_evm_block_lookback") }} +), +tbl AS ( + + SELECT + block_number + FROM + {{ ref('streamline__evm_blocks') }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref('streamline__complete_get_evm_receipts') }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) +), +ready_blocks AS ( + SELECT + block_number + FROM + tbl + UNION ALL + SELECT + block_number + FROM + {{ ref("_missing_receipts") }} +) +SELECT + block_number, + DATE_PART(epoch_second, SYSDATE())::STRING AS request_timestamp, + '{{ invocation_id }}' AS _invocation_id, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'id', + block_number, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockReceipts', + 'params', + ARRAY_CONSTRUCT( + utils.udf_int_to_hex(block_number) + ) + ), + 'Vault/{{ target.name }}/flow/evm/mainnet' + ) AS request +FROM + ready_blocks +ORDER BY + block_number DESC diff --git a/models/evm/silver/streamline/realtime/streamline__get_evm_traces_realtime.sql b/models/evm/silver/streamline/realtime/streamline__get_evm_traces_realtime.sql new file mode 100644 index 0000000..6d5e3fe --- /dev/null +++ b/models/evm/silver/streamline/realtime/streamline__get_evm_traces_realtime.sql @@ -0,0 +1,104 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_traces", + "sql_limit" :"25000", + "producer_batch_size" :"5000", + "worker_batch_size" :"1000", + "sql_source" :"{{this.identifier}}" } + ), + tags = ['streamline_realtime_evm'] +) }} + + +WITH last_3_days AS ( + + SELECT + ZEROIFNULL(block_number) AS block_number + FROM + {{ ref("_evm_block_lookback") }} +), +tbl AS ( + + SELECT + block_number + FROM + {{ ref('streamline__evm_blocks') }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref('streamline__complete_get_evm_traces') }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) +), +ready_blocks AS ( + SELECT + block_number + FROM + tbl + {# UNION ALL + SELECT + block_number + FROM + {{ ref("_missing_traces") }} #} +) +SELECT + block_number, + DATE_PART(epoch_second, SYSDATE())::STRING AS request_timestamp, + '{{ invocation_id }}' AS _invocation_id, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'id', + block_number, + 'jsonrpc', + '2.0', + 'method', + 'debug_traceBlockByNumber', + 'params', + ARRAY_CONSTRUCT( + utils.udf_int_to_hex(block_number), + OBJECT_CONSTRUCT( + 'tracer', 'callTracer', + 'timeout', '30s' + ) + ) + ), + 'Vault/{{ target.name }}/flow/evm/mainnet' + ) AS request +FROM + ready_blocks +ORDER BY + block_number DESC diff --git a/models/evm/silver/streamline/retry/_missing_receipts.sql b/models/evm/silver/streamline/retry/_missing_receipts.sql new file mode 100644 index 0000000..aa8c1dc --- /dev/null +++ b/models/evm/silver/streamline/retry/_missing_receipts.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +) +SELECT + DISTINCT t.block_number AS block_number +FROM + {{ ref("silver_evm__transactions") }} + t + LEFT JOIN {{ ref("silver_evm__receipts") }} + r USING ( + block_number, + block_hash, + tx_hash + ) +WHERE + r.tx_hash IS NULL + AND t.block_number >= ( + SELECT + block_number + FROM + lookback + ) + AND t.block_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND ( + r._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + OR r._inserted_timestamp IS NULL) diff --git a/models/evm/silver/streamline/retry/_missing_traces.sql b/models/evm/silver/streamline/retry/_missing_traces.sql new file mode 100644 index 0000000..80b0935 --- /dev/null +++ b/models/evm/silver/streamline/retry/_missing_traces.sql @@ -0,0 +1,31 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +) +SELECT + DISTINCT tx.block_number block_number +FROM + {{ ref("silver_evm__transactions") }} + tx + LEFT JOIN {{ ref("silver_evm__traces") }} + tr + ON tx.block_number = tr.block_number + AND tx.tx_hash = tr.tx_hash +WHERE + tx.block_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND tr.block_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND tr.tx_hash IS NULL + AND tx.block_number >= ( + SELECT + block_number + FROM + lookback + ) + AND tr.block_timestamp IS NOT NULL diff --git a/models/evm/silver/streamline/streamline__evm_blocks.sql b/models/evm/silver/streamline/streamline__evm_blocks.sql new file mode 100644 index 0000000..5bf1df6 --- /dev/null +++ b/models/evm/silver/streamline/streamline__evm_blocks.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = "view", + tags = ['streamline_realtime_evm'] +) }} + +{% if execute %} + {% set height = run_query("SELECT streamline.udf_get_evm_chainhead()") %} + {% set block_number = height.columns [0].values() [0] %} +{% else %} + {% set block_number = 0 %} +{% endif %} + +SELECT + _id AS block_number +FROM + {{ source( + 'silver_crosschain', + 'number_sequence' + ) }} +WHERE + _id <= {{ block_number }} diff --git a/models/sources.yml b/models/sources.yml index 7b8ae35..ae43f71 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -122,6 +122,10 @@ sources: - name: testnet_collections - name: testnet_transactions - name: testnet_transaction_results + - name: evm_blocks + - name: evm_receipts + - name: evm_traces + - name: crosschain_silver database: crosschain schema: silver