diff --git a/.github/workflows/dbt_run_evm.yml b/.github/workflows/dbt_run_evm.yml index 9aa903f..4ee1db5 100644 --- a/.github/workflows/dbt_run_evm.yml +++ b/.github/workflows/dbt_run_evm.yml @@ -43,4 +43,4 @@ jobs: - name: Update evm tables run: | - dbt run -s tag:evm + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -s tag:evm tag:streamline_decoded_logs_realtime tag:streamline_decoded_logs_complete diff --git a/.github/workflows/dbt_run_evm_decoded_logs.yml b/.github/workflows/dbt_run_evm_decoded_logs.yml new file mode 100644 index 0000000..ba59f18 --- /dev/null +++ b/.github/workflows/dbt_run_evm_decoded_logs.yml @@ -0,0 +1,46 @@ +name: dbt_run_evm_decoded_logs +run-name: dbt_run_evm_decoded_logs + +on: + workflow_dispatch: + schedule: + # Every hour on minute 45(see https://crontab.guru) + - cron: "45 * * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Update decoded tables + run: | + dbt run -s tag:evm_decoded_logs \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_abis.yml b/.github/workflows/dbt_run_scheduled_abis.yml new file mode 100644 index 0000000..087e1c3 --- /dev/null +++ b/.github/workflows/dbt_run_scheduled_abis.yml @@ -0,0 +1,50 @@ +name: dbt_run_scheduled_abis +run-name: dbt_run_scheduled_abis + +on: + workflow_dispatch: + schedule: + # Run at 23:27 UTC every day + - cron: "27 23 * * *" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run ABI Models + run: | + dbt run -m models/evm/silver/abis + + - name: Kick off decoded logs history, if there are new ABIs from users + run: | + dbt run-operation run_decoded_logs_history \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoded_logs_history.yml b/.github/workflows/dbt_run_streamline_decoded_logs_history.yml new file mode 100644 index 0000000..c7da94b --- /dev/null +++ b/.github/workflows/dbt_run_streamline_decoded_logs_history.yml @@ -0,0 +1,50 @@ +name: dbt_run_streamline_decoded_logs_history +run-name: dbt_run_streamline_decoded_logs_history + +on: + workflow_dispatch: + schedule: + # Run at 22:43 UTC every Saturday + - cron: "43 22 * * 6" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Update complete table + run: | + dbt run -m models/evm/streamline/decoder/streamline__decoded_logs_complete.sql + + - name: Decode historical logs + run: | + dbt run-operation decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}' \ No newline at end of file diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index 19e895d..2646e64 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -45,7 +45,7 @@ jobs: - name: Run DBT Jobs run: | - dbt test -s "flow_models,models/gold" tag:observability tag:evm tag:evm_gap_test --vars '{"TEST_RANGE": True, "START_HEIGHT": ${{ vars.STREAMLINE_START_BLOCK }}, "END_HEIGHT": 100000000}' + dbt test -s "flow_models,models/gold" tag:observability tag:evm_gap_test --vars '{"TEST_RANGE": True, "START_HEIGHT": ${{ vars.STREAMLINE_START_BLOCK }}, "END_HEIGHT": 100000000}' continue-on-error: true - name: Log test results diff --git a/.github/workflows/dbt_test_recent_evm.yml b/.github/workflows/dbt_test_recent_evm.yml new file mode 100644 index 0000000..ce75d9d --- /dev/null +++ b/.github/workflows/dbt_test_recent_evm.yml @@ -0,0 +1,50 @@ +name: dbt_test_recent_evm +run-name: dbt_test_recent_evm + +on: + workflow_dispatch: + schedule: + # Run at 0500 UTC daily (see https://crontab.guru) + - cron: "0 5 * * *" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Build Recent Testing Views + run: | + dbt run -m "flow_models,tag:recent_evm_test" + + - name: Run Recent Tests + run: | + dbt test -m "flow_models,tag:recent_evm_test" \ No newline at end of file diff --git a/macros/evm/decoded_logs_history.sql b/macros/evm/decoded_logs_history.sql new file mode 100644 index 0000000..6b7b81d --- /dev/null +++ b/macros/evm/decoded_logs_history.sql @@ -0,0 +1,129 @@ +{% macro decoded_logs_history(backfill_mode=false) %} + + {%- set params = { + "sql_limit": var("DECODED_LOGS_HISTORY_SQL_LIMIT", 8000000), + "producer_batch_size": var("DECODED_LOGS_HISTORY_PRODUCER_BATCH_SIZE", 400000), + "worker_batch_size": var("DECODED_LOGS_HISTORY_WORKER_BATCH_SIZE", 100000) + } -%} + + {% set wait_time = var("DECODED_LOGS_HISTORY_WAIT_TIME", 60) %} + + {% set find_months_query %} + SELECT + DISTINCT date_trunc('month', block_timestamp)::date as month + FROM {{ ref('core_evm__fact_blocks') }} + ORDER BY month ASC + {% endset %} + + {% set results = run_query(find_months_query) %} + + {% if execute %} + {% set months = results.columns[0].values() %} + + {% for month in months %} + {% set view_name = 'decoded_logs_history_' ~ month.strftime('%Y_%m') %} + + {% set create_view_query %} + create or replace view streamline.{{view_name}} as ( + WITH target_blocks AS ( + SELECT + block_number + FROM {{ ref('core_evm__fact_blocks') }} + WHERE date_trunc('month', block_timestamp) = '{{month}}'::timestamp + ), + new_abis AS ( + SELECT + abi, + parent_contract_address, + event_signature, + start_block, + end_block + FROM {{ ref('silver_evm__complete_event_abis') }} + {% if not backfill_mode %} + WHERE inserted_timestamp > dateadd('day', -30, sysdate()) + {% endif %} + ), + existing_logs_to_exclude AS ( + SELECT _log_id + FROM {{ ref('streamline__decoded_logs_complete') }} l + INNER JOIN target_blocks b using (block_number) + ), + candidate_logs AS ( + SELECT + l.block_number, + l.tx_hash, + l.event_index, + l.contract_address, + l.topics, + l.data, + concat(l.tx_hash::string, '-', l.event_index::string) as _log_id + FROM target_blocks b + INNER JOIN {{ ref('core_evm__fact_event_logs') }} l using (block_number) + WHERE l.tx_succeeded and date_trunc('month', l.block_timestamp) = '{{month}}'::timestamp + ) + SELECT + l.block_number, + l._log_id, + A.abi, + OBJECT_CONSTRUCT( + 'topics', l.topics, + 'data', l.data, + 'address', l.contract_address + ) AS data + FROM candidate_logs l + INNER JOIN new_abis A + ON A.parent_contract_address = l.contract_address + AND A.event_signature = l.topics[0]::STRING + AND l.block_number BETWEEN A.start_block AND A.end_block + WHERE NOT EXISTS ( + SELECT 1 + FROM existing_logs_to_exclude e + WHERE e._log_id = l._log_id + ) + LIMIT {{ params.sql_limit }} + ) + {% endset %} + + {# Create the view #} + {% do run_query(create_view_query) %} + {{ log("Created view for month " ~ month.strftime('%Y-%m'), info=True) }} + + {% if var("STREAMLINE_INVOKE_STREAMS", false) %} + {# Check if rows exist first #} + {% set check_rows_query %} + SELECT EXISTS(SELECT 1 FROM streamline.{{view_name}} LIMIT 1) + {% endset %} + + {% set results = run_query(check_rows_query) %} + {% set has_rows = results.columns[0].values()[0] %} + + {% if has_rows %} + {# Invoke streamline, if rows exist to decode #} + {% set decode_query %} + SELECT + streamline.udf_bulk_decode_logs_v2( + PARSE_JSON( + $${ "external_table": "decoded_logs", + "producer_batch_size": {{ params.producer_batch_size }}, + "sql_limit": {{ params.sql_limit }}, + "sql_source": "{{view_name}}", + "worker_batch_size": {{ params.worker_batch_size }} }$$ + ) + ); + {% endset %} + + {% do run_query(decode_query) %} + {{ log("Triggered decoding for month " ~ month.strftime('%Y-%m'), info=True) }} + + {# Call wait since we actually did some decoding #} + {% do run_query("call system$wait(" ~ wait_time ~ ")") %} + {{ log("Completed wait after decoding for month " ~ month.strftime('%Y-%m'), info=True) }} + {% else %} + {{ log("No rows to decode for month " ~ month.strftime('%Y-%m'), info=True) }} + {% endif %} + {% endif %} + + {% endfor %} + {% endif %} + +{% endmacro %} \ No newline at end of file diff --git a/macros/evm/run_decoded_logs_history.sql b/macros/evm/run_decoded_logs_history.sql new file mode 100644 index 0000000..5850eb9 --- /dev/null +++ b/macros/evm/run_decoded_logs_history.sql @@ -0,0 +1,31 @@ +{% macro run_decoded_logs_history() %} + +{% set blockchain = var('GLOBAL_PROD_DB_NAME','').lower() %} + +{% set check_for_new_user_abis_query %} + select 1 + from {{ ref('silver_evm__user_verified_abis') }} + where _inserted_timestamp::date = sysdate()::date + and dayname(sysdate()) <> 'Sat' +{% endset %} + +{% set results = run_query(check_for_new_user_abis_query) %} + +{% if execute %} + {% set new_user_abis = results.columns[0].values()[0] %} + + {% if new_user_abis %} + {% set invoke_workflow_query %} + SELECT + github_actions.workflow_dispatches( + 'FlipsideCrypto', + 'flow-models', + 'dbt_run_streamline_decoded_logs_history.yml', + NULL + ) + {% endset %} + + {% do run_query(invoke_workflow_query) %} + {% endif %} +{% endif %} +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql index 88116ec..5a97a05 100644 --- a/macros/streamline/streamline_udfs.sql +++ b/macros/streamline/streamline_udfs.sql @@ -94,11 +94,11 @@ {% macro create_udf_bulk_decode_logs() %} CREATE - OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_decode_logs( + OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_decode_logs_v2( json OBJECT ) returns ARRAY api_integration = {% if target.name == "prod" %} - aws_flow_evm_api_prod AS 'https://rajpkbgko9.execute-api.us-east-1.amazonaws.com/prod/bulk_decode_logs' + AWS_FLOW_EVM_API_PROD AS 'https://rajpkbgko9.execute-api.us-east-1.amazonaws.com/prod/bulk_decode_logs' {% else %} - aws_flow_evm_api_dev AS'https://pfv9lhg3kg.execute-api.us-east-1.amazonaws.com/stg/bulk_decode_logs' + AWS_FLOW_EVM_API_DEV AS'https://pfv9lhg3kg.execute-api.us-east-1.amazonaws.com/stg/bulk_decode_logs' {%- endif %}; {% endmacro %} \ No newline at end of file diff --git a/macros/tests/evm/events_match_txs.sql b/macros/tests/evm/events_match_txs.sql new file mode 100644 index 0000000..9249fea --- /dev/null +++ b/macros/tests/evm/events_match_txs.sql @@ -0,0 +1,35 @@ +{% test events_match_txs( + model, + transactions_model +) %} +WITH logs AS ( + SELECT + DISTINCT block_number, + tx_hash, + tx_position + FROM + {{ model }} +), +missing_transactions AS ( + SELECT + logs.block_number, + logs.tx_hash, + logs.tx_position + FROM + logs + LEFT JOIN {{ transactions_model }} + txs USING ( + block_number, + tx_hash, + tx_position + ) + WHERE + txs.tx_hash IS NULL + OR txs.tx_position IS NULL + OR txs.block_number IS NULL +) +SELECT + * +FROM + missing_transactions +{% endtest %} \ No newline at end of file diff --git a/macros/tests/evm/find_missing_decoded_logs.sql b/macros/tests/evm/find_missing_decoded_logs.sql new file mode 100644 index 0000000..b53bdfa --- /dev/null +++ b/macros/tests/evm/find_missing_decoded_logs.sql @@ -0,0 +1,15 @@ +{% test find_missing_decoded_logs(model, fact_logs_model, wrapped_asset_address) %} +SELECT + l.block_number, + l.fact_event_logs_id +FROM + {{ fact_logs_model }} + l + LEFT JOIN {{ model }} + d + ON d.ez_decoded_event_logs_id = l.fact_event_logs_id +WHERE + l.contract_address = '{{ wrapped_asset_address }}' + AND l.topics [0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' -- Transfer + AND d.ez_decoded_event_logs_id IS NULL +{% endtest %} \ No newline at end of file diff --git a/macros/tests/evm/txs_match_blocks.sql b/macros/tests/evm/txs_match_blocks.sql new file mode 100644 index 0000000..3f3835a --- /dev/null +++ b/macros/tests/evm/txs_match_blocks.sql @@ -0,0 +1,31 @@ +{% test txs_match_blocks( + model, + blocks_model +) %} +WITH count_txs AS ( + SELECT + block_number, + COUNT(*) AS record_count + FROM + {{ model }} + GROUP BY + ALL +), +block_txs AS ( + SELECT + block_number, + tx_count AS expected_count + FROM + {{ blocks_model }} +) +SELECT + block_number, + record_count AS actual_count, + expected_count +FROM + block_txs + LEFT JOIN count_txs USING (block_number) +WHERE + record_count != expected_count + OR expected_count IS NULL +{% endtest %} \ No newline at end of file diff --git a/models/evm/bronze/bronze__decoded_logs.sql b/models/evm/bronze/bronze__decoded_logs.sql new file mode 100644 index 0000000..022adfb --- /dev/null +++ b/models/evm/bronze/bronze__decoded_logs.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = 'view', + tags = ['evm'] +) }} + + WITH meta AS ( + SELECT + job_created_time AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 6), '_', 1) AS INTEGER) AS _partition_by_block_number, + TO_DATE( + concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)) + ) AS _partition_by_created_date + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", "evm_decoded_logs") }}') + ) A + ) + SELECT + block_number, + id :: STRING AS id, + DATA, + metadata, + b.file_name, + _inserted_timestamp, + s._partition_by_block_number AS _partition_by_block_number, + s._partition_by_created_date AS _partition_by_created_date + FROM + {{ source( + "bronze_streamline", + "evm_decoded_logs" + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + WHERE + b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + AND s._partition_by_created_date >= DATEADD('day', -2, CURRENT_TIMESTAMP()) + AND DATA :error IS NULL + AND DATA IS NOT NULL \ No newline at end of file diff --git a/models/evm/bronze/bronze__decoded_logs_fr.sql b/models/evm/bronze/bronze__decoded_logs_fr.sql new file mode 100644 index 0000000..cf81a10 --- /dev/null +++ b/models/evm/bronze/bronze__decoded_logs_fr.sql @@ -0,0 +1,44 @@ +{{ config ( + materialized = 'view', + tags = ['evm'] +) }} + +WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 6), '_', 1) AS INTEGER) AS _partition_by_block_number, + TO_DATE( + concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)) + ) AS _partition_by_created_date + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "evm_decoded_logs") }}' + ) + ) A + ) +SELECT + block_number, + id :: STRING AS id, + DATA, + metadata, + b.file_name, + _inserted_timestamp, + s._partition_by_block_number AS _partition_by_block_number, + s._partition_by_created_date AS _partition_by_created_date +FROM + {{ source( + "bronze_streamline", + "evm_decoded_logs" + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date +WHERE + b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + AND DATA :error IS NULL + AND DATA IS NOT NULL \ No newline at end of file diff --git a/models/evm/bronze/bronze_evm_api__token_reads.sql b/models/evm/bronze/bronze_evm_api__token_reads.sql new file mode 100644 index 0000000..75d9024 --- /dev/null +++ b/models/evm/bronze/bronze_evm_api__token_reads.sql @@ -0,0 +1,110 @@ +{{ config( + materialized = 'incremental', + unique_key = "contract_address", + full_refresh = false, + tags = ['evm'] +) }} + +WITH base AS ( + + SELECT + contract_address, + latest_event_block AS latest_block + FROM + {{ ref('silver_evm__relevant_contracts') }} + WHERE + total_event_count >= 25 + +{% if is_incremental() %} +AND contract_address NOT IN ( + SELECT + contract_address + FROM + {{ this }} +) +{% endif %} +ORDER BY + total_event_count DESC +LIMIT + 200 +), function_sigs AS ( + SELECT + '0x313ce567' AS function_sig, + 'decimals' AS function_name + UNION + SELECT + '0x06fdde03', + 'name' + UNION + SELECT + '0x95d89b41', + 'symbol' +), +all_reads AS ( + SELECT + * + FROM + base + JOIN function_sigs + ON 1 = 1 +), +ready_reads AS ( + SELECT + contract_address, + latest_block, + function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input, + utils.udf_json_rpc_call( + 'eth_call', + [{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(latest_block)], + concat_ws( + '-', + contract_address, + input, + latest_block + ) + ) AS rpc_request + FROM + all_reads +), +node_call AS ( + SELECT + contract_address, + latest_block, + function_sig, + input, + live.udf_api( + 'POST', + '{Service}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + rpc_request, + 'Vault/prod/flow/evm/mainnet' + ) AS response + FROM + ready_reads + WHERE + EXISTS ( + SELECT + 1 + FROM + ready_reads + LIMIT + 1 + ) +) +SELECT + contract_address, + latest_block AS block_number, + function_sig, + NULL AS function_input, + response, + SYSDATE() :: TIMESTAMP AS _inserted_timestamp +FROM + node_call \ No newline at end of file diff --git a/models/evm/bronze/bronze_evm_api__token_reads.yml b/models/evm/bronze/bronze_evm_api__token_reads.yml new file mode 100644 index 0000000..315d41e --- /dev/null +++ b/models/evm/bronze/bronze_evm_api__token_reads.yml @@ -0,0 +1,15 @@ +version: 2 +models: + - name: bronze_evm_api__token_reads + + columns: + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/descriptions/blocks.md b/models/evm/descriptions/blocks.md new file mode 100644 index 0000000..4ccd92b --- /dev/null +++ b/models/evm/descriptions/blocks.md @@ -0,0 +1,182 @@ +{% docs evm_blocks_table_doc %} + +This table contains block level data for this EVM blockchain. This table can be used to analyze trends at a block level, for example gas fees vs. total transactions over time. For more information, please see [Etherscan Resources](https://etherscan.io/directory/Learning_Resources/Ethereum) or [The Ethereum Organization](https://ethereum.org/en/developers/docs/blocks/) + +{% enddocs %} + + +{% docs evm_block_header_json %} + +This JSON column contains the block header details. + +{% enddocs %} + + +{% docs evm_blockchain %} + +The blockchain on which transactions are being confirmed. + +{% enddocs %} + + +{% docs evm_blocks_hash %} + +The hash of the block header for a given block. + +{% enddocs %} + + +{% docs evm_blocks_nonce %} + +Block nonce is a value used during mining to demonstrate proof of work for a given block. + +{% enddocs %} + + +{% docs evm_difficulty %} + +The effort required to mine the block. + +{% enddocs %} + + +{% docs evm_extra_data %} + +Any data included by the miner for a given block. + +{% enddocs %} + + +{% docs evm_gas_limit %} + +Total gas limit provided by all transactions in the block. + +{% enddocs %} + + +{% docs evm_gas_used %} + +Total gas used in the block. + +{% enddocs %} + +{% docs evm_network %} + +The network on the blockchain used by a transaction. + +{% enddocs %} + + +{% docs evm_parent_hash %} + +The hash of the block from which a given block is generated. Also known as the parent block. + +{% enddocs %} + + +{% docs evm_receipts_root %} + +The root of the state trie. + +{% enddocs %} + + +{% docs evm_sha3_uncles %} + +The mechanism which Ethereum Javascript RLP encodes an empty string. + +{% enddocs %} + + +{% docs evm_size %} + +Block size, which is determined by a given block's gas limit. + +{% enddocs %} + + +{% docs evm_total_difficulty %} + +Total difficulty of the chain at a given block. + +{% enddocs %} + + +{% docs evm_tx_count %} + +Total number of transactions within a block. + +{% enddocs %} + + +{% docs evm_uncle_blocks %} + +Uncle blocks occur when two blocks are mined and broadcasted at the same time, with the same block number. The block validated across the most nodes will be added to the primary chain, and the other one becomes an uncle block. Miners do receive rewards for uncle blocks. + +{% enddocs %} + +{% docs evm_miner %} + +The address of the beneficiary to whom the mining rewards were given + +{% enddocs %} + +{% docs evm_state_root %} + +The root of the final state trie of the block + +{% enddocs %} + +{% docs evm_transactions_root %} + +The root of the transaction trie of the block + +{% enddocs %} + +{% docs evm_logs_bloom %} + +The bloom filter for the logs of the block. + +{% enddocs %} + +{% docs evm_mix_hash %} + +A string of a 256-bit hash encoded as a hexadecimal + +{% enddocs %} + +{% docs evm_base_fee_per_gas %} + +A string of the base fee encoded in hexadecimal format. Please note that this response field will not be included in a block requested before the EIP-1559 upgrade + +{% enddocs %} + +{% docs evm_blob_gas_used %} + +The amount of gas used for blob transactions in the block. + +{% enddocs %} + +{% docs evm_excess_blob_gas %} + +The amount of excess blob gas in the block. + +{% enddocs %} + +{% docs evm_parent_beacon_block_root %} + +The root of the parent beacon block. + +{% enddocs %} + +{% docs evm_withdrawals %} + +The withdrawals for the block. + +{% enddocs %} + +{% docs evm_withdrawals_root %} + +The root of the withdrawals for the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/base_fee_per_gas.md b/models/evm/descriptions/blocks/base_fee_per_gas.md deleted file mode 100644 index f54a512..0000000 --- a/models/evm/descriptions/blocks/base_fee_per_gas.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_base_fee_per_gas %} - -A string of the base fee encoded in hexadecimal format, converted to integer. - -{% enddocs %} diff --git a/models/evm/descriptions/blocks/block_header_json.md b/models/evm/descriptions/blocks/block_header_json.md deleted file mode 100644 index cad7117..0000000 --- a/models/evm/descriptions/blocks/block_header_json.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_block_header_json %} - -This JSON column contains the block header details. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/blockchain.md b/models/evm/descriptions/blocks/blockchain.md deleted file mode 100644 index 0b9bee2..0000000 --- a/models/evm/descriptions/blocks/blockchain.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_blockchain %} - -The blockchain on which transactions are being confirmed. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/blocks_hash.md b/models/evm/descriptions/blocks/blocks_hash.md deleted file mode 100644 index b6269e3..0000000 --- a/models/evm/descriptions/blocks/blocks_hash.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_blocks_hash %} - -The hash of the block header for a given block. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/blocks_nonce.md b/models/evm/descriptions/blocks/blocks_nonce.md deleted file mode 100644 index 58958fc..0000000 --- a/models/evm/descriptions/blocks/blocks_nonce.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_blocks_nonce %} - -Block nonce is a value used during mining to demonstrate proof of work for a given block. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/blocks_table_doc.md b/models/evm/descriptions/blocks/blocks_table_doc.md deleted file mode 100644 index e145e90..0000000 --- a/models/evm/descriptions/blocks/blocks_table_doc.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_blocks_table_doc %} - -This table contains block level data for the flowEVM Blockchain. This table can be used to analyze trends at a block level, for example gas fees vs. total transactions over time. For more information on EVM transactions, please see [Etherscan Resources](https://etherscan.io/directory/Learning_Resources/Ethereum) or [The Ethereum Organization](https://ethereum.org/en/developers/docs/blocks/) - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/difficulty.md b/models/evm/descriptions/blocks/difficulty.md deleted file mode 100644 index 8b688bc..0000000 --- a/models/evm/descriptions/blocks/difficulty.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_difficulty %} - -The effort required to mine the block. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/extra_data.md b/models/evm/descriptions/blocks/extra_data.md deleted file mode 100644 index 9a365dc..0000000 --- a/models/evm/descriptions/blocks/extra_data.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_extra_data %} - -Any data included by the validator for a given block. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/gas_limit.md b/models/evm/descriptions/blocks/gas_limit.md deleted file mode 100644 index feea70a..0000000 --- a/models/evm/descriptions/blocks/gas_limit.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_gas_limit %} - -Total gas limit provided by all transactions in the block. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/gas_used.md b/models/evm/descriptions/blocks/gas_used.md deleted file mode 100644 index 9970c19..0000000 --- a/models/evm/descriptions/blocks/gas_used.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_gas_used %} - -Total gas used in the block. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/logs_bloom.md b/models/evm/descriptions/blocks/logs_bloom.md deleted file mode 100644 index 1740f4f..0000000 --- a/models/evm/descriptions/blocks/logs_bloom.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_logs_bloom %} - -The bloom filter for the logs of the block. null if pending - -{% enddocs %} diff --git a/models/evm/descriptions/blocks/miner.md b/models/evm/descriptions/blocks/miner.md deleted file mode 100644 index af9a32f..0000000 --- a/models/evm/descriptions/blocks/miner.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_miner %} - -Miner who successfully added a given block to the blockchain. - -{% enddocs %} diff --git a/models/evm/descriptions/blocks/mix_hash.md b/models/evm/descriptions/blocks/mix_hash.md deleted file mode 100644 index 154e923..0000000 --- a/models/evm/descriptions/blocks/mix_hash.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_mix_hash %} - -A string of a 256-bit hash encoded as a hexadecimal - -{% enddocs %} diff --git a/models/evm/descriptions/blocks/network.md b/models/evm/descriptions/blocks/network.md deleted file mode 100644 index 4286d44..0000000 --- a/models/evm/descriptions/blocks/network.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_network %} - -The network on the blockchain used by a transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/nonce.md b/models/evm/descriptions/blocks/nonce.md deleted file mode 100644 index 7890b7f..0000000 --- a/models/evm/descriptions/blocks/nonce.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_nonce %} - -The hash of the generated proof-of-work. null if pending - -{% enddocs %} diff --git a/models/evm/descriptions/blocks/parent_hash.md b/models/evm/descriptions/blocks/parent_hash.md deleted file mode 100644 index a9967b6..0000000 --- a/models/evm/descriptions/blocks/parent_hash.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_parent_hash %} - -The hash of the block from which a given block is generated. Also known as the parent block. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/receipts_root.md b/models/evm/descriptions/blocks/receipts_root.md deleted file mode 100644 index 70516ad..0000000 --- a/models/evm/descriptions/blocks/receipts_root.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_receipts_root %} - -The root of the state trie. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/sha3_uncles.md b/models/evm/descriptions/blocks/sha3_uncles.md deleted file mode 100644 index eaf652c..0000000 --- a/models/evm/descriptions/blocks/sha3_uncles.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_sha3_uncles %} - -The mechanism which Ethereum Javascript RLP encodes an empty string. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/size.md b/models/evm/descriptions/blocks/size.md deleted file mode 100644 index 486548e..0000000 --- a/models/evm/descriptions/blocks/size.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_size %} - -Block size, which is determined by a given block's gas limit. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/state_root.md b/models/evm/descriptions/blocks/state_root.md deleted file mode 100644 index 3cea7ca..0000000 --- a/models/evm/descriptions/blocks/state_root.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_state_root %} - -The root of the final state trie of the block - -{% enddocs %} diff --git a/models/evm/descriptions/blocks/total_difficulty.md b/models/evm/descriptions/blocks/total_difficulty.md deleted file mode 100644 index dab4ee5..0000000 --- a/models/evm/descriptions/blocks/total_difficulty.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_total_difficulty %} - -Total difficulty of the chain at a given block. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/transactions.md b/models/evm/descriptions/blocks/transactions.md deleted file mode 100644 index cf9b0cb..0000000 --- a/models/evm/descriptions/blocks/transactions.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_transactions %} - -An array containing transaction response data. - -{% enddocs %} diff --git a/models/evm/descriptions/blocks/transactions_root.md b/models/evm/descriptions/blocks/transactions_root.md deleted file mode 100644 index 57a5093..0000000 --- a/models/evm/descriptions/blocks/transactions_root.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_transactions_root %} - -The root of the transaction trie of the block - -{% enddocs %} diff --git a/models/evm/descriptions/blocks/tx_count.md b/models/evm/descriptions/blocks/tx_count.md deleted file mode 100644 index 500ec34..0000000 --- a/models/evm/descriptions/blocks/tx_count.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_count %} - -Total number of transactions within a block. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/blocks/uncle_blocks.md b/models/evm/descriptions/blocks/uncle_blocks.md deleted file mode 100644 index cdb2ca6..0000000 --- a/models/evm/descriptions/blocks/uncle_blocks.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_uncle_blocks %} - -Uncle blocks occur when two blocks are mined and broadcasted at the same time, with the same block number. The block validated across the most nodes will be added to the primary chain, and the other one becomes an uncle block. Miners do receive rewards for uncle blocks. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs.md b/models/evm/descriptions/event_logs.md new file mode 100644 index 0000000..52c8e0c --- /dev/null +++ b/models/evm/descriptions/event_logs.md @@ -0,0 +1,95 @@ +{% docs evm_logs_table_doc %} + +This table contains flattened event logs from transactions on this EVM blockchain. Transactions may have multiple events, which are denoted by the event index for a transaction hash. Therefore, this table is unique on the combination of transaction hash and event index. Please see `fact_decoded_event_logs` or `ez_decoded_event_logs` for the decoded event logs. + +{% enddocs %} + + +{% docs evm_event_index %} + +Event number within a transaction. + +{% enddocs %} + + +{% docs evm_event_inputs %} + +The decoded event inputs for a given event. + +{% enddocs %} + +{% docs evm_event_removed %} + +Whether the event has been removed from the transaction. + +{% enddocs %} + + +{% docs evm_log_id_events %} + +This is the primary key for this table. This is a concatenation of the transaction hash and the event index at which the event occurred. This field can be used within other event based tables such as ```fact_transfers``` & ```ez_token_transfers```. + +{% enddocs %} + + +{% docs evm_logs_contract_address %} + +The address interacted with for a given event. + +{% enddocs %} + + +{% docs evm_logs_contract_name %} + +The name of the contract or token, where possible. + +{% enddocs %} + + +{% docs evm_logs_data %} + +The un-decoded event data. + +{% enddocs %} + + +{% docs evm_logs_tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. This field will not be unique in this table, as a given transaction can include multiple events. + +{% enddocs %} + + +{% docs evm_topics %} + +The un-decoded event input topics. + +{% enddocs %} + + +{% docs evm_topic_0 %} + +The first topic of the event, which is a unique identifier for the event. + +{% enddocs %} + + +{% docs evm_topic_1 %} + +The second topic of the event, if applicable. + +{% enddocs %} + + +{% docs evm_topic_2 %} + +The third topic of the event, if applicable. + +{% enddocs %} + + +{% docs evm_topic_3 %} + +The fourth topic of the event, if applicable. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/event_index.md b/models/evm/descriptions/event_logs/event_index.md deleted file mode 100644 index 03583f2..0000000 --- a/models/evm/descriptions/event_logs/event_index.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_event_index %} - -Event number within a transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/event_inputs.md b/models/evm/descriptions/event_logs/event_inputs.md deleted file mode 100644 index be19f04..0000000 --- a/models/evm/descriptions/event_logs/event_inputs.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_event_inputs %} - -The decoded event inputs for a given event. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/event_name.md b/models/evm/descriptions/event_logs/event_name.md deleted file mode 100644 index 3bd263c..0000000 --- a/models/evm/descriptions/event_logs/event_name.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_event_name %} - -The decoded event name for a given event. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/event_removed.md b/models/evm/descriptions/event_logs/event_removed.md deleted file mode 100644 index 6821437..0000000 --- a/models/evm/descriptions/event_logs/event_removed.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_event_removed %} - -Whether the event has been removed from the transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/event_sig.md b/models/evm/descriptions/event_logs/event_sig.md deleted file mode 100644 index 6ab88be..0000000 --- a/models/evm/descriptions/event_logs/event_sig.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_origin_sig %} - -The function signature of this transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/log_id_events.md b/models/evm/descriptions/event_logs/log_id_events.md deleted file mode 100644 index 154e7ec..0000000 --- a/models/evm/descriptions/event_logs/log_id_events.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_log_id_events %} - -This is the primary key for this table. This is a concatenation of the transaction hash and the event index at which the event occurred. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/logs_contract_address.md b/models/evm/descriptions/event_logs/logs_contract_address.md deleted file mode 100644 index 5c36229..0000000 --- a/models/evm/descriptions/event_logs/logs_contract_address.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_logs_contract_address %} - -The address interacted with for a given event. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/logs_contract_name.md b/models/evm/descriptions/event_logs/logs_contract_name.md deleted file mode 100644 index 9f228d3..0000000 --- a/models/evm/descriptions/event_logs/logs_contract_name.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_logs_contract_name %} - -The name of the contract or token, where possible. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/logs_data.md b/models/evm/descriptions/event_logs/logs_data.md deleted file mode 100644 index 2828d5e..0000000 --- a/models/evm/descriptions/event_logs/logs_data.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_logs_data %} - -The un-decoded event data. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/logs_table_doc.md b/models/evm/descriptions/event_logs/logs_table_doc.md deleted file mode 100644 index 4adf6c0..0000000 --- a/models/evm/descriptions/event_logs/logs_table_doc.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_logs_table_doc %} - -This table contains flattened event logs from transactions on the flowEVM Blockchain. Transactions may have multiple events, which are denoted by the event index for a transaction hash. Therefore, this table is unique on the combination of transaction hash and event index. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/logs_tx_hash.md b/models/evm/descriptions/event_logs/logs_tx_hash.md deleted file mode 100644 index d6876fd..0000000 --- a/models/evm/descriptions/event_logs/logs_tx_hash.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_logs_tx_hash %} - -Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. This field will not be unique in this table, as a given transaction can include multiple events. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/origin_from.md b/models/evm/descriptions/event_logs/origin_from.md deleted file mode 100644 index 3099c62..0000000 --- a/models/evm/descriptions/event_logs/origin_from.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_origin_from %} - -The from address of this transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/origin_to.md b/models/evm/descriptions/event_logs/origin_to.md deleted file mode 100644 index fbefe39..0000000 --- a/models/evm/descriptions/event_logs/origin_to.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_origin_to %} - -The to address of this transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/event_logs/topics.md b/models/evm/descriptions/event_logs/topics.md deleted file mode 100644 index d3d13f1..0000000 --- a/models/evm/descriptions/event_logs/topics.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_topics %} - -The un-decoded event input topics. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/block_number.md b/models/evm/descriptions/general/block_number.md deleted file mode 100644 index 3c665d4..0000000 --- a/models/evm/descriptions/general/block_number.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_block_number %} - -Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. - -{% enddocs %} diff --git a/models/evm/descriptions/general/block_timestamp.md b/models/evm/descriptions/general/block_timestamp.md deleted file mode 100644 index 4c47d1f..0000000 --- a/models/evm/descriptions/general/block_timestamp.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_block_timestamp %} - -The date and time at which the block was produced. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/complete_general.md b/models/evm/descriptions/general/complete_general.md new file mode 100644 index 0000000..17d63f4 --- /dev/null +++ b/models/evm/descriptions/general/complete_general.md @@ -0,0 +1,89 @@ +{% docs evm_internal_column %} + +Deprecated. This column is no longer used. Please remove from your query by Jan. 10 2024. + +{% enddocs %} + +{% docs evm_amount_deprecation %} + +This column is being deprecated for standardization purposes on Jan. 10 2024. Please use the equivalent column without the native asset prefix. For example, use `amount` instead of `avax_amount`. + +{% enddocs %} + +{% docs evm_block_number %} + +Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. + +{% enddocs %} + +{% docs evm_block_timestamp %} + +The date and time at which the block was produced. + +{% enddocs %} + +{% docs evm_from_address %} + +The sending address of this transaction. + +{% enddocs %} + +{% docs evm_ingested_at %} + +Internal column. + +{% enddocs %} + +{% docs evm_to_address %} + +The receiving address of this transaction. This can be a contract address. + +{% enddocs %} + +{% docs evm_pk %} + +The unique identifier for each row in the table. + +{% enddocs %} + +{% docs evm_inserted_timestamp %} + +The UTC timestamp at which the row was inserted into the table. + +{% enddocs %} + +{% docs evm_modified_timestamp %} + +The UTC timestamp at which the row was last modified. + +{% enddocs %} + +{% docs evm_precise_amount_unadjusted %} + +The precise, unadjusted amount of the transaction. This is returned as a string to avoid precision loss. + +{% enddocs %} + +{% docs evm_precise_amount_adjusted %} + +The precise, adjusted amount of the transaction. This is returned as a string to avoid precision loss. + +{% enddocs %} + +{% docs evm_value_hex %} + +The value of the transaction in hexadecimal format. + +{% enddocs %} + +{% docs evm_column_deprecation_notice %} + +This column is being deprecated for standardization purposes. + +{% enddocs %} + +{% docs evm_table_deprecation_notice %} + +This table is being deprecated for standardization purposes. + +{% enddocs %} diff --git a/models/evm/descriptions/general/complete_labels.md b/models/evm/descriptions/general/complete_labels.md new file mode 100644 index 0000000..d5b0e9d --- /dev/null +++ b/models/evm/descriptions/general/complete_labels.md @@ -0,0 +1,69 @@ +{% docs evm_labels_table %} + +This table contains labels for addresses on this EVM blockchain. + +{% enddocs %} + + +{% docs evm_table_dim_labels %} + +The labels table is a store of one-to-one address identifiers, or an address name. Labels are broken out into a "type" (such as cex, dex, dapp, games, etc.) and a "subtype" (ex: contract_deployer, hot_wallet, token_contract, etc.) in order to help classify each address name into similar groups. Our labels are sourced from many different places, but can primarily be grouped into two categories: automatic and manual. Automatic labels are continuously labeled based on certain criteria, such as a known contract deploying another contract, behavior-based algorithms for finding deposit wallets, and consistent data pulls of custom protocol APIs. Manual labels are done periodically to find addresses that cannot be found programmatically such as finding new protocol addresses, centralized exchange hot wallets, or trending addresses. Labels can also be added by our community by using our add-a-label tool (https://science.flipsidecrypto.xyz/add-a-label/) or on-chain with near (https://near.social/lord1.near/widget/Form) and are reviewed by our labels team. A label can be removed by our labels team if it is found to be incorrect or no longer relevant; this generally will only happen for mislabeled deposit wallets. + +{% enddocs %} + +{% docs evm_project_name %} + +The name of the project for this address. + +{% enddocs %} + + +{% docs evm_label %} + +The label for this address. + +{% enddocs %} + + +{% docs evm_label_address %} + +Address that the label is for. This is the field that should be used to join other tables with labels. + +{% enddocs %} + + +{% docs evm_label_address_name %} + +The most granular label for this address. + +{% enddocs %} + + +{% docs evm_label_blockchain %} + +The name of the blockchain. + +{% enddocs %} + + +{% docs evm_label_creator %} + +The name of the creator of the label. + +{% enddocs %} + + +{% docs evm_label_subtype %} + +A sub-category nested within label type providing further detail. + +{% enddocs %} + + +{% docs evm_label_type %} + +A high-level category describing the address's main function or ownership. + +{% enddocs %} + + diff --git a/models/evm/descriptions/general/decimals.md b/models/evm/descriptions/general/decimals.md deleted file mode 100644 index a828460..0000000 --- a/models/evm/descriptions/general/decimals.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_decimals %} - -The number of decimal places this contract needs adjusted where token values exist. For example, use the decimal field to correctly transform raw amounts in ```fact_token_transfers```. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/deprecation.md b/models/evm/descriptions/general/deprecation.md deleted file mode 100644 index 004e578..0000000 --- a/models/evm/descriptions/general/deprecation.md +++ /dev/null @@ -1,11 +0,0 @@ -{% docs internal_column %} - -Deprecated. This column is no longer used. Please remove from your query by Jan. 31 2024. - -{% enddocs %} - -{% docs amount_deprecation %} - -This column is being deprecated for standardization purposes on Jan. 31 2024. Please use the equivalent column without the native asset prefix. For example, use `amount` instead of `eth_amount`. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/from_address.md b/models/evm/descriptions/general/from_address.md deleted file mode 100644 index 1016bda..0000000 --- a/models/evm/descriptions/general/from_address.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_from_address %} - -The sending address of this transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/ingested_at.md b/models/evm/descriptions/general/ingested_at.md deleted file mode 100644 index a17f341..0000000 --- a/models/evm/descriptions/general/ingested_at.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_ingested_at %} - -Internal column. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/is_pending.md b/models/evm/descriptions/general/is_pending.md deleted file mode 100644 index 2e02dd2..0000000 --- a/models/evm/descriptions/general/is_pending.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs is_pending %} - -A boolean flag indicating if the record is pending or complete. - -{% enddocs %} diff --git a/models/evm/descriptions/general/precise_amounts.md b/models/evm/descriptions/general/precise_amounts.md deleted file mode 100644 index 96a815c..0000000 --- a/models/evm/descriptions/general/precise_amounts.md +++ /dev/null @@ -1,17 +0,0 @@ -{% docs precise_amount_unadjusted %} - -The precise, unadjusted amount of the transaction. This is returned as a string to avoid precision loss. - -{% enddocs %} - -{% docs precise_amount_adjusted %} - -The precise, adjusted amount of the transaction. This is returned as a string to avoid precision loss. - -{% enddocs %} - -{% docs tx_fee_precise %} - -The precise amount of the transaction fee. This is returned as a string to avoid precision loss. - -{% enddocs %} diff --git a/models/evm/descriptions/general/symbol.md b/models/evm/descriptions/general/symbol.md deleted file mode 100644 index 22fee0e..0000000 --- a/models/evm/descriptions/general/symbol.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_symbol %} - -The symbol belonging to the address of the token - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/general/to_address.md b/models/evm/descriptions/general/to_address.md deleted file mode 100644 index 87bf89a..0000000 --- a/models/evm/descriptions/general/to_address.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_to_address %} - -The receiving address of this transaction. This can be a contract address. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/traces.md b/models/evm/descriptions/traces.md new file mode 100644 index 0000000..c6b601e --- /dev/null +++ b/models/evm/descriptions/traces.md @@ -0,0 +1,136 @@ +{% docs evm_traces_table_doc %} + +This table contains flattened trace data for internal contract calls. User-defined functions (UDFs) are available to decode hex encoded fields, including `utils.udf_hex_to_int()`. + +{% enddocs %} + + +{% docs evm_traces_block_no %} + +The block number of this transaction. + +{% enddocs %} + + +{% docs evm_traces_blocktime %} + +The block timestamp of this transaction. + +{% enddocs %} + + +{% docs evm_traces_call_data %} + +The raw JSON data for this trace. + +{% enddocs %} + + +{% docs evm_traces_value %} + +The amount of the native asset transferred in this trace. + +{% enddocs %} + + +{% docs evm_traces_from %} + +The sending address of this trace. This is not necessarily the from address of the transaction. + +{% enddocs %} + + +{% docs evm_traces_gas %} + +The gas supplied for this trace. + +{% enddocs %} + + +{% docs evm_traces_gas_used %} + +The gas used for this trace. + +{% enddocs %} + + +{% docs evm_traces_identifier %} + +This field represents the position and type of the trace within the transaction. + +{% enddocs %} + + +{% docs evm_trace_index %} + +The index of the trace within the transaction. + +{% enddocs %} + + +{% docs evm_traces_input %} + +The input data for this trace. + +{% enddocs %} + + +{% docs evm_traces_output %} + +The output data for this trace. + +{% enddocs %} + + +{% docs evm_sub_traces %} + +The amount of nested sub traces for this trace. + +{% enddocs %} + + +{% docs evm_traces_to %} + +The receiving address of this trace. This is not necessarily the to address of the transaction. + +{% enddocs %} + + +{% docs evm_traces_tx_hash %} + +The transaction hash for the trace. Please note, this is not necessarily unique in this table as transactions frequently have multiple traces. + +{% enddocs %} + + +{% docs evm_traces_type %} + +The type of internal transaction. Common trace types are `CALL`, `DELEGATECALL`, and `STATICCALL`. + +{% enddocs %} + +{% docs evm_trace_succeeded %} + +The boolean value representing if the trace succeeded. + +{% enddocs %} + +{% docs evm_trace_error_reason %} + +The reason for the trace failure, if any. + +{% enddocs %} + +{% docs evm_trace_address %} + +The trace address for this trace. + +{% enddocs %} + + +{% docs evm_revert_reason %} + +The reason for the revert, if available. + +{% enddocs %} + diff --git a/models/evm/descriptions/transactions.md b/models/evm/descriptions/transactions.md new file mode 100644 index 0000000..e521c0d --- /dev/null +++ b/models/evm/descriptions/transactions.md @@ -0,0 +1,249 @@ +{% docs evm_tx_table_doc %} + +This table contains transaction level data for this EVM blockchain. Each transaction will have a unique transaction hash, along with transaction fees and a value transferred in the native asset when applicable. Transactions may be native asset transfers or interactions with contract addresses. For more information, please see [The Ethereum Organization - Transactions](https://ethereum.org/en/developers/docs/transactions/) + +Below are the specific native tokens that correspond to each EVM chain: + +| Status | Description | +|------------|-------------| +| ETHEREUM | ETH | +| BINANCE | BNB | +| POLYGON | POL | +| AVALANCHE | AVAX | +| ARBITRUM | ETH | +| OPTIMISM | ETH | +| GNOSIS | xDAI | +| KAIA | KLAY | +| SEI | SEI | +| CORE | CORE | + +{% enddocs %} + + +{% docs evm_cumulative_gas_used %} + +The total amount of gas used when this transaction was executed in the block. + +{% enddocs %} + + +{% docs evm_tx_block_hash %} + +Block hash is a unique 66-character identifier that is generated when a block is produced. + +{% enddocs %} + + +{% docs evm_tx_fee %} + +Amount paid to validate the transaction in the native asset. + +{% enddocs %} + + +{% docs evm_tx_gas_limit %} + +Maximum amount of gas allocated for the transaction. + +{% enddocs %} + + +{% docs evm_tx_gas_price %} + +Cost per unit of gas in Gwei. + +{% enddocs %} + + +{% docs evm_tx_gas_used %} + +Gas used by the transaction. + +{% enddocs %} + + +{% docs evm_tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. + +{% enddocs %} + + +{% docs evm_tx_input_data %} + +This column contains additional data for this transaction, and is commonly used as part of a contract interaction or as a message to the recipient. + +{% enddocs %} + + +{% docs evm_tx_json %} + +This JSON column contains the transaction details, including event logs. + +{% enddocs %} + + +{% docs evm_tx_nonce %} + +The number of transactions sent from a given address. + +{% enddocs %} + + +{% docs evm_tx_origin_sig %} + +The function signature of the call that triggered this transaction. + +{% enddocs %} + +{% docs evm_origin_sig %} + +The function signature of the contract call that triggered this transaction. + +{% enddocs %} + + +{% docs evm_tx_position %} + +The position of the transaction within the block. + +{% enddocs %} + + +{% docs evm_tx_status %} + +Status of the transaction. + +{% enddocs %} + + +{% docs evm_value %} + +The value transacted in the native asset. + +{% enddocs %} + + +{% docs evm_effective_gas_price %} + +The total base charge plus tip paid for each unit of gas, in Gwei. + +{% enddocs %} + +{% docs evm_max_fee_per_gas %} + +The maximum fee per gas of the transaction, in Gwei. + +{% enddocs %} + + +{% docs evm_max_priority_fee_per_gas %} + +The maximum priority fee per gas of the transaction, in Gwei. + +{% enddocs %} + + +{% docs evm_r %} + +The r value of the transaction signature. + +{% enddocs %} + + +{% docs evm_s %} + +The s value of the transaction signature. + +{% enddocs %} + + +{% docs evm_v %} + +The v value of the transaction signature. + +{% enddocs %} + +{% docs evm_tx_succeeded %} + +Whether the transaction was successful, returned as a boolean. + +{% enddocs %} + +{% docs evm_tx_fee_precise %} + +The precise amount of the transaction fee. This is returned as a string to avoid precision loss. + +{% enddocs %} + +{% docs evm_tx_type %} + +The type of transaction. + +{% enddocs %} + +{% docs evm_mint %} + +The minting event associated with the transaction + +{% enddocs %} + +{% docs evm_source_hash %} + +The hash of the source transaction that created this transaction + +{% enddocs %} + +{% docs evm_eth_value %} + +The eth value for the transaction + +{% enddocs %} + +{% docs evm_chain_id %} + +The unique identifier for the chain the transaction was executed on. + +{% enddocs %} + +{% docs evm_l1_fee_precise_raw %} + +The raw l1 fee for the transaction, in Gwei. + +{% enddocs %} + +{% docs evm_l1_fee_precise %} + +The precise l1 fee for the transaction, in Gwei. + +{% enddocs %} + +{% docs evm_y_parity %} + +The y parity for the transaction. + +{% enddocs %} + +{% docs evm_access_list %} + +The access list for the transaction. + +{% enddocs %} + +{% docs evm_l1_base_fee_scalar %} + +The scalar l1 base fee for the transaction. + +{% enddocs %} + +{% docs evm_l1_blob_base_fee %} + +The blob base fee for the transaction. + +{% enddocs %} + +{% docs evm_l1_blob_base_fee_scalar %} + +The scalar blob base fee for the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/block_hash.md b/models/evm/descriptions/transactions/block_hash.md deleted file mode 100644 index 85876ac..0000000 --- a/models/evm/descriptions/transactions/block_hash.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_block_hash %} - -Block hash is a unique 66-character identifier that is generate when a block is produced. - -{% enddocs %} diff --git a/models/evm/descriptions/transactions/chain_id.md b/models/evm/descriptions/transactions/chain_id.md deleted file mode 100644 index fe1e124..0000000 --- a/models/evm/descriptions/transactions/chain_id.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_chain_id %} - -The chain id returned by the node. - -{% enddocs %} diff --git a/models/evm/descriptions/transactions/cumulative_gas_used.md b/models/evm/descriptions/transactions/cumulative_gas_used.md deleted file mode 100644 index f29c1e9..0000000 --- a/models/evm/descriptions/transactions/cumulative_gas_used.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_cumulative_gas_used %} - -The total amount of gas used when this transaction was executed in the block. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/receipt_type.md b/models/evm/descriptions/transactions/receipt_type.md deleted file mode 100644 index 3d8586d..0000000 --- a/models/evm/descriptions/transactions/receipt_type.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_receipt_type %} - -The transaction type as returned by the receipt body. - -{% enddocs %} diff --git a/models/evm/descriptions/transactions/tx_fee.md b/models/evm/descriptions/transactions/tx_fee.md deleted file mode 100644 index 74f46ae..0000000 --- a/models/evm/descriptions/transactions/tx_fee.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_fee %} - -Amount paid to validate the transaction in ETH. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_gas_limit.md b/models/evm/descriptions/transactions/tx_gas_limit.md deleted file mode 100644 index 7069642..0000000 --- a/models/evm/descriptions/transactions/tx_gas_limit.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_gas_limit %} - -Maximum amount of gas allocated for the transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_gas_price.md b/models/evm/descriptions/transactions/tx_gas_price.md deleted file mode 100644 index bea902e..0000000 --- a/models/evm/descriptions/transactions/tx_gas_price.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_gas_price %} - -Cost per unit of gas in Gwei. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_gas_used.md b/models/evm/descriptions/transactions/tx_gas_used.md deleted file mode 100644 index e530435..0000000 --- a/models/evm/descriptions/transactions/tx_gas_used.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_gas_used %} - -Gas used by transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_hash.md b/models/evm/descriptions/transactions/tx_hash.md deleted file mode 100644 index 66f3ba7..0000000 --- a/models/evm/descriptions/transactions/tx_hash.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_hash %} - -Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_input_data.md b/models/evm/descriptions/transactions/tx_input_data.md deleted file mode 100644 index 37f4aa6..0000000 --- a/models/evm/descriptions/transactions/tx_input_data.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_input_data %} - -This column contains additional data for this transaction, and is commonly used as part of a contract interaction or as a message to the recipient. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_json.md b/models/evm/descriptions/transactions/tx_json.md deleted file mode 100644 index 7190ddb..0000000 --- a/models/evm/descriptions/transactions/tx_json.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_json %} - -This JSON column contains the transaction details, including event logs. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_nonce.md b/models/evm/descriptions/transactions/tx_nonce.md deleted file mode 100644 index 9b97341..0000000 --- a/models/evm/descriptions/transactions/tx_nonce.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_nonce %} - -The number of transactions sent from a given address. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_origin_sig.md b/models/evm/descriptions/transactions/tx_origin_sig.md deleted file mode 100644 index 57ae164..0000000 --- a/models/evm/descriptions/transactions/tx_origin_sig.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_origin_sig %} - -The function signature of the contract call. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_position.md b/models/evm/descriptions/transactions/tx_position.md deleted file mode 100644 index 15aabdc..0000000 --- a/models/evm/descriptions/transactions/tx_position.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_position %} - -The position of the transaction within the block. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_status.md b/models/evm/descriptions/transactions/tx_status.md deleted file mode 100644 index 982674a..0000000 --- a/models/evm/descriptions/transactions/tx_status.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_status %} - -Status of the transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_table_doc.md b/models/evm/descriptions/transactions/tx_table_doc.md deleted file mode 100644 index 5f12009..0000000 --- a/models/evm/descriptions/transactions/tx_table_doc.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_table_doc %} - -This table contains transaction level data for the flowEVM Blockchain. Each transaction will have a unique transaction hash, along with transactions fees and a ETH value transferred when applicable. Transactions may be native ETH transfers or interactions with contract addresses. For more information, please see [The Ethereum Organization - Transactions](https://ethereum.org/en/developers/docs/transactions/) - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/descriptions/transactions/tx_type.md b/models/evm/descriptions/transactions/tx_type.md deleted file mode 100644 index 24faf70..0000000 --- a/models/evm/descriptions/transactions/tx_type.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_tx_type %} - -Transaction Type as returned by the transaction body. - -{% enddocs %} diff --git a/models/evm/descriptions/transactions/value.md b/models/evm/descriptions/transactions/value.md deleted file mode 100644 index fcdd1d1..0000000 --- a/models/evm/descriptions/transactions/value.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs flowevm_value %} - -The value transacted in ETH. - -{% enddocs %} \ No newline at end of file diff --git a/models/evm/gold/core/core_evm__fact_blocks.sql b/models/evm/gold/core/core_evm__fact_blocks.sql index a99dcf7..0d76b90 100644 --- a/models/evm/gold/core/core_evm__fact_blocks.sql +++ b/models/evm/gold/core/core_evm__fact_blocks.sql @@ -1,83 +1,64 @@ -{{ config( - materialized = 'incremental', - unique_key = "fact_blocks_id", - incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - cluster_by = ['inserted_timestamp :: DATE', 'ROUND(block_number, -3)'], - persist_docs ={ "relation": true, - "columns": true }, +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['block_timestamp::DATE'], tags = ['evm'] ) }} SELECT block_number, - block_timestamp, + block_json :hash :: STRING AS block_hash, + utils.udf_hex_to_int( + block_json :timestamp :: STRING + ) :: TIMESTAMP AS block_timestamp, 'mainnet' AS network, - 'flow' AS blockchain, - transaction_count AS tx_count, - difficulty, - total_difficulty, - extra_data, - gas_limit, - gas_used, - block_hash AS HASH, - parent_hash, - receipts_root, - sha3_uncles, - SIZE, - uncles AS uncle_blocks, - OBJECT_CONSTRUCT( - 'baseFeePerGas', - base_fee_per_gas, - 'difficulty', - difficulty, - 'extraData', - extra_data, - 'gasLimit', - gas_limit, - 'gasUsed', - gas_used, - 'hash', - block_hash, - 'logsBloom', - logs_bloom, - 'miner', - miner, - 'nonce', - nonce, - 'number', - block_number, - 'parentHash', - parent_hash, - 'receiptsRoot', - receipts_root, - 'sha3Uncles', - sha3_uncles, - 'size', - SIZE, - 'stateRoot', - state_root, - 'timestamp', - block_timestamp, - 'totalDifficulty', - total_difficulty, - 'transactionsRoot', - transactions_root, - 'uncles', - uncles - ) AS block_header_json, - evm_blocks_id AS fact_blocks_id, - inserted_timestamp, - modified_timestamp + ARRAY_SIZE( + block_json :transactions + ) AS tx_count, + utils.udf_hex_to_int( + block_json :size :: STRING + ) :: bigint AS SIZE, + block_json :miner :: STRING AS miner, + block_json :mixHash :: STRING AS mix_hash, + block_json :extraData :: STRING AS extra_data, + block_json :parentHash :: STRING AS parent_hash, + utils.udf_hex_to_int( + block_json :gasUsed :: STRING + ) :: bigint AS gas_used, + utils.udf_hex_to_int( + block_json :gasLimit :: STRING + ) :: bigint AS gas_limit, + utils.udf_hex_to_int( + block_json :baseFeePerGas :: STRING + ) :: bigint AS base_fee_per_gas, + utils.udf_hex_to_int( + block_json :difficulty :: STRING + ) :: bigint AS difficulty, + utils.udf_hex_to_int( + block_json :totalDifficulty :: STRING + ) :: bigint AS total_difficulty, + block_json :sha3Uncles :: STRING AS sha3_uncles, + block_json :uncles AS uncle_blocks, + utils.udf_hex_to_int( + block_json :nonce :: STRING + ) :: bigint AS nonce, + block_json :receiptsRoot :: STRING AS receipts_root, + block_json :stateRoot :: STRING AS state_root, + block_json :transactionsRoot :: STRING AS transactions_root, + block_json :logsBloom :: STRING AS logs_bloom, + {{ dbt_utils.generate_surrogate_key(['block_number']) }} AS fact_blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp FROM {{ ref('silver_evm__blocks') }} +WHERE 1=1 {% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} +AND modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }} ) -{% endif %} +{% endif %} \ No newline at end of file diff --git a/models/evm/gold/core/core_evm__fact_blocks.yml b/models/evm/gold/core/core_evm__fact_blocks.yml index c406080..28a7bc1 100644 --- a/models/evm/gold/core/core_evm__fact_blocks.yml +++ b/models/evm/gold/core/core_evm__fact_blocks.yml @@ -1,51 +1,56 @@ version: 2 models: - name: core_evm__fact_blocks - description: '{{ doc("flowevm_blocks_table_doc") }}' - tests: - - dbt_utils.recency: - datepart: hour - field: block_timestamp - interval: 2 - + description: '{{ doc("evm_blocks_table_doc") }}' + columns: - name: BLOCK_NUMBER - description: '{{ doc("flowevm_block_number") }}' + description: '{{ doc("evm_block_number") }}' + - name: BLOCK_HASH + description: '{{ doc("evm_blocks_hash") }}' - name: BLOCK_TIMESTAMP - description: '{{ doc("flowevm_block_timestamp") }}' + description: '{{ doc("evm_block_timestamp") }}' - name: NETWORK - description: '{{ doc("flowevm_network") }}' - - name: BLOCKCHAIN - description: '{{ doc("flowevm_blockchain") }}' + description: '{{ doc("evm_network") }}' - name: TX_COUNT - description: '{{ doc("flowevm_tx_count") }}' - - name: DIFFICULTY - description: '{{ doc("flowevm_difficulty") }}' - - name: TOTAL_DIFFICULTY - description: '{{ doc("flowevm_total_difficulty") }}' - - name: EXTRA_DATA - description: '{{ doc("flowevm_extra_data") }}' - - name: GAS_LIMIT - description: '{{ doc("flowevm_gas_limit") }}' - - name: GAS_USED - description: '{{ doc("flowevm_gas_used") }}' - - name: HASH - description: '{{ doc("flowevm_blocks_hash") }}' - - name: PARENT_HASH - description: '{{ doc("flowevm_parent_hash") }}' - - name: RECEIPTS_ROOT - description: '{{ doc("flowevm_receipts_root") }}' - - name: SHA3_UNCLES - description: '{{ doc("flowevm_sha3_uncles") }}' + description: '{{ doc("evm_tx_count") }}' - name: SIZE - description: '{{ doc("flowevm_size") }}' + description: '{{ doc("evm_size") }}' + - name: MINER + description: '{{ doc("evm_miner") }}' + - name: BASE_FEE_PER_GAS + description: '{{ doc("evm_base_fee_per_gas") }}' + - name: MIX_HASH + description: '{{ doc("evm_mix_hash") }}' + - name: EXTRA_DATA + description: '{{ doc("evm_extra_data") }}' + - name: PARENT_HASH + description: '{{ doc("evm_parent_hash") }}' + - name: GAS_USED + description: '{{ doc("evm_gas_used") }}' + - name: GAS_LIMIT + description: '{{ doc("evm_gas_limit") }}' + - name: DIFFICULTY + description: '{{ doc("evm_difficulty") }}' + - name: TOTAL_DIFFICULTY + description: '{{ doc("evm_total_difficulty") }}' + - name: SHA3_UNCLES + description: '{{ doc("evm_sha3_uncles") }}' - name: UNCLE_BLOCKS - description: '{{ doc("flowevm_uncle_blocks") }}' - - name: BLOCK_HEADER_JSON - description: '{{ doc("flowevm_block_header_json") }}' + description: '{{ doc("evm_uncle_blocks") }}' + - name: NONCE + description: '{{ doc("evm_blocks_nonce") }}' + - name: RECEIPTS_ROOT + description: '{{ doc("evm_receipts_root") }}' + - name: STATE_ROOT + description: '{{ doc("evm_state_root") }}' + - name: TRANSACTIONS_ROOT + description: '{{ doc("evm_transactions_root") }}' + - name: LOGS_BLOOM + description: '{{ doc("evm_logs_bloom") }}' - name: FACT_BLOCKS_ID - description: '{{ doc("pk_id") }}' + description: '{{ doc("evm_pk") }}' - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' + description: '{{ doc("evm_inserted_timestamp") }}' - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' \ No newline at end of file + description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/evm/gold/core/core_evm__fact_event_logs.sql b/models/evm/gold/core/core_evm__fact_event_logs.sql index 5c05178..2aa8ed6 100644 --- a/models/evm/gold/core/core_evm__fact_event_logs.sql +++ b/models/evm/gold/core/core_evm__fact_event_logs.sql @@ -1,40 +1,223 @@ -{{ config( - materialized = 'incremental', - unique_key = "fact_event_logs_id", - incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - cluster_by = ['inserted_timestamp :: DATE', 'ROUND(block_number, -3)'], - persist_docs ={ "relation": true, - "columns": true }, +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['block_timestamp::DATE'], tags = ['evm'] ) }} +WITH base AS ( + + SELECT + block_number, + {% if uses_receipts_by_hash %} + tx_hash, + {% else %} + receipts_json :transactionHash :: STRING AS tx_hash, + {% endif %} + receipts_json, + receipts_json :logs AS full_logs + FROM + {{ ref('silver_evm__receipts') }} + WHERE + 1 = 1 + AND ARRAY_SIZE(receipts_json :logs) > 0 + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }}) + {% endif %} +), +flattened_logs AS ( + SELECT + block_number, + tx_hash, + lower(receipts_json :from :: STRING) AS origin_from_address, + lower(receipts_json :to :: STRING) AS origin_to_address, + CASE + WHEN receipts_json :status :: STRING = '0x1' THEN TRUE + WHEN receipts_json :status :: STRING = '1' THEN TRUE + WHEN receipts_json :status :: STRING = '0x0' THEN FALSE + WHEN receipts_json :status :: STRING = '0' THEN FALSE + ELSE NULL + END AS tx_succeeded, + VALUE :address :: STRING AS contract_address, + VALUE :blockHash :: STRING AS block_hash, + VALUE :blockNumber :: STRING AS block_number_hex, + VALUE :data :: STRING AS DATA, + utils.udf_hex_to_int( + VALUE :logIndex :: STRING + ) :: INT AS event_index, + VALUE :removed :: BOOLEAN AS event_removed, + VALUE :topics AS topics, + VALUE :transactionHash :: STRING AS transaction_hash, + utils.udf_hex_to_int( + VALUE :transactionIndex :: STRING + ) :: INT AS transaction_index + FROM + base, + LATERAL FLATTEN ( + input => full_logs + ) +), +new_logs AS ( + SELECT + l.block_number, + b.block_timestamp, + l.tx_hash, + l.transaction_index AS tx_position, + l.event_index, + l.contract_address, + l.topics, + l.topics [0] :: STRING AS topic_0, + l.topics [1] :: STRING AS topic_1, + l.topics [2] :: STRING AS topic_2, + l.topics [3] :: STRING AS topic_3, + l.data, + l.event_removed, + txs.from_address AS origin_from_address, + txs.to_address AS origin_to_address, + txs.origin_function_signature, + l.tx_succeeded + FROM + flattened_logs l + LEFT JOIN {{ ref('core_evm__fact_blocks') }} + b + ON l.block_number = b.block_number + +{% if is_incremental() %} +AND b.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +LEFT JOIN {{ ref('core_evm__fact_transactions') }} +txs +ON l.tx_hash = txs.tx_hash +AND l.block_number = txs.block_number + +{% if is_incremental() %} +AND txs.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + b.block_timestamp AS block_timestamp_heal, + t.tx_hash, + t.tx_position, + t.event_index, + t.contract_address, + t.topics, + t.topic_0, + t.topic_1, + t.topic_2, + t.topic_3, + t.data, + t.event_removed, + txs.from_address AS origin_from_address_heal, + txs.to_address AS origin_to_address_heal, + txs.origin_function_signature AS origin_function_signature_heal, + t.tx_succeeded + FROM + {{ this }} + t + LEFT JOIN {{ ref('core_evm__fact_transactions') }} + txs + ON t.tx_hash = txs.tx_hash + AND t.block_number = txs.block_number + LEFT JOIN {{ ref('core_evm__fact_blocks') }} + b + ON t.block_number = b.block_number + WHERE + t.block_timestamp IS NULL + OR t.origin_function_signature IS NULL +) +{% endif %}, +all_logs AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + tx_position, + event_index, + contract_address, + topics, + topic_0, + topic_1, + topic_2, + topic_3, + DATA, + event_removed, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_succeeded + FROM + new_logs + +{% if is_incremental() %} +UNION ALL +SELECT + block_number, + block_timestamp_heal AS block_timestamp, + tx_hash, + tx_position, + event_index, + contract_address, + topics, + topic_0, + topic_1, + topic_2, + topic_3, + DATA, + event_removed, + origin_from_address_heal AS origin_from_address, + origin_to_address_heal AS origin_to_address, + origin_function_signature_heal AS origin_function_signature, + tx_succeeded +FROM + missing_data +{% endif %} +) SELECT block_number, block_timestamp, tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, + tx_position, event_index, contract_address, topics, + topic_0, + topic_1, + topic_2, + topic_3, DATA, event_removed, - tx_status, - _log_id, - evm_logs_id AS fact_event_logs_id, - inserted_timestamp, - modified_timestamp + origin_from_address, + origin_to_address, + origin_function_signature, + tx_succeeded, + {{ dbt_utils.generate_surrogate_key(['tx_hash','event_index']) }} AS fact_event_logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp FROM - {{ ref('silver_evm__logs') }} - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) -{% endif %} + all_logs qualify ROW_NUMBER() over ( + PARTITION BY fact_event_logs_id + ORDER BY + block_number DESC, + block_timestamp DESC nulls last, + origin_function_signature DESC nulls last + ) = 1 \ No newline at end of file diff --git a/models/evm/gold/core/core_evm__fact_event_logs.yml b/models/evm/gold/core/core_evm__fact_event_logs.yml index 1b0b74b..358c470 100644 --- a/models/evm/gold/core/core_evm__fact_event_logs.yml +++ b/models/evm/gold/core/core_evm__fact_event_logs.yml @@ -1,43 +1,46 @@ version: 2 models: - name: core_evm__fact_event_logs - description: '{{ doc("flowevm_logs_table_doc") }}' - tests: - - dbt_utils.recency: - datepart: hour - field: block_timestamp - interval: 2 - + description: '{{ doc("evm_logs_table_doc") }}' + columns: - name: BLOCK_NUMBER - description: '{{ doc("flowevm_block_number") }}' + description: '{{ doc("evm_block_number") }}' - name: BLOCK_TIMESTAMP - description: '{{ doc("flowevm_block_timestamp") }}' + description: '{{ doc("evm_block_timestamp") }}' - name: TX_HASH - description: '{{ doc("flowevm_logs_tx_hash") }}' + description: '{{ doc("evm_tx_hash") }}' + - name: TX_POSITION + description: '{{ doc("evm_tx_position") }}' - name: EVENT_INDEX - description: '{{ doc("flowevm_event_index") }}' + description: '{{ doc("evm_event_index") }}' - name: CONTRACT_ADDRESS - description: '{{ doc("flowevm_logs_contract_address") }}' + description: '{{ doc("evm_logs_contract_address") }}' - name: TOPICS - description: '{{ doc("flowevm_topics") }}' + description: '{{ doc("evm_topics") }}' + - name: TOPIC_0 + description: '{{ doc("evm_topic_0") }}' + - name: TOPIC_1 + description: '{{ doc("evm_topic_1") }}' + - name: TOPIC_2 + description: '{{ doc("evm_topic_2") }}' + - name: TOPIC_3 + description: '{{ doc("evm_topic_3") }}' - name: DATA - description: '{{ doc("flowevm_logs_data") }}' + description: '{{ doc("evm_logs_data") }}' - name: EVENT_REMOVED - description: '{{ doc("flowevm_event_removed") }}' - - name: _LOG_ID - description: '{{ doc("internal_column") }}' - - name: TX_STATUS - description: '{{ doc("flowevm_tx_status") }}' - - name: ORIGIN_FUNCTION_SIGNATURE - description: '{{ doc("flowevm_origin_sig") }}' + description: '{{ doc("evm_event_removed") }}' - name: ORIGIN_FROM_ADDRESS - description: '{{ doc("flowevm_origin_from") }}' + description: '{{ doc("evm_from_address") }}' - name: ORIGIN_TO_ADDRESS - description: '{{ doc("flowevm_origin_to") }}' + description: '{{ doc("evm_to_address") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("evm_origin_sig") }}' + - name: TX_SUCCEEDED + description: '{{ doc("evm_tx_succeeded") }}' - name: FACT_EVENT_LOGS_ID - description: '{{ doc("pk_id") }}' + description: '{{ doc("evm_pk") }}' - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' + description: '{{ doc("evm_inserted_timestamp") }}' - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' + description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/evm/gold/core/core_evm__fact_traces.sql b/models/evm/gold/core/core_evm__fact_traces.sql new file mode 100644 index 0000000..39d1606 --- /dev/null +++ b/models/evm/gold/core/core_evm__fact_traces.sql @@ -0,0 +1,379 @@ +{{ config( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['block_timestamp::DATE'], + tags = ['evm'] +) }} + +WITH silver_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + parent_trace_address, + trace_address_array, + trace_json, + traces_id, + 'regular' AS source + FROM {{ ref('silver_evm__traces') }} + WHERE 1 = 1 + {% if is_incremental() %} + AND modified_timestamp > ( + SELECT COALESCE(MAX(modified_timestamp), '1970-01-01'::TIMESTAMP) AS modified_timestamp + FROM {{ this }} + ) + {% endif %} +), + +sub_traces AS ( + SELECT + block_number, + tx_position, + parent_trace_address, + COUNT(*) AS sub_traces + FROM silver_traces + GROUP BY + block_number, + tx_position, + parent_trace_address +), + +trace_index_array AS ( + SELECT + block_number, + tx_position, + trace_address, + ARRAY_AGG(flat_value) AS number_array + FROM ( + SELECT + block_number, + tx_position, + trace_address, + IFF(VALUE::STRING = 'ORIGIN', -1, VALUE::INT) AS flat_value + FROM silver_traces, + LATERAL FLATTEN(input => trace_address_array) + ) + GROUP BY + block_number, + tx_position, + trace_address +), + +trace_index_sub_traces AS ( + SELECT + b.block_number, + b.tx_position, + b.trace_address, + IFNULL(sub_traces, 0) AS sub_traces, + number_array, + ROW_NUMBER() OVER ( + PARTITION BY b.block_number, b.tx_position + ORDER BY number_array ASC + ) - 1 AS trace_index, + b.trace_json, + b.traces_id, + b.source + FROM silver_traces b + LEFT JOIN sub_traces s + ON b.block_number = s.block_number + AND b.tx_position = s.tx_position + AND b.trace_address = s.parent_trace_address + JOIN trace_index_array n + ON b.block_number = n.block_number + AND b.tx_position = n.tx_position + AND b.trace_address = n.trace_address +), + +errored_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + trace_json + FROM trace_index_sub_traces + WHERE trace_json:error::STRING IS NOT NULL +), + +error_logic AS ( + SELECT + b0.block_number, + b0.tx_position, + b0.trace_address, + b0.trace_json:error::STRING AS error, + b1.trace_json:error::STRING AS any_error, + b2.trace_json:error::STRING AS origin_error + FROM trace_index_sub_traces b0 + LEFT JOIN errored_traces b1 + ON b0.block_number = b1.block_number + AND b0.tx_position = b1.tx_position + AND b0.trace_address RLIKE CONCAT('^', b1.trace_address, '(_[0-9]+)*$') + LEFT JOIN errored_traces b2 + ON b0.block_number = b2.block_number + AND b0.tx_position = b2.tx_position + AND b2.trace_address = 'ORIGIN' +), + +aggregated_errors AS ( + SELECT + block_number, + tx_position, + trace_address, + error, + IFF( + MAX(any_error) IS NULL AND error IS NULL AND origin_error IS NULL, + TRUE, + FALSE + ) AS trace_succeeded + FROM error_logic + GROUP BY + block_number, + tx_position, + trace_address, + error, + origin_error +), + +json_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + sub_traces, + number_array, + trace_index, + trace_succeeded, + trace_json:error::STRING AS error_reason, + trace_json:revertReason::STRING AS revert_reason, + lower(trace_json:from::STRING) AS from_address, + lower(trace_json:to::STRING) AS to_address, + IFNULL(trace_json:value::STRING, '0x0') AS value_hex, + IFNULL(utils.udf_hex_to_int(trace_json:value::STRING), '0') AS value_precise_raw, + utils.udf_decimal_adjust(value_precise_raw, 18) AS value_precise, + value_precise::FLOAT AS value, + utils.udf_hex_to_int(trace_json:gas::STRING)::INT AS gas, + utils.udf_hex_to_int(trace_json:gasUsed::STRING)::INT AS gas_used, + trace_json:input::STRING AS input, + trace_json:output::STRING AS output, + trace_json:type::STRING AS type, + traces_id + FROM trace_index_sub_traces + JOIN aggregated_errors USING ( + block_number, + tx_position, + trace_address + ) +), + +incremental_traces AS ( + SELECT + f.block_number, + t.tx_hash, + t.block_timestamp, + t.origin_function_signature, + t.from_address AS origin_from_address, + t.to_address AS origin_to_address, + t.tx_position, + f.trace_index, + f.from_address AS from_address, + f.to_address AS to_address, + f.value_hex, + f.value_precise_raw, + f.value_precise, + f.value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.sub_traces, + f.error_reason, + f.revert_reason, + f.traces_id, + f.trace_succeeded, + f.trace_address, + t.tx_succeeded + FROM json_traces f + LEFT OUTER JOIN {{ ref('core_evm__fact_transactions') }} t + ON f.tx_position = t.tx_position + AND f.block_number = t.block_number + {% if is_incremental() %} + AND t.modified_timestamp >= ( + SELECT DATEADD('hour', -24, MAX(modified_timestamp)) + FROM {{ this }} + ) + {% endif %} +) +{% if is_incremental() %}, +overflow_blocks AS ( + SELECT DISTINCT block_number + FROM silver_traces + WHERE source = 'overflow' +), + +heal_missing_data AS ( + SELECT + t.block_number, + txs.tx_hash, + txs.block_timestamp AS block_timestamp_heal, + txs.origin_function_signature AS origin_function_signature_heal, + txs.from_address AS origin_from_address_heal, + txs.to_address AS origin_to_address_heal, + txs.tx_position, + t.trace_index, + t.from_address, + t.to_address, + t.value_hex, + t.value_precise_raw, + t.value_precise, + t.value, + t.gas, + t.gas_used, + t.input, + t.output, + t.type, + t.sub_traces, + t.error_reason, + t.revert_reason, + t.fact_traces_id AS traces_id, + t.trace_succeeded, + t.trace_address, + txs.tx_succeeded AS tx_succeeded_heal + FROM {{ this }} t + JOIN {{ ref('core_evm__fact_transactions') }} txs + ON t.tx_position = txs.tx_position + AND t.block_number = txs.block_number + WHERE t.tx_position IS NULL + OR t.block_timestamp IS NULL + OR t.tx_succeeded IS NULL + +) +{% endif %}, +all_traces AS ( + SELECT + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + value, + gas, + gas_used, + input, + output, + type, + sub_traces, + error_reason, + revert_reason, + trace_succeeded, + trace_address, + tx_succeeded + FROM incremental_traces + +{% if is_incremental() %} + + UNION ALL + + SELECT + block_number, + tx_hash, + block_timestamp_heal AS block_timestamp, + origin_function_signature_heal AS origin_function_signature, + origin_from_address_heal AS origin_from_address, + origin_to_address_heal AS origin_to_address, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + value, + gas, + gas_used, + input, + output, + type, + sub_traces, + error_reason, + revert_reason, + trace_succeeded, + trace_address, + tx_succeeded_heal AS tx_succeeded + FROM heal_missing_data + + UNION ALL + + SELECT + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + value, + gas, + gas_used, + input, + output, + type, + sub_traces, + error_reason, + revert_reason, + trace_succeeded, + trace_address, + tx_succeeded + FROM {{ this }} + JOIN overflow_blocks USING (block_number) +{% endif %} +) + +SELECT + block_number, + block_timestamp, + tx_hash, + tx_position, + trace_index, + from_address, + to_address, + input, + output, + type, + trace_address, + sub_traces, + value, + value_precise_raw, + value_precise, + value_hex, + gas, + gas_used, + origin_from_address, + origin_to_address, + origin_function_signature, + trace_succeeded, + error_reason, + revert_reason, + tx_succeeded, + {{ dbt_utils.generate_surrogate_key(['tx_hash', 'trace_index']) }} AS fact_traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM all_traces +QUALIFY (ROW_NUMBER() OVER ( + PARTITION BY block_number, tx_position, trace_index + ORDER BY modified_timestamp DESC, block_timestamp DESC NULLS LAST +)) = 1 \ No newline at end of file diff --git a/models/evm/gold/core/core_evm__fact_traces.yml b/models/evm/gold/core/core_evm__fact_traces.yml new file mode 100644 index 0000000..0ea1131 --- /dev/null +++ b/models/evm/gold/core/core_evm__fact_traces.yml @@ -0,0 +1,62 @@ +version: 2 +models: + - name: core_evm__fact_traces + description: '{{ doc("evm_traces_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("evm_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("evm_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("evm_tx_hash") }}' + - name: TX_POSITION + description: '{{ doc("evm_tx_position") }}' + - name: TRACE_INDEX + description: '{{ doc("evm_trace_index") }}' + - name: FROM_ADDRESS + description: '{{ doc("evm_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("evm_to_address") }}' + - name: INPUT + description: '{{ doc("evm_traces_input") }}' + - name: OUTPUT + description: '{{ doc("evm_traces_output") }}' + - name: TYPE + description: '{{ doc("evm_traces_type") }}' + - name: TRACE_ADDRESS + description: '{{ doc("evm_trace_address") }}' + - name: SUB_TRACES + description: '{{ doc("evm_sub_traces") }}' + - name: VALUE + description: '{{ doc("evm_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("evm_precise_amount_unadjusted") }}' + - name: VALUE_PRECISE + description: '{{ doc("evm_precise_amount_adjusted") }}' + - name: VALUE_HEX + description: '{{ doc("evm_value_hex") }}' + - name: GAS + description: '{{ doc("evm_traces_gas") }}' + - name: GAS_USED + description: '{{ doc("evm_traces_gas_used") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("evm_traces_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("evm_traces_to") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("evm_origin_sig") }}' + - name: TRACE_SUCCEEDED + description: '{{ doc("evm_trace_succeeded") }}' + - name: ERROR_REASON + description: '{{ doc("evm_trace_error_reason") }}' + - name: REVERT_REASON + description: '{{ doc("evm_revert_reason") }}' + - name: TX_SUCCEEDED + description: '{{ doc("evm_tx_succeeded") }}' + - name: FACT_TRACES_ID + description: '{{ doc("evm_pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("evm_inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/evm/gold/core/core_evm__fact_transactions.sql b/models/evm/gold/core/core_evm__fact_transactions.sql index 951e373..5132c97 100644 --- a/models/evm/gold/core/core_evm__fact_transactions.sql +++ b/models/evm/gold/core/core_evm__fact_transactions.sql @@ -1,50 +1,348 @@ -{{ config( - materialized = 'incremental', - unique_key = "fact_transactions_id", +{{ config ( + materialized = "incremental", incremental_strategy = 'delete+insert', - cluster_by = ['inserted_timestamp :: DATE', 'ROUND(block_number, -3)'], - persist_docs ={ "relation": true, - "columns": true }, + unique_key = "block_number", + cluster_by = ['block_timestamp::DATE'], tags = ['evm'] ) }} -SELECT - block_number, - block_timestamp, - block_hash, - tx_hash, - nonce, - POSITION, - origin_function_signature, - from_address, - to_address, - VALUE, - value_precise_unadj AS value_precise_raw, - value_precise_adj AS value_precise, - tx_fee, - tx_fee_precise, - gas_price_adj AS gas_price, - effective_gas_price, - gas AS gas_limit, - gas_used, - cumulative_gas_used, - input_data, - tx_status AS status, - tx_succeeded, - r, - s, - v, - evm_txs_id AS fact_transactions_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver_evm__transactions') }} + +WITH base AS ( + + SELECT + block_number, + tx_position, + transaction_json + FROM + {{ ref('silver_evm__transactions') }} {% if is_incremental() %} WHERE - modified_timestamp >= ( + modified_timestamp > ( SELECT - MAX(modified_timestamp) + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp FROM - {{ this }} - ) + {{ this }}) + {% endif %} + ), + transactions_fields AS ( + SELECT + block_number, + tx_position, + transaction_json :blockHash :: STRING AS block_hash, + transaction_json :blockNumber :: STRING AS block_number_hex, + lower(transaction_json :from :: STRING) AS from_address, + utils.udf_hex_to_int( + transaction_json :gas :: STRING + ) :: bigint AS gas_limit, + utils.udf_hex_to_int( + transaction_json :gasPrice :: STRING + ) :: bigint AS gas_price, + transaction_json :hash :: STRING AS tx_hash, + transaction_json :input :: STRING AS input_data, + LEFT( + input_data, + 10 + ) AS origin_function_signature, + utils.udf_hex_to_int( + transaction_json :nonce :: STRING + ) :: bigint AS nonce, + transaction_json :r :: STRING AS r, + transaction_json :s :: STRING AS s, + lower(transaction_json :to :: STRING) AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + utils.udf_hex_to_int( + transaction_json :transactionIndex :: STRING + ) :: bigint AS transaction_index, + utils.udf_hex_to_int( + transaction_json :type :: STRING + ) :: bigint AS tx_type, + utils.udf_hex_to_int( + transaction_json :v :: STRING + ) :: bigint AS v, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + transaction_json :maxFeePerGas :: STRING + ) + ) / pow( + 10, + 9 + ) AS max_fee_per_gas, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + transaction_json :maxPriorityFeePerGas :: STRING + ) + ) / pow( + 10, + 9 + ) AS max_priority_fee_per_gas, + utils.udf_hex_to_int( + transaction_json :value :: STRING + ) AS value_precise_raw, + utils.udf_decimal_adjust( + value_precise_raw, + 18 + ) AS value_precise, + value_precise :: FLOAT AS VALUE, + utils.udf_hex_to_int(transaction_json :yParity :: STRING):: bigint AS y_parity, + transaction_json :accessList AS access_list, + FROM + base + ), + new_transactions AS ( + SELECT + txs.block_number, + txs.block_hash, + b.block_timestamp, + txs.tx_hash, + txs.from_address, + txs.to_address, + txs.origin_function_signature, + txs.value, + txs.value_precise_raw, + txs.value_precise, + txs.max_fee_per_gas, + txs.max_priority_fee_per_gas, + txs.y_parity, + txs.access_list, + utils.udf_decimal_adjust( + txs.gas_price * utils.udf_hex_to_int( + r.receipts_json :gasUsed :: STRING + ) :: bigint, + 18 + ) AS tx_fee_precise, + COALESCE( + tx_fee_precise :: FLOAT, + 0 + ) AS tx_fee, + CASE + WHEN r.receipts_json :status :: STRING = '0x1' THEN TRUE + WHEN r.receipts_json :status :: STRING = '1' THEN TRUE + WHEN r.receipts_json :status :: STRING = '0x0' THEN FALSE + WHEN r.receipts_json :status :: STRING = '0' THEN FALSE + ELSE NULL + END AS tx_succeeded, + txs.tx_type, + txs.nonce, + txs.tx_position, + txs.input_data, + txs.gas_price / pow( + 10, + 9 + ) AS gas_price, + utils.udf_hex_to_int( + r.receipts_json :gasUsed :: STRING + ) :: bigint AS gas_used, + txs.gas_limit, + utils.udf_hex_to_int( + r.receipts_json :cumulativeGasUsed :: STRING + ) :: bigint AS cumulative_gas_used, + utils.udf_hex_to_int( + r.receipts_json :effectiveGasPrice :: STRING + ) :: bigint AS effective_gas_price, + txs.r, + txs.s, + txs.v + FROM + transactions_fields txs + LEFT JOIN {{ ref('core_evm__fact_blocks') }} + b + ON txs.block_number = b.block_number + +{% if is_incremental() %} +AND b.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) :: DATE - 1 + FROM + {{ this }} +) {% endif %} +LEFT JOIN {{ ref('silver_evm__receipts') }} +r +ON txs.block_number = r.block_number +AND txs.tx_hash = r.receipts_json :transactionHash :: STRING + +{% if is_incremental() %} +AND r.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + b.block_timestamp AS block_timestamp_heal, + t.tx_hash, + t.from_address, + t.to_address, + t.origin_function_signature, + t.value, + t.value_precise_raw, + t.value_precise, + t.max_fee_per_gas, + t.max_priority_fee_per_gas, + t.y_parity, + t.access_list, + utils.udf_decimal_adjust( + t.gas_price * utils.udf_hex_to_int( + r.receipts_json :gasUsed :: STRING + ) :: bigint, + 9 + ) AS tx_fee_precise_heal, + COALESCE( + tx_fee_precise_heal :: FLOAT, + 0 + ) AS tx_fee_heal, + CASE + WHEN r.receipts_json :status :: STRING = '0x1' THEN TRUE + WHEN r.receipts_json :status :: STRING = '0x0' THEN FALSE + ELSE NULL + END AS tx_succeeded_heal, + t.tx_type, + t.nonce, + t.tx_position, + t.input_data, + t.gas_price, + utils.udf_hex_to_int( + r.receipts_json :gasUsed :: STRING + ) :: bigint AS gas_used_heal, + t.gas_limit, + utils.udf_hex_to_int( + r.receipts_json :cumulativeGasUsed :: STRING + ) :: bigint AS cumulative_gas_used_heal, + utils.udf_hex_to_int( + r.receipts_json :effectiveGasPrice :: STRING + ) :: bigint AS effective_gas_price_heal, + t.r, + t.s, + t.v + FROM + {{ this }} + t + LEFT JOIN {{ ref('core_evm__fact_blocks') }} + b + ON t.block_number = b.block_number + LEFT JOIN {{ ref('silver_evm__receipts') }} + r + ON t.block_number = r.block_number + AND t.tx_hash = r.receipts_json :transactionHash :: STRING + WHERE + t.block_timestamp IS NULL + OR t.tx_succeeded IS NULL +) +{% endif %}, +all_transactions AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + from_address, + to_address, + origin_function_signature, + VALUE, + value_precise_raw, + value_precise, + max_fee_per_gas, + max_priority_fee_per_gas, + y_parity, + access_list, + tx_fee, + tx_fee_precise, + tx_succeeded, + tx_type, + nonce, + tx_position, + input_data, + gas_price, + gas_used, + gas_limit, + cumulative_gas_used, + effective_gas_price, + r, + s, + v + FROM + new_transactions + +{% if is_incremental() %} +UNION ALL +SELECT + block_number, + block_timestamp_heal AS block_timestamp, + tx_hash, + from_address, + to_address, + origin_function_signature, + VALUE, + value_precise_raw, + value_precise, + max_fee_per_gas, + max_priority_fee_per_gas, + y_parity, + access_list, + tx_fee_heal AS tx_fee, + tx_fee_precise_heal AS tx_fee_precise, + tx_succeeded_heal AS tx_succeeded, + tx_type, + nonce, + tx_position, + input_data, + gas_price, + gas_used_heal AS gas_used, + gas_limit, + cumulative_gas_used_heal AS cumulative_gas_used, + effective_gas_price_heal AS effective_gas_price, + r, + s, + v +FROM + missing_data +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + from_address, + to_address, + origin_function_signature, + VALUE, + value_precise_raw, + value_precise, + tx_fee, + tx_fee_precise, + tx_succeeded, + tx_type, + nonce, + tx_position, + input_data, + gas_price, + gas_used, + gas_limit, + cumulative_gas_used, + effective_gas_price, + max_fee_per_gas, + max_priority_fee_per_gas, + y_parity, + access_list, + r, + s, + v, + {{ dbt_utils.generate_surrogate_key(['tx_hash']) }} AS fact_transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + all_transactions qualify ROW_NUMBER() over ( + PARTITION BY fact_transactions_id + ORDER BY + block_number DESC, + block_timestamp DESC nulls last, + tx_succeeded DESC nulls last + ) = 1 \ No newline at end of file diff --git a/models/evm/gold/core/core_evm__fact_transactions.yml b/models/evm/gold/core/core_evm__fact_transactions.yml index 9f24e74..3d507dc 100644 --- a/models/evm/gold/core/core_evm__fact_transactions.yml +++ b/models/evm/gold/core/core_evm__fact_transactions.yml @@ -1,67 +1,68 @@ version: 2 models: - name: core_evm__fact_transactions - description: '{{ doc("flowevm_tx_table_doc") }}' - tests: - - dbt_utils.recency: - datepart: hour - field: block_timestamp - interval: 2 - + description: '{{ doc("evm_tx_table_doc") }}' + columns: - name: BLOCK_NUMBER - description: '{{ doc("flowevm_block_number") }}' + description: '{{ doc("evm_block_number") }}' - name: BLOCK_TIMESTAMP - description: '{{ doc("flowevm_block_timestamp") }}' - - name: BLOCK_HASH - description: '{{ doc("flowevm_block_hash") }}' + description: '{{ doc("evm_block_timestamp") }}' - name: TX_HASH - description: '{{ doc("flowevm_tx_hash") }}' - - name: NONCE - description: '{{ doc("flowevm_tx_nonce") }}' - - name: POSITION - description: '{{ doc("flowevm_tx_position") }}' + description: '{{ doc("evm_tx_hash") }}' - name: FROM_ADDRESS - description: '{{ doc("flowevm_from_address") }}' + description: '{{ doc("evm_from_address") }}' - name: TO_ADDRESS - description: '{{ doc("flowevm_to_address") }}' + description: '{{ doc("evm_to_address") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("evm_tx_origin_sig") }}' - name: VALUE - description: '{{ doc("flowevm_value") }}' + description: '{{ doc("evm_value") }}' - name: VALUE_PRECISE_RAW - description: '{{ doc("precise_amount_unadjusted") }}' + description: '{{ doc("evm_precise_amount_unadjusted") }}' - name: VALUE_PRECISE - description: '{{ doc("precise_amount_adjusted") }}' + description: '{{ doc("evm_precise_amount_adjusted") }}' - name: TX_FEE - description: '{{ doc("flowevm_tx_fee") }}' + description: '{{ doc("evm_tx_fee") }}' - name: TX_FEE_PRECISE - description: '{{ doc("tx_fee_precise") }}' - - name: GAS_PRICE - description: '{{ doc("flowevm_tx_gas_price") }}' - - name: EFFECTIVE_GAS_PRICE - description: The total base charge plus tip paid for each unit of gas, in Gwei. - - name: GAS_LIMIT - description: '{{ doc("flowevm_tx_gas_limit") }}' - - name: GAS_USED - description: '{{ doc("flowevm_tx_gas_used") }}' - - name: CUMULATIVE_GAS_USED - description: '{{ doc("flowevm_cumulative_gas_used") }}' - - name: STATUS - description: '{{ doc("flowevm_tx_status") }}' + description: '{{ doc("evm_tx_fee_precise") }}' - name: TX_SUCCEEDED - description: '{{ doc("tx_succeeded") }}' + description: '{{ doc("evm_tx_succeeded") }}' + - name: TX_TYPE + description: '{{ doc("evm_tx_type") }}' + - name: NONCE + description: '{{ doc("evm_tx_nonce") }}' + - name: TX_POSITION + description: '{{ doc("evm_tx_position") }}' - name: INPUT_DATA - description: '{{ doc("flowevm_tx_input_data") }}' - - name: ORIGIN_FUNCTION_SIGNATURE - description: '{{ doc("flowevm_tx_origin_sig") }}' + description: '{{ doc("evm_tx_input_data") }}' + - name: GAS_PRICE + description: '{{ doc("evm_tx_gas_price") }}' + - name: GAS_USED + description: '{{ doc("evm_tx_gas_used") }}' + - name: GAS_LIMIT + description: '{{ doc("evm_tx_gas_limit") }}' + - name: CUMULATIVE_GAS_USED + description: '{{ doc("evm_cumulative_gas_used") }}' + - name: EFFECTIVE_GAS_PRICE + description: '{{ doc("evm_effective_gas_price") }}' - name: R - description: The r value of the transaction signature. + description: '{{ doc("evm_r") }}' - name: S - description: The s value of the transaction signature. + description: '{{ doc("evm_s") }}' - name: V - description: The v value of the transaction signature. + description: '{{ doc("evm_v") }}' + - name: MAX_FEE_PER_GAS + description: '{{ doc("evm_max_fee_per_gas") }}' + - name: MAX_PRIORITY_FEE_PER_GAS + description: '{{ doc("evm_max_priority_fee_per_gas") }}' + - name: Y_PARITY + description: '{{ doc("evm_y_parity") }}' + - name: ACCESS_LIST + description: '{{ doc("evm_access_list") }}' - name: FACT_TRANSACTIONS_ID - description: '{{ doc("pk_id") }}' + description: '{{ doc("evm_pk") }}' - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' + description: '{{ doc("evm_inserted_timestamp") }}' - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' \ No newline at end of file + description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_full.sql b/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_full.sql new file mode 100644 index 0000000..15c4b85 --- /dev/null +++ b/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('core_evm__fact_blocks') }} \ No newline at end of file diff --git a/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_full.yml b/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_full.yml new file mode 100644 index 0000000..6c54fa6 --- /dev/null +++ b/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_full.yml @@ -0,0 +1,145 @@ +version: 2 +models: + - name: test_gold_evm__fact_blocks_full + description: "This is a view used to test all of the gold fact blocks model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: NETWORK + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: ^[a-zA-Z0-9_]+$ + - name: TX_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SIZE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MINER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: EXTRA_DATA + tests: + - not_null + - name: PARENT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_LIMIT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SHA3_UNCLES + tests: + - not_null + - name: UNCLE_BLOCKS + tests: + - not_null + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: RECEIPTS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: STATE_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TRANSACTIONS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: LOGS_BLOOM + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FACT_BLOCKS_ID + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_unique + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_recent.sql b/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_recent.sql new file mode 100644 index 0000000..a632bfa --- /dev/null +++ b/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('core_evm__fact_blocks') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_block_lookback') }} + ) diff --git a/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_recent.yml b/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_recent.yml new file mode 100644 index 0000000..b3c98b9 --- /dev/null +++ b/models/evm/gold/tests/blocks/test_gold_evm__fact_blocks_recent.yml @@ -0,0 +1,147 @@ +version: 2 +models: + - name: test_gold_evm__fact_blocks_recent + description: "This is a view used to test the last three days of fact blocks." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">10" + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: NETWORK + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: ^[a-zA-Z0-9_]+$ + - name: TX_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SIZE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MINER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: EXTRA_DATA + tests: + - not_null + - name: PARENT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_LIMIT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SHA3_UNCLES + tests: + - not_null + - name: UNCLE_BLOCKS + tests: + - not_null + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: RECEIPTS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: STATE_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TRANSACTIONS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: LOGS_BLOOM + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FACT_BLOCKS_ID + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_unique + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_full.sql b/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_full.sql new file mode 100644 index 0000000..9f7b017 --- /dev/null +++ b/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('core_evm__fact_event_logs') }} \ No newline at end of file diff --git a/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_full.yml b/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_full.yml new file mode 100644 index 0000000..848fae9 --- /dev/null +++ b/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_full.yml @@ -0,0 +1,101 @@ +version: 2 +models: + - name: test_gold_evm__fact_event_logs_full + description: "This is a view used to test all of the gold fact event logs model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - EVENT_INDEX + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: EVENT_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + - events_match_txs: + transactions_model: ref('test_gold_evm__fact_transactions_full') + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - fsc_utils.tx_block_count: + config: + severity: error + error_if: "!=0" + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TOPICS + tests: + - not_null + - name: DATA + tests: + - not_null + - name: EVENT_REMOVED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_EVENT_LOGS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_recent.sql b/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_recent.sql new file mode 100644 index 0000000..c1b950a --- /dev/null +++ b/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('core_evm__fact_event_logs') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_block_lookback') }} + ) diff --git a/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_recent.yml b/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_recent.yml new file mode 100644 index 0000000..810231e --- /dev/null +++ b/models/evm/gold/tests/event_logs/test_gold_evm__fact_event_logs_recent.yml @@ -0,0 +1,100 @@ +version: 2 +models: + - name: test_gold_evm__fact_event_logs_recent + description: "This is a view used to test the last three days of fact event logs." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - EVENT_INDEX + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: EVENT_INDEX + - events_match_txs: + transactions_model: ref('test_gold_evm__fact_transactions_recent') + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - fsc_utils.tx_block_count: + config: + severity: error + error_if: "!=0" + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TOPICS + tests: + - not_null + - name: DATA + tests: + - not_null + - name: EVENT_REMOVED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_EVENT_LOGS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/gold/tests/traces/test_gold_evm__fact_traces_full.sql b/models/evm/gold/tests/traces/test_gold_evm__fact_traces_full.sql new file mode 100644 index 0000000..c4bdba2 --- /dev/null +++ b/models/evm/gold/tests/traces/test_gold_evm__fact_traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('core_evm__fact_traces') }} \ No newline at end of file diff --git a/models/evm/gold/tests/traces/test_gold_evm__fact_traces_full.yml b/models/evm/gold/tests/traces/test_gold_evm__fact_traces_full.yml new file mode 100644 index 0000000..9cbd543 --- /dev/null +++ b/models/evm/gold/tests/traces/test_gold_evm__fact_traces_full.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold_evm__fact_traces_full + description: "This is a view used to test all of the gold fact traces model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 AND TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/gold/tests/traces/test_gold_evm__fact_traces_recent.sql b/models/evm/gold/tests/traces/test_gold_evm__fact_traces_recent.sql new file mode 100644 index 0000000..69f9146 --- /dev/null +++ b/models/evm/gold/tests/traces/test_gold_evm__fact_traces_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('core_evm__fact_traces') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_block_lookback') }} + ) diff --git a/models/evm/gold/tests/traces/test_gold_evm__fact_traces_recent.yml b/models/evm/gold/tests/traces/test_gold_evm__fact_traces_recent.yml new file mode 100644 index 0000000..889295f --- /dev/null +++ b/models/evm/gold/tests/traces/test_gold_evm__fact_traces_recent.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold_evm__fact_traces_recent + description: "This is a view used to test the last three days of fact traces." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_full.sql b/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_full.sql new file mode 100644 index 0000000..454ce1c --- /dev/null +++ b/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('core_evm__fact_transactions') }} \ No newline at end of file diff --git a/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_full.yml b/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_full.yml new file mode 100644 index 0000000..dc2d4b9 --- /dev/null +++ b/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_full.yml @@ -0,0 +1,125 @@ +version: 2 +models: + - name: test_gold_evm__fact_transactions_full + description: "This is a view used to test all of the gold fact transactions model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: TX_POSITION + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + - txs_match_blocks: + blocks_model: ref('test_gold_evm__fact_blocks_full') + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: TX_FEE + tests: + - not_null + - name: TX_FEE_PRECISE + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: TX_TYPE + tests: + - not_null + - name: NONCE + tests: + - not_null + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: INPUT_DATA + tests: + - not_null + - name: GAS_PRICE + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: GAS_LIMIT + tests: + - not_null + - name: CUMULATIVE_GAS_USED + tests: + - not_null + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null + - name: R + tests: + - not_null + - name: S + tests: + - not_null + - name: V + tests: + - not_null + - name: FACT_TRANSACTIONS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_recent.sql b/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_recent.sql new file mode 100644 index 0000000..ea66fa0 --- /dev/null +++ b/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('core_evm__fact_transactions') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_block_lookback') }} + ) diff --git a/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_recent.yml b/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_recent.yml new file mode 100644 index 0000000..18fbd49 --- /dev/null +++ b/models/evm/gold/tests/transactions/test_gold_evm__fact_transactions_recent.yml @@ -0,0 +1,124 @@ +version: 2 +models: + - name: test_gold_evm__fact_transactions_recent + description: "This is a view used to test the last three days of fact transactions." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: TX_POSITION + - txs_match_blocks: + blocks_model: ref('test_gold_evm__fact_blocks_recent') + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: TX_FEE + tests: + - not_null + - name: TX_FEE_PRECISE + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: TX_TYPE + tests: + - not_null + - name: NONCE + tests: + - not_null + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: INPUT_DATA + tests: + - not_null + - name: GAS_PRICE + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: GAS_LIMIT + tests: + - not_null + - name: CUMULATIVE_GAS_USED + tests: + - not_null + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null + - name: R + tests: + - not_null + - name: S + tests: + - not_null + - name: V + tests: + - not_null + - name: FACT_TRANSACTIONS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/evm/silver/abis/silver_evm__abis.sql b/models/evm/silver/abis/silver_evm__abis.sql new file mode 100644 index 0000000..0197eef --- /dev/null +++ b/models/evm/silver/abis/silver_evm__abis.sql @@ -0,0 +1,162 @@ +{{ config ( + materialized = "incremental", + unique_key = "contract_address", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(contract_address,abi_hash,bytecode), SUBSTRING(contract_address,abi_hash,bytecode)", + tags = ['abis'] +) }} + +WITH verified_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + 2 as priority, + discord_username, + abi_hash + FROM + {{ ref('silver_evm__verified_abis') }} + WHERE + abi_source = 'foundation' +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + COALESCE( + MAX( + _inserted_timestamp + ), + '1970-01-01' + ) + FROM + {{ this }} + WHERE + abi_source = 'foundation' +) +{% endif %} +), +user_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + 3 AS priority + FROM + {{ ref('silver_evm__verified_abis') }} + WHERE + abi_source = 'user' + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + COALESCE( + MAX( + _inserted_timestamp + ), + '1970-01-01' + ) + FROM + {{ this }} + WHERE + abi_source = 'user' +) +{% endif %} +), +bytecode_abis AS ( + SELECT + contract_address, + abi AS DATA, + _inserted_timestamp, + 'bytecode_matched' AS abi_source, + NULL AS discord_username, + abi_hash, + 4 AS priority + FROM + {{ ref('silver_evm__bytecode_abis') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + COALESCE( + MAX( + _inserted_timestamp + ), + '1970-01-01' + ) + FROM + {{ this }} + WHERE + abi_source = 'bytecode_matched' +) +{% endif %} +), +all_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + priority + FROM + verified_abis + UNION ALL + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + priority + FROM + user_abis + UNION ALL + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + priority + FROM + bytecode_abis +), +priority_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + priority + FROM + all_abis qualify(ROW_NUMBER() over(PARTITION BY contract_address + ORDER BY + priority ASC)) = 1 +) +SELECT + p.contract_address, + p.data, + p._inserted_timestamp, + p.abi_source, + p.discord_username, + p.abi_hash, + created_contract_input AS bytecode, + {{ dbt_utils.generate_surrogate_key( + ['p.contract_address'] + ) }} AS abis_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + priority_abis p + LEFT JOIN {{ ref('silver_evm__created_contracts') }} + ON p.contract_address = created_contract_address diff --git a/models/evm/silver/abis/silver_evm__bytecode_abis.sql b/models/evm/silver/abis/silver_evm__bytecode_abis.sql new file mode 100644 index 0000000..2e92575 --- /dev/null +++ b/models/evm/silver/abis/silver_evm__bytecode_abis.sql @@ -0,0 +1,76 @@ +{{ config ( + materialized = "incremental", + unique_key = "contract_address", + tags = ['abis'] +) }} + +WITH contracts_with_abis AS ( + -- Identifying contracts with verified ABIs + + SELECT + created_contract_address AS contract_address + FROM + {{ ref('silver_evm__created_contracts') }} + JOIN {{ ref('silver_evm__verified_abis') }} A + ON A.contract_address = created_contract_address +), +contracts_without_abis AS ( + -- Contracts that are missing ABIs + SELECT + created_contract_address AS contract_address, + created_contract_input AS bytecode + FROM + {{ ref('silver_evm__created_contracts') }} + WHERE + created_contract_address NOT IN ( + SELECT + contract_address + FROM + contracts_with_abis + ) + +{% if is_incremental() %} +AND created_contract_address NOT IN ( + SELECT + contract_address + FROM + {{ this }} +) +{% endif %} +), +unique_bytecode_abis AS ( + -- Bytecodes from created_contracts with a unique ABI + SELECT + cc.created_contract_input AS bytecode, + va.data AS abi, + va.abi_hash + FROM + {{ ref('silver_evm__created_contracts') }} + cc + JOIN {{ ref('silver_evm__verified_abis') }} + va + ON cc.created_contract_address = va.contract_address + GROUP BY + cc.created_contract_input, + va.data, + va.abi_hash + HAVING + COUNT( + DISTINCT va.data + ) = 1 -- Ensuring there's only one ABI per bytecode +) -- Final matching +SELECT + contract_address, + abi, + abi_hash, + +{% if is_incremental() %} +SYSDATE() +{% else %} + TO_TIMESTAMP_NTZ('2000-01-01 00:00:00') +{% endif %} + +AS _inserted_timestamp +FROM + contracts_without_abis + JOIN unique_bytecode_abis USING (bytecode) \ No newline at end of file diff --git a/models/evm/silver/abis/silver_evm__complete_event_abis.sql b/models/evm/silver/abis/silver_evm__complete_event_abis.sql new file mode 100644 index 0000000..ee578f8 --- /dev/null +++ b/models/evm/silver/abis/silver_evm__complete_event_abis.sql @@ -0,0 +1,265 @@ +{{ config ( + materialized = 'incremental', + unique_key = ["parent_contract_address","event_signature","start_block"], + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['abis'] +) }} + +WITH new_abis AS ( + + SELECT + DISTINCT contract_address + FROM + {{ ref('silver_evm__flat_event_abis') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '18 hours' + FROM + {{ this }} + ) +UNION + -- catches any late arriving implementation contracts - when we get its ABI but no delegatecalls where made yet +SELECT + DISTINCT implementation_contract AS contract_address +FROM + {{ ref('silver_evm__proxies') }} +WHERE + start_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '18 hours' + FROM + {{ this }} + ) +{% endif %} +), +implementations AS ( + SELECT + p0.created_block, + p0.implementation_created_block, + p0.contract_address, + p0.implementation_contract, + p0.start_block, + p0._id, + p0._inserted_timestamp + FROM + {{ ref('silver_evm__proxies') }} + p0 + JOIN new_abis na0 + ON p0.contract_address = na0.contract_address + UNION + SELECT + p1.created_block, + p1.implementation_created_block, + p1.contract_address, + p1.implementation_contract, + p1.start_block, + p1._id, + p1._inserted_timestamp + FROM + {{ ref('silver_evm__proxies') }} + p1 + JOIN new_abis na1 + ON p1.implementation_contract = na1.contract_address +), +all_relevant_contracts AS ( + SELECT + DISTINCT contract_address + FROM + implementations + UNION + SELECT + DISTINCT implementation_contract AS contract_address + FROM + implementations + UNION + SELECT + contract_address + FROM + new_abis +), +flat_abis AS ( + SELECT + contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp + FROM + {{ ref('silver_evm__flat_event_abis') }} + JOIN all_relevant_contracts USING (contract_address) +), +base AS ( + SELECT + ea.contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + ea._inserted_timestamp, + pb._inserted_timestamp AS implementation_inserted_timestamp, + pb.start_block, + pb.implementation_created_block, + pb.contract_address AS base_contract_address, + 1 AS priority + FROM + flat_abis ea + JOIN implementations pb + ON ea.contract_address = pb.implementation_contract + UNION ALL + SELECT + eab.contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + eab._inserted_timestamp, + pbb._inserted_timestamp AS implementation_inserted_timestamp, + pbb.created_block AS start_block, + pbb.implementation_created_block, + pbb.contract_address AS base_contract_address, + 2 AS priority + FROM + flat_abis eab + JOIN ( + SELECT + DISTINCT contract_address, + created_block, + implementation_created_block, + _inserted_timestamp + FROM + implementations + ) pbb + ON eab.contract_address = pbb.contract_address + UNION ALL + SELECT + contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp, + NULL AS implementation_inserted_timestamp, + 0 AS start_block, + NULL AS implementation_created_block, + contract_address AS base_contract_address, + 3 AS priority + FROM + flat_abis eac + WHERE + contract_address NOT IN ( + SELECT + DISTINCT contract_address + FROM + implementations + ) +), +new_records AS ( + SELECT + base_contract_address AS parent_contract_address, + contract_address, + event_name, + abi, + start_block, + implementation_created_block, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp, + implementation_inserted_timestamp + FROM + base qualify ROW_NUMBER() over ( + PARTITION BY parent_contract_address, + NAME, + event_type, + event_signature, + start_block + ORDER BY + priority ASC, + _inserted_timestamp DESC, + implementation_created_block DESC nulls last, + implementation_inserted_timestamp DESC nulls last + ) = 1 +), +FINAL AS ( + SELECT + parent_contract_address, + contract_address AS implementation_contract, + event_name, + abi, + start_block, + implementation_created_block, + simple_event_name, + event_signature, + IFNULL(LEAD(start_block) over (PARTITION BY parent_contract_address, event_signature + ORDER BY + start_block) -1, 1e18) AS end_block, + _inserted_timestamp, + implementation_inserted_timestamp, + SYSDATE() AS _updated_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['parent_contract_address','event_signature','start_block'] + ) }} AS complete_event_abis_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + new_records qualify ROW_NUMBER() over ( + PARTITION BY parent_contract_address, + event_name, + event_signature, + start_block + ORDER BY + _inserted_timestamp DESC + ) = 1 +) +SELECT + parent_contract_address, + implementation_contract, + event_name, + abi, + start_block, + implementation_created_block, + simple_event_name, + event_signature, + end_block, + _inserted_timestamp, + implementation_inserted_timestamp, + _updated_timestamp, + complete_event_abis_id, + inserted_timestamp, + modified_timestamp, + _invocation_id +FROM + FINAL f + +{% if is_incremental() %} +LEFT JOIN {{ this }} +t USING ( + parent_contract_address, + event_name, + event_signature, + start_block, + end_block +) +WHERE + t.event_signature IS NULL +{% endif %} \ No newline at end of file diff --git a/models/evm/silver/abis/silver_evm__flat_event_abis.sql b/models/evm/silver/abis/silver_evm__flat_event_abis.sql new file mode 100644 index 0000000..3b219d1 --- /dev/null +++ b/models/evm/silver/abis/silver_evm__flat_event_abis.sql @@ -0,0 +1,117 @@ +{{ config ( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "contract_address", + cluster_by = "_inserted_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY (contract_address)", + tags = ['abis'] +) }} + +WITH abi_base AS ( + + SELECT + contract_address, + DATA, + _inserted_timestamp + FROM + {{ ref('silver_evm__abis') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + COALESCE( + MAX( + _inserted_timestamp + ) - INTERVAL '18 hours', + '1970-01-01' + ) + FROM + {{ this }} + ) +{% endif %} +), +flat_abi AS ( + SELECT + contract_address, + _inserted_timestamp, + DATA, + VALUE :inputs AS inputs, + VALUE :payable :: BOOLEAN AS payable, + VALUE :stateMutability :: STRING AS stateMutability, + VALUE :type :: STRING AS TYPE, + VALUE :anonymous :: BOOLEAN AS anonymous, + VALUE :name :: STRING AS NAME + FROM + abi_base, + LATERAL FLATTEN ( + input => DATA + ) + WHERE + TYPE = 'event' qualify ROW_NUMBER() over ( + PARTITION BY contract_address, + NAME, + inputs + ORDER BY + LENGTH(inputs) + ) = 1 +), +event_types AS ( + SELECT + contract_address, + _inserted_timestamp, + inputs, + anonymous, + NAME, + ARRAY_AGG( + VALUE :type :: STRING + ) AS event_type + FROM + flat_abi, + LATERAL FLATTEN ( + input => inputs + ) + GROUP BY + contract_address, + _inserted_timestamp, + inputs, + anonymous, + NAME +), +apply_udfs AS ( + SELECT + contract_address, + NAME AS event_name, + PARSE_JSON( + OBJECT_CONSTRUCT( + 'anonymous', + anonymous, + 'inputs', + inputs, + 'name', + NAME, + 'type', + 'event' + ) :: STRING + ) AS abi, + utils.udf_evm_text_signature(abi) AS simple_event_name, + utils.udf_keccak256(simple_event_name) AS event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp + FROM + event_types +) +SELECT + contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp +FROM + apply_udfs \ No newline at end of file diff --git a/models/evm/silver/abis/silver_evm__user_verified_abis.sql b/models/evm/silver/abis/silver_evm__user_verified_abis.sql new file mode 100644 index 0000000..b44f06f --- /dev/null +++ b/models/evm/silver/abis/silver_evm__user_verified_abis.sql @@ -0,0 +1,547 @@ +{{ config ( + materialized = "incremental", + unique_key = "id", + tags = ['abis'] +) }} + +WITH base AS ( + + SELECT + contract_address, + abi, + PARSE_JSON(abi) AS DATA, + SHA2(PARSE_JSON(abi)) AS abi_hash, + discord_username, + _inserted_timestamp + FROM + {{ source( + "crosschain_public", + "user_abis" + ) }} + WHERE + blockchain = 'flow_evm' + AND NOT duplicate_abi + +{% if is_incremental() %} +AND contract_address NOT IN ( + SELECT + contract_address + FROM + {{ this }} +) +AND _inserted_timestamp > ( + SELECT + COALESCE( + MAX( + _inserted_timestamp + ), + '1970-01-01' + ) + FROM + {{ this }} +) +AND _inserted_timestamp > DATEADD('day', -2, SYSDATE()) +{% endif %}), +flat_event_abi AS ( + SELECT + contract_address, + _inserted_timestamp, + DATA, + VALUE :inputs AS inputs, + VALUE :payable :: BOOLEAN AS payable, + VALUE :stateMutability :: STRING AS stateMutability, + VALUE :type :: STRING AS TYPE, + VALUE :anonymous :: BOOLEAN AS anonymous, + VALUE :name :: STRING AS NAME + FROM + base, + LATERAL FLATTEN ( + input => DATA + ) + WHERE + TYPE = 'event' qualify ROW_NUMBER() over ( + PARTITION BY contract_address, + NAME, + inputs + ORDER BY + LENGTH(inputs) + ) = 1 +), +event_types AS ( + SELECT + contract_address, + _inserted_timestamp, + inputs, + anonymous, + NAME, + ARRAY_AGG( + VALUE :type :: STRING + ) AS event_type + FROM + flat_event_abi, + LATERAL FLATTEN ( + input => inputs + ) + GROUP BY + contract_address, + _inserted_timestamp, + inputs, + anonymous, + NAME +), +apply_event_udfs AS ( + SELECT + contract_address, + NAME AS event_name, + PARSE_JSON( + OBJECT_CONSTRUCT( + 'anonymous', + anonymous, + 'inputs', + inputs, + 'name', + NAME, + 'type', + 'event' + ) :: STRING + ) AS abi, + utils.udf_evm_text_signature(abi) AS simple_event_name, + utils.udf_keccak256(simple_event_name) AS event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp + FROM + event_types +), +final_flat_event_abis AS ( + SELECT + contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp + FROM + apply_event_udfs +), +flat_function_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + VALUE :inputs AS inputs, + VALUE :outputs AS outputs, + VALUE :payable :: BOOLEAN AS payable, + VALUE :stateMutability :: STRING AS stateMutability, + VALUE :type :: STRING AS TYPE, + VALUE :name :: STRING AS NAME + FROM + base, + LATERAL FLATTEN ( + input => DATA + ) + WHERE + TYPE = 'function' +), +udf_function_abis AS ( + SELECT + *, + PARSE_JSON( + object_construct_keep_null( + 'inputs', + IFNULL( + inputs, + [] + ), + 'outputs', + IFNULL( + outputs, + [] + ), + 'name', + NAME, + 'type', + 'function' + ) :: STRING + ) AS abi, + utils.udf_evm_text_signature(abi) AS simple_function_name, + utils.udf_keccak256(simple_function_name) AS function_signature + FROM + flat_function_abis qualify ROW_NUMBER() over ( + PARTITION BY contract_address, + function_signature + ORDER BY + _inserted_timestamp DESC + ) = 1 +), +flat_inputs AS ( + SELECT + contract_address, + inputs, + NAME, + simple_function_name, + function_signature, + ARRAY_AGG( + VALUE :type :: STRING + ) AS inputs_type + FROM + udf_function_abis, + LATERAL FLATTEN ( + input => inputs + ) + GROUP BY + ALL +), +fill_missing_input_names AS ( + SELECT + contract_address, + NAME, + inputs_type, + simple_function_name, + function_signature, + VALUE :internalType :: STRING AS internalType, + VALUE :type :: STRING AS TYPE, + CASE + WHEN VALUE :name :: STRING = '' THEN CONCAT('input_', ROW_NUMBER() over (PARTITION BY contract_address, function_signature + ORDER BY + INDEX ASC) :: STRING) + ELSE VALUE :name :: STRING + END AS name_fixed, + inputs, + INDEX, + VALUE :components AS components + FROM + flat_inputs, + LATERAL FLATTEN ( + input => inputs + ) +), +final_flat_inputs AS ( + SELECT + contract_address, + NAME, + inputs_type, + simple_function_name, + function_signature, + ARRAY_AGG( + OBJECT_CONSTRUCT( + 'internalType', + internalType, + 'name', + name_fixed, + 'type', + TYPE, + 'components', + components + ) + ) within GROUP ( + ORDER BY + INDEX + ) AS inputs + FROM + fill_missing_input_names + GROUP BY + ALL +), +flat_outputs AS ( + SELECT + contract_address, + outputs, + simple_function_name, + function_signature, + NAME, + ARRAY_AGG( + VALUE :type :: STRING + ) AS outputs_type + FROM + udf_function_abis, + LATERAL FLATTEN ( + input => outputs + ) + GROUP BY + ALL +), +fill_missing_output_names AS ( + SELECT + contract_address, + NAME, + outputs_type, + simple_function_name, + function_signature, + VALUE :internalType :: STRING AS internalType, + VALUE :type :: STRING AS TYPE, + CASE + WHEN VALUE :name :: STRING = '' THEN CONCAT('output_', ROW_NUMBER() over (PARTITION BY contract_address, function_signature + ORDER BY + INDEX ASC) :: STRING) + ELSE VALUE :name :: STRING + END AS name_fixed, + outputs, + INDEX, + VALUE :components AS components + FROM + flat_outputs, + LATERAL FLATTEN ( + input => outputs + ) +), +final_flat_outputs AS ( + SELECT + contract_address, + NAME, + outputs_type, + simple_function_name, + function_signature, + ARRAY_AGG( + OBJECT_CONSTRUCT( + 'internalType', + internalType, + 'name', + name_fixed, + 'type', + TYPE, + 'components', + components + ) + ) within GROUP ( + ORDER BY + INDEX + ) AS outputs + FROM + fill_missing_output_names + GROUP BY + ALL +), +all_contracts AS ( + SELECT + A.contract_address, + A.name AS function_name, + i.inputs, + o.outputs, + i.inputs_type, + o.outputs_type, + A._inserted_timestamp, + A.function_signature, + A.simple_function_name + FROM + udf_function_abis A + LEFT JOIN final_flat_inputs i + ON A.contract_address = i.contract_address + AND A.function_signature = i.function_signature + LEFT JOIN final_flat_outputs o + ON A.contract_address = o.contract_address + AND A.function_signature = o.function_signature +), +apply_function_udfs AS ( + SELECT + contract_address, + function_name, + PARSE_JSON( + object_construct_keep_null( + 'inputs', + IFNULL( + inputs, + [] + ), + 'outputs', + IFNULL( + outputs, + [] + ), + 'name', + function_name, + 'type', + 'function' + ) :: STRING + ) AS abi, + simple_function_name, + function_signature, + inputs, + outputs, + inputs_type, + outputs_type, + _inserted_timestamp + FROM + all_contracts +), +final_function_abis AS ( + SELECT + contract_address, + function_name, + abi, + simple_function_name, + function_signature, + inputs, + outputs, + inputs_type, + outputs_type, + _inserted_timestamp + FROM + apply_function_udfs +), +new_abis AS ( + SELECT + DISTINCT contract_address + FROM + base +), +contracts AS ( + SELECT + contract_address + FROM + {{ ref('silver_evm__proxies') }} + JOIN new_abis USING (contract_address) +), +proxies AS ( + SELECT + p.implementation_contract, + p.contract_address + FROM + {{ ref('silver_evm__proxies') }} + p + JOIN new_abis n + ON p.implementation_contract = n.contract_address +), +final_groupings AS ( + SELECT + b.contract_address AS address, + C.contract_address, + implementation_contract, + CASE + WHEN C.contract_address IS NOT NULL + AND implementation_contract IS NOT NULL THEN 'contract' + WHEN C.contract_address IS NOT NULL THEN 'contract' + WHEN implementation_contract IS NOT NULL THEN 'implementation' + WHEN C.contract_address IS NULL + AND implementation_contract IS NULL THEN 'contract' + END AS TYPE, + p.contract_address AS proxy_address, + CASE + WHEN TYPE = 'contract' THEN address + ELSE proxy_address + END AS final_address + FROM + base b + LEFT JOIN ( + SELECT + DISTINCT contract_address + FROM + contracts + ) C + ON b.contract_address = C.contract_address + LEFT JOIN ( + SELECT + DISTINCT implementation_contract, + contract_address + FROM + proxies + ) p + ON b.contract_address = p.implementation_contract +), +identified_addresses AS ( + SELECT + DISTINCT address AS base_address, + final_address AS contract_address + FROM + final_groupings +), +function_mapping AS ( + SELECT + ia.base_address, + ia.contract_address, + LEFT( + function_signature, + 10 + ) AS function_sig + FROM + identified_addresses ia + JOIN final_function_abis ffa + ON ia.base_address = ffa.contract_address +), +valid_traces AS ( + SELECT + DISTINCT base_address + FROM + ( + SELECT + base_address + FROM + {{ ref('core_evm__fact_traces') }} + JOIN function_mapping + ON function_sig = LEFT( + input, + 10 + ) + AND IFF( + TYPE = 'DELEGATECALL', + from_address, + to_address + ) = contract_address + WHERE + block_timestamp > DATEADD('month', -12, SYSDATE()) + LIMIT + 50000) + ), event_mapping AS ( + SELECT + ia.base_address, + ia.contract_address, + event_signature + FROM + identified_addresses ia + JOIN final_flat_event_abis fea + ON ia.base_address = fea.contract_address + ), + valid_logs AS ( + SELECT + DISTINCT base_address + FROM + ( + SELECT + base_address + FROM + {{ ref('core_evm__fact_event_logs') }} + l + JOIN event_mapping ia + ON ia.contract_address = l.contract_address + AND event_signature = topics [0] :: STRING + WHERE + block_timestamp > DATEADD('month', -12, SYSDATE()) + LIMIT + 50000) + ), all_valid_addresses AS ( + SELECT + base_address + FROM + valid_traces + UNION + SELECT + base_address + FROM + valid_logs + ) + SELECT + contract_address, + abi, + discord_username, + _inserted_timestamp, + abi_hash, + CONCAT( + contract_address, + '-', + abi_hash + ) AS id + FROM + base + WHERE + contract_address IN ( + SELECT + base_address + FROM + all_valid_addresses + ) qualify(ROW_NUMBER() over(PARTITION BY contract_address + ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/evm/silver/abis/silver_evm__verified_abis.sql b/models/evm/silver/abis/silver_evm__verified_abis.sql new file mode 100644 index 0000000..0f948a3 --- /dev/null +++ b/models/evm/silver/abis/silver_evm__verified_abis.sql @@ -0,0 +1,107 @@ +{{ config ( + materialized = "incremental", + unique_key = "contract_address", + merge_update_columns = ["contract_address"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(contract_address)", + tags = ['abis'] +) }} + +WITH base AS ( + + SELECT + lower(value :"CONTRACT_ADDRESS") AS contract_address, + PARSE_JSON(DATA:abi) AS DATA, + _inserted_timestamp + FROM + +{% if is_incremental() %} + {{ ref('bronze_api__contract_abis') }} +WHERE _inserted_timestamp >= ( + SELECT + COALESCE( + MAX( + _inserted_timestamp + ), + '1970-01-01' + ) +) +{% else %} + {{ ref('bronze_api__FR_contract_abis') }} +{% endif %} + +), +block_explorer_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + 'foundation' AS abi_source + FROM + base +), +user_abis AS ( + SELECT + contract_address, + abi, + discord_username, + _inserted_timestamp, + 'user' AS abi_source, + abi_hash + FROM + {{ ref('silver_evm__user_verified_abis') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + COALESCE( + MAX(_inserted_timestamp), + '1970-01-01' + ) + FROM + {{ this }} + WHERE + abi_source = 'user' + ) + AND contract_address NOT IN ( + SELECT + contract_address + FROM + {{ this }} + ) +{% endif %} +), +all_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + NULL AS discord_username, + SHA2(DATA) AS abi_hash + FROM + block_explorer_abis + UNION + SELECT + contract_address, + PARSE_JSON(abi) AS DATA, + _inserted_timestamp, + 'user' AS abi_source, + discord_username, + abi_hash + FROM + user_abis +) +SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash +FROM + all_abis + +qualify(ROW_NUMBER() over(PARTITION BY contract_address +ORDER BY + _INSERTED_TIMESTAMP DESC)) = 1 \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__blocks.sql b/models/evm/silver/core/silver_evm__blocks.sql index a3860de..6449c2f 100644 --- a/models/evm/silver/core/silver_evm__blocks.sql +++ b/models/evm/silver/core/silver_evm__blocks.sql @@ -1,82 +1,42 @@ -- depends_on: {{ ref('bronze_evm__blocks') }} --- depends_on: {{ ref('bronze_evm__FR_blocks') }} -{{ config( - materialized = 'incremental', - unique_key = "evm_blocks_id", - incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - cluster_by = ['_inserted_timestamp :: DATE', '_partition_by_block_id'], + +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['modified_timestamp::DATE','partition_key'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", tags = ['evm'] ) }} -SELECT +WITH bronze_blocks AS ( + SELECT + block_number, + partition_key, + DATA:result AS block_json, + _inserted_timestamp + FROM + {% if is_incremental() %} + {{ ref('bronze_evm__blocks') }} + WHERE _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1900-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM {{ this }} + ) AND DATA:result IS NOT NULL + {% else %} + {{ ref('bronze_evm__FR_blocks') }} + WHERE DATA:result IS NOT NULL + {% endif %} +) + +SELECT block_number, - DATA :result :hash :: STRING AS block_hash, - TO_TIMESTAMP( - utils.udf_hex_to_int( - DATA :result :timestamp :: STRING - ) - ) AS block_timestamp, - ARRAY_SIZE( - DATA :result :transactions :: ARRAY - ) AS transaction_count, - utils.udf_hex_to_int( - DATA :result :baseFeePerGas :: STRING - ) AS base_fee_per_gas, - utils.udf_hex_to_int( - DATA :result :difficulty :: STRING - ) AS difficulty, - DATA :result :extraData :: STRING AS extra_data, - utils.udf_hex_to_int( - DATA :result :gasLimit :: STRING - ) AS gas_limit, - utils.udf_hex_to_int( - DATA :result :gasUsed :: STRING - ) AS gas_used, - DATA :result :logsBloom :: STRING AS logs_bloom, - DATA :result :miner :: STRING AS miner, - DATA :result :mixHash :: STRING AS mix_hash, - utils.udf_hex_to_int( - DATA :result :nonce :: STRING - ) AS nonce, - DATA :result :parentHash :: STRING AS parent_hash, - DATA :result :receiptsRoot :: STRING AS receipts_root, - DATA :result :sha3Uncles :: STRING AS sha3_uncles, - utils.udf_hex_to_int( - DATA :result :size :: STRING - ) AS SIZE, - DATA :result :stateRoot :: STRING AS state_root, - ZEROIFNULL( - utils.udf_hex_to_int( - DATA :result :totalDifficulty :: STRING - ) - ) AS total_difficulty, - DATA :result :transactions :: ARRAY AS transactions, - DATA :result :transactionsRoot :: STRING AS transactions_root, - DATA :result :uncles :: ARRAY AS uncles, - partition_key AS _partition_by_block_id, - {{ dbt_utils.generate_surrogate_key( - ['data:result:hash::STRING'] - ) }} AS evm_blocks_id, + partition_key, + block_json, _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['block_number']) }} AS blocks_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id -FROM - -{% if is_incremental() %} -{{ ref('bronze_evm__blocks') }} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) _inserted_timestamp - FROM - {{ this }} - ) -{% else %} - {{ ref('bronze_evm__FR_blocks') }} -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY evm_blocks_id -ORDER BY - _inserted_timestamp DESC)) = 1 +FROM bronze_blocks +QUALIFY ROW_NUMBER() OVER (PARTITION BY blocks_id ORDER BY _inserted_timestamp DESC) = 1 \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__blocks.yml b/models/evm/silver/core/silver_evm__blocks.yml deleted file mode 100644 index 393828f..0000000 --- a/models/evm/silver/core/silver_evm__blocks.yml +++ /dev/null @@ -1,111 +0,0 @@ -version: 2 -models: - - name: silver_evm__blocks - description: '{{ doc("flowevm_blocks_table_doc") }}' - tests: - - dbt_utils.recency: - datepart: hour - field: block_timestamp - interval: 2 - - columns: - - name: BLOCK_NUMBER - description: '{{ doc("flowevm_block_number") }}' - tests: - - unique - - not_null - - name: BLOCK_TIMESTAMP - description: '{{ doc("flowevm_block_timestamp") }}' - tests: - - not_null - - name: NETWORK - description: '{{ doc("flowevm_network") }}' - - name: BLOCKCHAIN - description: '{{ doc("flowevm_blockchain") }}' - - name: TRANSACTION_COUNT - description: '{{ doc("flowevm_tx_count") }}' - tests: - - not_null - - name: BASE_FEE_PER_GAS - description: '{{ doc("flowevm_base_fee_per_gas") }}' - tests: - - not_null - - name: DIFFICULTY - description: '{{ doc("flowevm_difficulty") }}' - tests: - - not_null - - name: TOTAL_DIFFICULTY - description: '{{ doc("flowevm_total_difficulty") }}' - tests: - - not_null - - name: EXTRA_DATA - description: '{{ doc("flowevm_extra_data") }}' - tests: - - not_null - - name: GAS_LIMIT - description: '{{ doc("flowevm_gas_limit") }}' - tests: - - not_null - - name: GAS_USED - description: '{{ doc("flowevm_gas_used") }}' - tests: - - not_null - - name: LOGS_BLOOM - description: '{{ doc("flowevm_logs_bloom") }}' - tests: - - not_null - - name: MINER - description: '{{ doc("flowevm_miner") }}' - tests: - - not_null - - name: MIX_HASH - description: '{{ doc("flowevm_mix_hash") }}' - tests: - - not_null - - name: NONCE - description: '{{ doc("flowevm_nonce") }}' - tests: - - not_null - - name: BLOCK_HASH - description: '{{ doc("flowevm_blocks_hash") }}' - tests: - - not_null - - name: PARENT_HASH - description: '{{ doc("flowevm_parent_hash") }}' - tests: - - not_null - - name: RECEIPTS_ROOT - description: '{{ doc("flowevm_receipts_root") }}' - tests: - - not_null - - name: SHA3_UNCLES - description: '{{ doc("flowevm_sha3_uncles") }}' - tests: - - not_null - - name: SIZE - description: '{{ doc("flowevm_size") }}' - tests: - - not_null - - name: STATE_ROOT - description: '{{ doc("flowevm_state_root") }}' - tests: - - not_null - - name: TRANSACTIONS - description: '{{ doc("flowevm_transactions") }}' - tests: - - not_null - - name: TRANSACTIONS_ROOT - description: '{{ doc("flowevm_transactions_root") }}' - - name: UNCLES - description: '{{ doc("flowevm_uncle_blocks") }}' - tests: - - not_null - - name: EVM_BLOCKS_ID - description: '{{ doc("pk_id") }}' - tests: - - not_null - - unique - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' diff --git a/models/evm/silver/core/silver_evm__contracts.sql b/models/evm/silver/core/silver_evm__contracts.sql new file mode 100644 index 0000000..803a9eb --- /dev/null +++ b/models/evm/silver/core/silver_evm__contracts.sql @@ -0,0 +1,110 @@ +{{ config( + materialized = 'incremental', + unique_key = 'contract_address', + merge_exclude_columns = ["inserted_timestamp"], + tags = ['evm'] +) }} + +WITH base_metadata AS ( + + SELECT + contract_address, + block_number, + function_sig AS function_signature, + response :data:result::string AS read_output, + _inserted_timestamp + FROM + {{ ref('bronze_evm_api__token_reads') }} + WHERE response :data:result::string is not null + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + COALESCE( + MAX( + _inserted_timestamp + ), + '1970-01-01' + ) + FROM + {{ this }} +) +{% endif %} +), +token_names AS ( + SELECT + contract_address, + block_number, + function_signature, + read_output, + utils.udf_hex_to_string( + SUBSTR(read_output,(64 * 2 + 3), len(read_output))) AS token_name + FROM + base_metadata + WHERE + function_signature = '0x06fdde03' + AND token_name IS NOT NULL + ), + token_symbols AS ( + SELECT + contract_address, + block_number, + function_signature, + read_output, + utils.udf_hex_to_string( + SUBSTR(read_output,(64 * 2 + 3), len(read_output))) AS token_symbol + FROM + base_metadata + WHERE + function_signature = '0x95d89b41' + AND token_symbol IS NOT NULL + ), + token_decimals AS ( + SELECT + contract_address, + CASE + WHEN read_output IS NOT NULL THEN utils.udf_hex_to_int( + read_output :: STRING + ) + ELSE NULL + END AS token_decimals, + LENGTH(token_decimals) AS dec_length + FROM + base_metadata + WHERE + function_signature = '0x313ce567' + AND read_output IS NOT NULL + AND read_output <> '0x' + ), + contracts AS ( + SELECT + contract_address, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + base_metadata + GROUP BY + 1 + ) + SELECT + c1.contract_address :: STRING AS contract_address, + token_name, + TRY_TO_NUMBER(token_decimals) AS token_decimals, + token_symbol, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['c1.contract_address'] + ) }} AS contracts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + contracts c1 + LEFT JOIN token_names + ON c1.contract_address = token_names.contract_address + LEFT JOIN token_symbols + ON c1.contract_address = token_symbols.contract_address + LEFT JOIN token_decimals + ON c1.contract_address = token_decimals.contract_address + AND dec_length < 3 qualify(ROW_NUMBER() over(PARTITION BY c1.contract_address + ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__created_contracts.sql b/models/evm/silver/core/silver_evm__created_contracts.sql new file mode 100644 index 0000000..5a31cd3 --- /dev/null +++ b/models/evm/silver/core/silver_evm__created_contracts.sql @@ -0,0 +1,44 @@ +{{ config ( + materialized = "incremental", + unique_key = "created_contract_address", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_timestamp, tx_hash, created_contract_address, creator_address), SUBSTRING(created_contract_address, creator_address)", + tags = ['evm'] +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + to_address AS created_contract_address, + from_address AS creator_address, + input AS created_contract_input, + inserted_timestamp AS _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['to_address'] + ) }} AS created_contracts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('core_evm__fact_traces') }} +WHERE + TYPE ILIKE 'create%' + AND to_address IS NOT NULL + AND input IS NOT NULL + AND input != '0x' + AND trace_succeeded + AND tx_succeeded + +{% if is_incremental() %} +AND inserted_timestamp >= ( + SELECT + MAX(inserted_timestamp) - INTERVAL '4 hours' + FROM + {{ this }} +) +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY created_contract_address +ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__logs.sql b/models/evm/silver/core/silver_evm__logs.sql deleted file mode 100644 index 294d97a..0000000 --- a/models/evm/silver/core/silver_evm__logs.sql +++ /dev/null @@ -1,194 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'merge', - unique_key = "evm_logs_id", - cluster_by = ['_inserted_timestamp :: DATE', '_partition_by_block_id'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['evm'] -) }} - -WITH base AS ( - - SELECT - block_number, - tx_hash, - from_address AS origin_from_address, - to_address AS origin_to_address, - tx_status, - logs, - _inserted_timestamp, - _partition_by_block_id - FROM - {{ ref('silver_evm__receipts') }} - WHERE - ARRAY_SIZE(logs) > 0 - -{% if is_incremental() %} -AND _INSERTED_TIMESTAMP >= ( - SELECT - MAX(_INSERTED_TIMESTAMP) _INSERTED_TIMESTAMP - FROM - {{ this }} -) -{% endif %} -), -flat_logs AS ( - SELECT - block_number, - tx_hash, - origin_from_address, - origin_to_address, - tx_status, - VALUE :address :: STRING AS contract_address, - VALUE :blockHash :: STRING AS block_hash, - VALUE :data :: STRING AS DATA, - utils.udf_hex_to_int( - VALUE :logIndex :: STRING - ) :: INT AS event_index, - VALUE :removed :: BOOLEAN AS event_removed, - VALUE :topics AS topics, - _inserted_timestamp, - _partition_by_block_id - FROM - base, - LATERAL FLATTEN( - input => logs - ) -), -new_records AS ( - SELECT - l.block_number, - txs.block_timestamp, - l.tx_hash, - l.origin_from_address, - l.origin_to_address, - txs.origin_function_signature, - l.tx_status, - l.contract_address, - l.block_hash, - l.data, - l.event_index, - l.event_removed, - l.topics, - l._inserted_timestamp, - CASE - WHEN txs.block_timestamp IS NULL - OR txs.origin_function_signature IS NULL THEN TRUE - ELSE FALSE - END AS is_pending, - CONCAT( - l.tx_hash :: STRING, - '-', - l.event_index :: STRING - ) AS _log_id, - l._partition_by_block_id - FROM - flat_logs l - LEFT OUTER JOIN {{ ref('silver_evm__transactions') }} - txs - ON l.block_number = txs.block_number - AND l.tx_hash = txs.tx_hash - -{% if is_incremental() %} -AND txs._INSERTED_TIMESTAMP >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 1 - FROM - {{ this }} -) -{% endif %} -) - -{% if is_incremental() %}, -missing_data AS ( - SELECT - t.block_number, - txs.block_timestamp, - t.tx_hash, - t.origin_from_address, - t.origin_to_address, - txs.origin_function_signature, - t.tx_status, - t.contract_address, - t.block_hash, - t.data, - t.event_index, - t.event_removed, - t.topics, - GREATEST( - t._inserted_timestamp, - txs._inserted_timestamp - ) AS _inserted_timestamp, - _log_id, - FALSE AS is_pending, - t._partition_by_block_id - FROM - {{ this }} - t - INNER JOIN {{ ref('silver_evm__transactions') }} - txs USING ( - block_number, - tx_hash - ) - WHERE - t.is_pending -) -{% endif %}, -FINAL AS ( - SELECT - block_number, - block_timestamp, - tx_hash, - origin_from_address, - origin_to_address, - origin_function_signature, - tx_status, - contract_address, - block_hash, - DATA, - event_index, - event_removed, - topics, - _inserted_timestamp, - _log_id, - is_pending, - _partition_by_block_id - FROM - new_records - -{% if is_incremental() %} -UNION -SELECT - block_number, - block_timestamp, - tx_hash, - origin_from_address, - origin_to_address, - origin_function_signature, - tx_status, - contract_address, - block_hash, - DATA, - event_index, - event_removed, - topics, - _inserted_timestamp, - _log_id, - is_pending, - _partition_by_block_id -FROM - missing_data -{% endif %} -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'event_index'] - ) }} AS evm_logs_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_hash, event_index -ORDER BY - _inserted_timestamp DESC, is_pending ASC)) = 1 diff --git a/models/evm/silver/core/silver_evm__logs.yml b/models/evm/silver/core/silver_evm__logs.yml deleted file mode 100644 index f190b7e..0000000 --- a/models/evm/silver/core/silver_evm__logs.yml +++ /dev/null @@ -1,74 +0,0 @@ -version: 2 -models: - - name: silver_evm__logs - description: '{{ doc("flowevm_logs_table_doc") }}' - tests: - - dbt_utils.recency: - datepart: hour - field: block_timestamp - interval: 2 - - columns: - - name: BLOCK_NUMBER - description: '{{ doc("flowevm_block_number") }}' - tests: - - not_null - - name: BLOCK_HASH - description: '{{ doc("flowevm_block_hash") }}' - tests: - - not_null - - name: BLOCK_TIMESTAMP - description: '{{ doc("flowevm_block_timestamp") }}' - tests: - - not_null - - name: TX_HASH - description: '{{ doc("flowevm_logs_tx_hash") }}' - tests: - - not_null - - name: EVENT_INDEX - description: '{{ doc("flowevm_event_index") }}' - tests: - - not_null - - name: CONTRACT_ADDRESS - description: '{{ doc("flowevm_logs_contract_address") }}' - tests: - - not_null - - name: TOPICS - description: '{{ doc("flowevm_topics") }}' - tests: - - not_null - - name: DATA - description: '{{ doc("flowevm_logs_data") }}' - tests: - - not_null - - name: EVENT_REMOVED - description: '{{ doc("flowevm_event_removed") }}' - tests: - - not_null - - name: _LOG_ID - description: '{{ doc("internal_column") }}' - tests: - - not_null - - name: TX_STATUS - description: '{{ doc("flowevm_tx_status") }}' - tests: - - not_null - - name: ORIGIN_FUNCTION_SIGNATURE - description: '{{ doc("flowevm_origin_sig") }}' - tests: - - not_null - - name: ORIGIN_FROM_ADDRESS - description: '{{ doc("flowevm_origin_from") }}' - - name: ORIGIN_TO_ADDRESS - description: '{{ doc("flowevm_origin_to") }}' - - name: IS_PENDING - description: '{{ doc("is_pending") }}' - - name: EVM_LOGS_ID - description: '{{ doc("pk_id") }}' - tests: - - not_null - - unique - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' diff --git a/models/evm/silver/core/silver_evm__proxies.sql b/models/evm/silver/core/silver_evm__proxies.sql new file mode 100644 index 0000000..649f03c --- /dev/null +++ b/models/evm/silver/core/silver_evm__proxies.sql @@ -0,0 +1,116 @@ +{{ config ( + materialized = 'incremental', + unique_key = ["contract_address", "implementation_contract"], + cluster_by = ["start_timestamp::date"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['evm'] +) }} + +WITH base AS ( + + SELECT + from_address, + to_address, + MIN(block_number) AS start_block, + MIN(block_timestamp) AS start_timestamp, + MAX(inserted_timestamp) AS _inserted_timestamp + FROM + {{ ref('core_evm__fact_traces') }} + WHERE + TYPE = 'DELEGATECALL' + AND trace_succeeded + AND tx_succeeded + AND from_address != to_address -- exclude self-calls + +{% if is_incremental() %} +AND inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '4 hours' + FROM + {{ this }} +) +{% endif %} +GROUP BY + ALL +), +create_id AS ( + SELECT + from_address AS contract_address, + to_address AS implementation_contract, + start_block, + start_timestamp, + CONCAT( + from_address, + '-', + to_address + ) AS _id, + _inserted_timestamp + FROM + base +), +heal AS ( + SELECT + contract_address, + implementation_contract, + start_block, + start_timestamp, + _id, + _inserted_timestamp + FROM + create_id + +{% if is_incremental() %} +UNION ALL +SELECT + contract_address, + implementation_contract, + start_block, + start_timestamp, + _id, + _inserted_timestamp +FROM + {{ this }} + JOIN create_id USING ( + contract_address, + implementation_contract + ) +{% endif %} +), +FINAL AS ( + SELECT + contract_address, + implementation_contract, + start_block, + start_timestamp, + _id, + _inserted_timestamp + FROM + heal qualify ROW_NUMBER() over ( + PARTITION BY contract_address, + implementation_contract + ORDER BY + start_block ASC + ) = 1 +) +SELECT + f.contract_address, + f.implementation_contract, + f.start_block, + f.start_timestamp, + f._id, + f._inserted_timestamp, + COALESCE( + C.block_number, + 0 + ) AS created_block, + COALESCE( + p.block_number, + 0 + ) AS implementation_created_block +FROM + FINAL f + LEFT JOIN {{ ref('silver_evm__created_contracts') }} C + ON f.contract_address = C.created_contract_address + LEFT JOIN {{ ref('silver_evm__created_contracts') }} + p + ON f.implementation_contract = p.created_contract_address \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__receipts.sql b/models/evm/silver/core/silver_evm__receipts.sql index 42f1420..64da30d 100644 --- a/models/evm/silver/core/silver_evm__receipts.sql +++ b/models/evm/silver/core/silver_evm__receipts.sql @@ -1,20 +1,20 @@ -- depends_on: {{ ref('bronze_evm__receipts') }} -- depends_on: {{ ref('bronze_evm__FR_receipts') }} -{{ config( - materialized = 'incremental', - unique_key = "evm_receipts_id", - incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - cluster_by = ['_inserted_timestamp :: DATE', '_partition_by_block_id'], +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['modified_timestamp::DATE','partition_key'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", tags = ['evm'] ) }} -WITH receipts AS ( +WITH bronze AS ( SELECT block_number, DATA, - partition_key AS _partition_by_block_id, + partition_key, _inserted_timestamp FROM @@ -26,106 +26,36 @@ WHERE MAX(_inserted_timestamp) _inserted_timestamp FROM {{ this }} - ) + ) AND data:result[0] is not null {% else %} {{ ref('bronze_evm__FR_receipts') }} + WHERE data:result[0] is not null {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number ORDER BY _inserted_timestamp DESC)) = 1 ), -FINAL AS ( - SELECT +flat_receipts AS ( + SELECT block_number, - COALESCE( - VALUE :PrecompiledCalls :: STRING, - VALUE :precompiledCalls :: STRING - ) AS precompiled_calls, - VALUE :blobGasPrice :: INT AS blob_gas_price, - VALUE :blockHash :: STRING AS block_hash, - IFF(LEFT(VALUE :blockNumber :: STRING, 2) = '0x', utils.udf_hex_to_int(VALUE :blockNumber :: STRING), VALUE :blockNumber) :: INT AS blockNumber, - VALUE :contractAddress :: STRING AS contract_address, - IFF(LEFT(VALUE :cumulativeGasUsed :: STRING, 2) = '0x', utils.udf_hex_to_int(VALUE :cumulativeGasUsed :: STRING), VALUE :cumulativeGasUsed) :: INT AS cumulative_gas_used, - IFF(LEFT(VALUE :effectiveGasPrice :: STRING, 2) = '0x', utils.udf_hex_to_int(VALUE :effectiveGasPrice :: STRING), VALUE :effectiveGasPrice) :: INT AS effective_gas_price_unadj, - VALUE :from :: STRING AS from_address, - effective_gas_price_unadj / pow( - 10, - 9 - ) AS effective_gas_price_adj, - ZEROIFNULL( - IFF(LEFT(VALUE :gasUsed :: STRING, 2) = '0x', utils.udf_hex_to_int(VALUE :gasUsed :: STRING), VALUE :gasUsed) :: INT - ) AS gas_used, - VALUE :logs :: ARRAY AS logs, - VALUE :logsBloom :: STRING AS logs_bloom, - VALUE :revertReason :: STRING AS revert_reason, - VALUE :root :: STRING AS root, - IFF(LEFT(VALUE :status :: STRING, 2) = '0x', utils.udf_hex_to_int(VALUE :status :: STRING), VALUE :status) :: INT AS status, - status = 1 AS tx_succeeded, - IFF( - tx_succeeded, - 'SUCCESS', - 'FAIL' - ) AS tx_status, - VALUE :transactionHash :: STRING AS tx_hash, - IFF( - LEFT( - VALUE :transactionIndex :: STRING, - 2 - ) = '0x', - utils.udf_hex_to_int( - VALUE :transactionIndex :: STRING - ), - VALUE :transactionIndex - ) :: INT AS tx_index, - CASE - WHEN block_number <> blockNumber THEN NULL - ELSE tx_index - END AS POSITION, - IFF(LEFT(VALUE :type :: STRING, 2) = '0x', utils.udf_hex_to_int(VALUE :type :: STRING), VALUE :type) :: INT AS receipt_type, - VALUE :to :: STRING AS to_address, - _partition_by_block_id, + partition_key, + index :: INT AS array_index, + value AS receipts_json, _inserted_timestamp - FROM - receipts, - LATERAL FLATTEN ( - DATA :result :: variant - ) + FROM bronze, + LATERAL FLATTEN(input => data:result) AS receipt ) -SELECT +SELECT block_number, - precompiled_calls, - blob_gas_price, - block_hash, - blockNumber, - contract_address, - cumulative_gas_used, - effective_gas_price_unadj, - effective_gas_price_adj, - from_address, - gas_used, - logs, - logs_bloom, - revert_reason, - root, - status, - tx_succeeded, - tx_status, - tx_hash, - tx_index, - POSITION, - receipt_type, - to_address, - _partition_by_block_id, + partition_key, + array_index, + receipts_json, _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'tx_hash'] - ) }} AS evm_receipts_id, + {{ dbt_utils.generate_surrogate_key(['block_number','array_index']) }} AS receipts_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id -FROM - FINAL -WHERE - tx_hash IS NOT NULL - AND POSITION IS NOT NULL +FROM flat_receipts + +QUALIFY(ROW_NUMBER() OVER (PARTITION BY block_number, array_index ORDER BY _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__relevant_contracts.sql b/models/evm/silver/core/silver_evm__relevant_contracts.sql new file mode 100644 index 0000000..a4b2d9c --- /dev/null +++ b/models/evm/silver/core/silver_evm__relevant_contracts.sql @@ -0,0 +1,134 @@ +{{ config( + materialized = 'incremental', + unique_key = "contract_address", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(contract_address)", + tags = ['silver_core'] +) }} + +WITH emitted_events AS ( + + SELECT + contract_address, + COUNT(*) AS event_count, + MAX(inserted_timestamp) AS max_inserted_timestamp_logs, + MAX(block_number) AS latest_event_block + FROM + {{ ref('core_evm__fact_event_logs') }} + +{% if is_incremental() %} +WHERE + inserted_timestamp > ( + SELECT + MAX(max_inserted_timestamp_logs) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + contract_address +), +function_calls AS ( + SELECT + to_address AS contract_address, + COUNT(*) AS function_call_count, + MAX(inserted_timestamp) AS max_inserted_timestamp_traces, + MAX(block_number) AS latest_call_block + FROM + {{ ref('core_evm__fact_traces') }} + WHERE + tx_succeeded + AND trace_succeeded + AND to_address IS NOT NULL + AND input IS NOT NULL + AND input <> '0x' + +{% if is_incremental() %} +AND inserted_timestamp > ( + SELECT + MAX(max_inserted_timestamp_traces) + FROM + {{ this }} +) +{% endif %} +GROUP BY + 1 +), +active_contracts AS ( + SELECT + contract_address + FROM + emitted_events + UNION + SELECT + contract_address + FROM + function_calls +), +previous_totals AS ( + +{% if is_incremental() %} +SELECT + contract_address, total_event_count, total_call_count, max_inserted_timestamp_logs, latest_event_block, max_inserted_timestamp_traces, latest_call_block +FROM + {{ this }} +{% else %} +SELECT + NULL AS contract_address, 0 AS total_event_count, 0 AS total_call_count, '1970-01-01 00:00:00' AS max_inserted_timestamp_logs, 0 AS latest_event_block, '1970-01-01 00:00:00' AS max_inserted_timestamp_traces, 0 AS latest_call_block +{% endif %}) +SELECT + C.contract_address, + COALESCE( + p.total_event_count, + 0 + ) + COALESCE( + e.event_count, + 0 + ) AS total_event_count, + COALESCE( + p.total_call_count, + 0 + ) + COALESCE( + f.function_call_count, + 0 + ) AS total_call_count, + COALESCE( + p.total_event_count, + 0 + ) + COALESCE( + e.event_count, + 0 + ) + COALESCE( + p.total_call_count, + 0 + ) + COALESCE( + f.function_call_count, + 0 + ) AS total_interaction_count, + COALESCE( + e.max_inserted_timestamp_logs, + p.max_inserted_timestamp_logs, + '1970-01-01 00:00:00' + ) AS max_inserted_timestamp_logs, + COALESCE( + f.max_inserted_timestamp_traces, + p.max_inserted_timestamp_traces, + '1970-01-01 00:00:00' + ) AS max_inserted_timestamp_traces, + COALESCE( + e.latest_event_block, + p.latest_event_block, + 0 + ) AS latest_event_block, + COALESCE( + f.latest_call_block, + p.latest_call_block, + 0 + ) AS latest_call_block +FROM + active_contracts C + LEFT JOIN emitted_events e + ON C.contract_address = e.contract_address + LEFT JOIN function_calls f + ON C.contract_address = f.contract_address + LEFT JOIN previous_totals p + ON C.contract_address = p.contract_address \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__traces.yml b/models/evm/silver/core/silver_evm__traces.yml deleted file mode 100644 index 0e015a6..0000000 --- a/models/evm/silver/core/silver_evm__traces.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 -models: - - name: silver_evm__traces - tests: - - dbt_utils.recency: - datepart: hour - field: inserted_timestamp - interval: 2 - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - traces_id \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__transactions.sql b/models/evm/silver/core/silver_evm__transactions.sql index 6aa3cfc..e90554f 100644 --- a/models/evm/silver/core/silver_evm__transactions.sql +++ b/models/evm/silver/core/silver_evm__transactions.sql @@ -1,335 +1,53 @@ -{{ config( - materialized = 'incremental', - unique_key = "evm_txs_id", +-- depends_on: {{ ref('bronze_evm__blocks') }} + +{{ config ( + materialized = "incremental", incremental_strategy = 'delete+insert', - cluster_by = ['block_number'], + unique_key = "block_number", + cluster_by = ['modified_timestamp::DATE','partition_key'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", tags = ['evm'] ) }} -WITH tx_array AS ( - - SELECT - block_number, - block_hash, - block_timestamp, - transactions, - _partition_by_block_id, - _inserted_timestamp - FROM - {{ ref('silver_evm__blocks') }} - WHERE - transaction_count > 0 - -{% if is_incremental() %} -AND modified_timestamp >= ( - SELECT - MAX(modified_timestamp) modified_timestamp - FROM - {{ this }} -) -{% endif %} -), -flatten_txs AS ( - SELECT - block_number, - block_hash, - block_timestamp, - INDEX AS array_index, - VALUE :: variant AS tx_response, - _partition_by_block_id, - _inserted_timestamp - FROM - tx_array, - LATERAL FLATTEN (transactions) -), -base_tx AS ( - SELECT - block_number, - block_hash, - block_timestamp, - utils.udf_hex_to_int( - tx_response :blockNumber :: STRING - ) AS blockNumber, - utils.udf_hex_to_int( - tx_response :chainId :: STRING - ) AS chain_id, - tx_response :from :: STRING AS from_address, - utils.udf_hex_to_int( - tx_response :gas :: STRING - ) AS gas, - utils.udf_hex_to_int( - tx_response :gasPrice :: STRING - ) AS gas_price_unadj, - ZEROIFNULL(gas_price_unadj / pow(10, 9)) AS gas_price_adj, - tx_response :hash :: STRING AS tx_hash, - tx_response :input :: STRING AS input_data, - SUBSTR( - input_data, - 1, - 10 - ) AS origin_function_signature, - -- note, no maxFeePerGas or maxPriorityFeePerGas - utils.udf_hex_to_int( - tx_response :nonce :: STRING - ) AS nonce, - utils.udf_hex_to_int( - tx_response :r :: STRING - ) AS r, - utils.udf_hex_to_int( - tx_response :s :: STRING - ) AS s, - -- note, no sourceHash - tx_response :to :: STRING AS to_address, - utils.udf_hex_to_int( - tx_response :transactionIndex :: STRING - ) AS POSITION, - utils.udf_hex_to_int( - tx_response :type :: STRING - ) AS tx_type, - utils.udf_hex_to_int( - tx_response :v :: STRING - ) AS v, - utils.udf_hex_to_int( - tx_response :value :: STRING - ) AS value_precise_unadj, - value_precise_unadj / pow( - 10, - 18 - ) AS value_precise_adj, - value_precise_adj :: FLOAT AS VALUE, - -- note, no yParity - _partition_by_block_id, - _inserted_timestamp - FROM - flatten_txs -), -new_records AS ( - SELECT - t.block_number, - t.block_hash, - t.chain_id, - t.from_address, - t.gas, - t.gas_price_unadj, - t.gas_price_adj, - t.tx_hash, - t.input_data, - t.origin_function_signature, - t.nonce, - t.r, - t.s, - t.to_address, - t.position, - t.tx_type, - t.v, - t.value_precise_unadj, - t.value_precise_adj, - t.value, - t.block_timestamp, - r.tx_status IS NULL AS is_pending, - r.gas_used, - r.gas_used * t.gas_price_unadj :: bigint / pow( - 10, - 18 - ) AS tx_fee_precise, - ZEROIFNULL( - tx_fee_precise :: FLOAT - ) AS tx_fee, - r.tx_succeeded, - r.tx_status, - r.cumulative_gas_used, - r.effective_gas_price_adj AS effective_gas_price, - r.receipt_type, - t._inserted_timestamp, - t._partition_by_block_id - FROM - base_tx t - LEFT OUTER JOIN {{ ref('silver_evm__receipts') }} - r - ON t.block_number = r.block_number - AND t.tx_hash = r.tx_hash - -{% if is_incremental() %} -AND r._INSERTED_TIMESTAMP >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 1 - FROM - {{ this }} -) -{% endif %} -), - -{% if is_incremental() %} -missing_data AS ( - SELECT - t.block_number, - t.block_hash, - t.chain_id, - t.from_address, - t.gas, - t.gas_price_adj, - t.gas_price_unadj, - t.tx_hash, - t.input_data, - t.origin_function_signature, - t.nonce, - t.r, - t.s, - t.to_address, - t.position, - t.tx_type, - t.v, - t.value_precise_unadj, - t.value_precise_adj, - t.value, - t.block_timestamp, - FALSE AS is_pending, - r.gas_used, - r.tx_succeeded, - r.tx_status, - r.cumulative_gas_used, - r.effective_gas_price_adj AS effective_gas_price, - r.gas_used * t.gas_price_unadj :: bigint / pow( - 10, - 18 - ) AS tx_fee_precise_heal, - ZEROIFNULL( - tx_fee_precise_heal :: FLOAT - ) AS tx_fee, - r.receipt_type, - GREATEST( - t._inserted_timestamp, - r._inserted_timestamp - ) AS _inserted_timestamp, - t._partition_by_block_id - FROM - {{ this }} - t - INNER JOIN {{ ref('silver_evm__receipts') }} - r - ON t.tx_hash = r.tx_hash - AND t.block_number = r.block_number - WHERE - t.is_pending -), -{% endif %} - -FINAL AS ( - SELECT - block_number, - block_hash, - chain_id, - from_address, - gas, - gas_price_adj, - gas_price_unadj, - tx_hash, - input_data, - origin_function_signature, - nonce, - r, - s, - to_address, - POSITION, - tx_type, - v, - VALUE, - value_precise_unadj, - value_precise_adj, - block_timestamp, - is_pending, - gas_used, - tx_succeeded, - tx_status, - cumulative_gas_used, - effective_gas_price, - tx_fee, - tx_fee_precise, - receipt_type, +WITH flat_txs as ( + SELECT + block_number, + partition_key, _inserted_timestamp, - _partition_by_block_id - FROM - new_records - -{% if is_incremental() %} -UNION -SELECT - block_number, - block_hash, - chain_id, - from_address, - gas, - gas_price_adj, - gas_price_unadj, - tx_hash, - input_data, - origin_function_signature, - nonce, - r, - s, - to_address, - POSITION, - tx_type, - v, - VALUE, - value_precise_unadj, - value_precise_adj, - block_timestamp, - is_pending, - gas_used, - tx_succeeded, - tx_status, - cumulative_gas_used, - effective_gas_price, - tx_fee, - tx_fee_precise_heal AS tx_fee_precise, - receipt_type, - _inserted_timestamp, - _partition_by_block_id -FROM - missing_data -{% endif %} + data + FROM + {% if is_incremental() %} + {{ ref('bronze_evm__blocks') }} + WHERE _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1900-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM {{ this }} + ) AND data:result:transactions[0] is not null + {% else %} + {{ ref('bronze_evm__FR_blocks') }} + WHERE data:result:transactions[0] is not null + {% endif %} +), +bronze_transactions AS ( + SELECT + block_number, + partition_key, + index :: INT AS tx_position, + value AS transaction_json, + _inserted_timestamp + FROM flat_txs, + LATERAL FLATTEN(input => data:result:transactions) AS tx ) -SELECT + +SELECT block_number, - block_hash, - chain_id, - from_address, - gas, - gas_price_adj, - gas_price_unadj, - tx_hash, - input_data, - origin_function_signature, - nonce, - r, - s, - to_address, - POSITION, - tx_type AS TYPE, - v, - VALUE, - value_precise_unadj, - value_precise_adj, - block_timestamp, - is_pending, - gas_used, - tx_succeeded, - tx_status, - cumulative_gas_used, - effective_gas_price, - tx_fee, - tx_fee_precise, - receipt_type AS tx_type, + partition_key, + tx_position, + transaction_json, _inserted_timestamp, - _partition_by_block_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash'] - ) }} AS evm_txs_id, + {{ dbt_utils.generate_surrogate_key(['block_number','tx_position']) }} AS transactions_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id -FROM - FINAL qualify(ROW_NUMBER() over (PARTITION BY tx_hash -ORDER BY - _inserted_timestamp DESC) = 1) +FROM bronze_transactions +QUALIFY ROW_NUMBER() OVER (PARTITION BY transactions_id ORDER BY _inserted_timestamp DESC) = 1 \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__transactions.yml b/models/evm/silver/core/silver_evm__transactions.yml deleted file mode 100644 index b0ea8df..0000000 --- a/models/evm/silver/core/silver_evm__transactions.yml +++ /dev/null @@ -1,130 +0,0 @@ -version: 2 -models: - - name: silver_evm__transactions - description: '{{ doc("flowevm_tx_table_doc") }}' - tests: - - dbt_utils.recency: - datepart: hour - field: block_timestamp - interval: 2 - - columns: - - name: BLOCK_NUMBER - description: '{{ doc("flowevm_block_number") }}' - tests: - - not_null - - name: BLOCK_TIMESTAMP - description: '{{ doc("flowevm_block_timestamp") }}' - tests: - - not_null - - name: BLOCK_HASH - description: '{{ doc("flowevm_block_hash") }}' - tests: - - not_null - - name: CHAIN_ID - description: '{{ doc("flowevm_chain_id") }}' - tests: - - not_null - - name: TX_HASH - description: '{{ doc("flowevm_tx_hash") }}' - tests: - - not_null - - unique - - name: NONCE - description: '{{ doc("flowevm_tx_nonce") }}' - tests: - - not_null - - name: POSITION - description: '{{ doc("flowevm_tx_position") }}' - tests: - - not_null - - name: FROM_ADDRESS - description: '{{ doc("flowevm_from_address") }}' - - name: TO_ADDRESS - description: '{{ doc("flowevm_to_address") }}' - - name: TX_TYPE - description: '{{ doc("flowevm_tx_type") }}' - - name: VALUE - description: '{{ doc("flowevm_value") }}' - tests: - - not_null - - name: VALUE_PRECISE_UNADJ - description: '{{ doc("precise_amount_unadjusted") }}' - tests: - - not_null - - name: VALUE_PRECISE_ADJ - description: '{{ doc("precise_amount_adjusted") }}' - tests: - - not_null - - name: IS_PENDING - description: '{{ doc("is_pending") }}' - tests: - - not_null - - name: TX_FEE - description: '{{ doc("flowevm_tx_fee") }}' - tests: - - not_null - - name: TX_FEE_PRECISE - description: '{{ doc("tx_fee_precise") }}' - tests: - - not_null - - name: GAS_PRICE_ADJ - description: '{{ doc("flowevm_tx_gas_price") }}' - tests: - - not_null - - name: GAS_PRICE_UNADJ - description: '{{ doc("flowevm_tx_gas_price") }}' - tests: - - not_null - - name: EFFECTIVE_GAS_PRICE - description: The total base charge plus tip paid for each unit of gas, in Gwei. - tests: - - not_null - - name: GAS - description: '{{ doc("flowevm_tx_gas_limit") }}' - tests: - - not_null - - name: GAS_USED - description: '{{ doc("flowevm_tx_gas_used") }}' - tests: - - not_null - - name: CUMULATIVE_GAS_USED - description: '{{ doc("flowevm_cumulative_gas_used") }}' - tests: - - not_null - - name: TX_SUCCEEDED - description: '{{ doc("flowevm_tx_status") }}' - tests: - - not_null - - name: TX_STATUS - description: '{{ doc("flowevm_tx_status") }}' - tests: - - not_null - - name: INPUT_DATA - description: '{{ doc("flowevm_tx_input_data") }}' - tests: - - not_null - - name: ORIGIN_FUNCTION_SIGNATURE - description: '{{ doc("flowevm_tx_origin_sig") }}' - tests: - - not_null - - name: R - description: The r value of the transaction signature. - tests: - - not_null - - name: S - description: The s value of the transaction signature. - tests: - - not_null - - name: V - description: The v value of the transaction signature. - tests: - - not_null - - name: EVM_TXS_ID - description: '{{ doc("pk_id") }}' - tests: - - not_null - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' diff --git a/models/evm/silver/tests/blocks/test_silver_evm__blocks_full.sql b/models/evm/silver/tests/blocks/test_silver_evm__blocks_full.sql new file mode 100644 index 0000000..83c52b3 --- /dev/null +++ b/models/evm/silver/tests/blocks/test_silver_evm__blocks_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__blocks') }} \ No newline at end of file diff --git a/models/evm/silver/tests/blocks/test_silver_evm__blocks_full.yml b/models/evm/silver/tests/blocks/test_silver_evm__blocks_full.yml new file mode 100644 index 0000000..2a7b3b2 --- /dev/null +++ b/models/evm/silver/tests/blocks/test_silver_evm__blocks_full.yml @@ -0,0 +1,51 @@ +version: 2 +models: + - name: test_silver_evm__blocks_full + description: "This is a view used to test all of the silver blocks model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">10" + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/silver/tests/blocks/test_silver_evm__blocks_recent.sql b/models/evm/silver/tests/blocks/test_silver_evm__blocks_recent.sql new file mode 100644 index 0000000..d180a6a --- /dev/null +++ b/models/evm/silver/tests/blocks/test_silver_evm__blocks_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__blocks') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_block_lookback') }} + ) \ No newline at end of file diff --git a/models/evm/silver/tests/blocks/test_silver_evm__blocks_recent.yml b/models/evm/silver/tests/blocks/test_silver_evm__blocks_recent.yml new file mode 100644 index 0000000..d403a97 --- /dev/null +++ b/models/evm/silver/tests/blocks/test_silver_evm__blocks_recent.yml @@ -0,0 +1,51 @@ +version: 2 +models: + - name: test_silver_evm__blocks_recent + description: "This is a view used to test the last three days of blocks." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">10" + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/silver/tests/contracts/test_silver_evm__contracts_full.sql b/models/evm/silver/tests/contracts/test_silver_evm__contracts_full.sql new file mode 100644 index 0000000..001229d --- /dev/null +++ b/models/evm/silver/tests/contracts/test_silver_evm__contracts_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__contracts') }} diff --git a/models/evm/silver/tests/contracts/test_silver_evm__contracts_full.yml b/models/evm/silver/tests/contracts/test_silver_evm__contracts_full.yml new file mode 100644 index 0000000..dea278d --- /dev/null +++ b/models/evm/silver/tests/contracts/test_silver_evm__contracts_full.yml @@ -0,0 +1,11 @@ +version: 2 +models: + - name: test_silver_evm__contracts_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CONTRACT_ADDRESS + columns: + - name: CONTRACT_ADDRESS + tests: + - not_null \ No newline at end of file diff --git a/models/evm/silver/tests/contracts/test_silver_evm__contracts_recent.sql b/models/evm/silver/tests/contracts/test_silver_evm__contracts_recent.sql new file mode 100644 index 0000000..5130d66 --- /dev/null +++ b/models/evm/silver/tests/contracts/test_silver_evm__contracts_recent.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__contracts') }} +WHERE + inserted_timestamp > DATEADD(DAY, -5, SYSDATE()) diff --git a/models/evm/silver/tests/contracts/test_silver_evm__contracts_recent.yml b/models/evm/silver/tests/contracts/test_silver_evm__contracts_recent.yml new file mode 100644 index 0000000..c54e44c --- /dev/null +++ b/models/evm/silver/tests/contracts/test_silver_evm__contracts_recent.yml @@ -0,0 +1,11 @@ +version: 2 +models: + - name: test_silver_evm__contracts_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CONTRACT_ADDRESS + columns: + - name: CONTRACT_ADDRESS + tests: + - not_null \ No newline at end of file diff --git a/models/evm/silver/tests/receipts/test_silver_evm__receipts_full.sql b/models/evm/silver/tests/receipts/test_silver_evm__receipts_full.sql new file mode 100644 index 0000000..a9be207 --- /dev/null +++ b/models/evm/silver/tests/receipts/test_silver_evm__receipts_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__receipts') }} diff --git a/models/evm/silver/tests/receipts/test_silver_evm__receipts_full.yml b/models/evm/silver/tests/receipts/test_silver_evm__receipts_full.yml new file mode 100644 index 0000000..cdeb045 --- /dev/null +++ b/models/evm/silver/tests/receipts/test_silver_evm__receipts_full.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: test_silver_evm__receipts_full + description: "This is a view used to test all of the silver receipts model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - RECEIPTS_ID + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: ARRAY_INDEX + tests: + - not_null + - name: RECEIPTS_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/silver/tests/receipts/test_silver_evm__receipts_recent.sql b/models/evm/silver/tests/receipts/test_silver_evm__receipts_recent.sql new file mode 100644 index 0000000..9984ff4 --- /dev/null +++ b/models/evm/silver/tests/receipts/test_silver_evm__receipts_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__receipts') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_block_lookback') }} + ) diff --git a/models/evm/silver/tests/receipts/test_silver_evm__receipts_recent.yml b/models/evm/silver/tests/receipts/test_silver_evm__receipts_recent.yml new file mode 100644 index 0000000..36757a3 --- /dev/null +++ b/models/evm/silver/tests/receipts/test_silver_evm__receipts_recent.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: test_silver_evm__receipts_recent + description: "This is a view used to test the last three days of receipts." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - RECEIPTS_ID + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: ARRAY_INDEX + tests: + - not_null + - name: RECEIPTS_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/silver/tests/traces/test_silver_evm__traces_full.sql b/models/evm/silver/tests/traces/test_silver_evm__traces_full.sql new file mode 100644 index 0000000..4facbad --- /dev/null +++ b/models/evm/silver/tests/traces/test_silver_evm__traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__traces') }} diff --git a/models/evm/silver/tests/traces/test_silver_evm__traces_full.yml b/models/evm/silver/tests/traces/test_silver_evm__traces_full.yml new file mode 100644 index 0000000..8f43ac2 --- /dev/null +++ b/models/evm/silver/tests/traces/test_silver_evm__traces_full.yml @@ -0,0 +1,52 @@ +version: 2 +models: + - name: test_silver_evm__traces_full + description: "This is a view used to test all of the silver traces model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TRACES_ID + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_POSITION + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: TRACE_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/silver/tests/traces/test_silver_evm__traces_recent.sql b/models/evm/silver/tests/traces/test_silver_evm__traces_recent.sql new file mode 100644 index 0000000..7dae606 --- /dev/null +++ b/models/evm/silver/tests/traces/test_silver_evm__traces_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__traces') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_block_lookback') }} + ) diff --git a/models/evm/silver/tests/traces/test_silver_evm__traces_recent.yml b/models/evm/silver/tests/traces/test_silver_evm__traces_recent.yml new file mode 100644 index 0000000..ba0107b --- /dev/null +++ b/models/evm/silver/tests/traces/test_silver_evm__traces_recent.yml @@ -0,0 +1,52 @@ +version: 2 +models: + - name: test_silver_evm__traces_recent + description: "This is a view used to test the last three days of traces." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TRACES_ID + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_POSITION + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: TRACE_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/silver/tests/transactions/test_silver_evm__transactions_full.sql b/models/evm/silver/tests/transactions/test_silver_evm__transactions_full.sql new file mode 100644 index 0000000..4b1a1c8 --- /dev/null +++ b/models/evm/silver/tests/transactions/test_silver_evm__transactions_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__transactions') }} diff --git a/models/evm/silver/tests/transactions/test_silver_evm__transactions_full.yml b/models/evm/silver/tests/transactions/test_silver_evm__transactions_full.yml new file mode 100644 index 0000000..64153cc --- /dev/null +++ b/models/evm/silver/tests/transactions/test_silver_evm__transactions_full.yml @@ -0,0 +1,53 @@ +version: 2 +models: + - name: test_silver_evm__transactions_full + description: "This is a view used to test all of the silver transactions model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TRANSACTIONS_ID + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: TX_POSITION + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_POSITION + tests: + - not_null + - name: TRANSACTION_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/silver/tests/transactions/test_silver_evm__transactions_recent.sql b/models/evm/silver/tests/transactions/test_silver_evm__transactions_recent.sql new file mode 100644 index 0000000..d114ca5 --- /dev/null +++ b/models/evm/silver/tests/transactions/test_silver_evm__transactions_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__transactions') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_evm_block_lookback') }} + ) diff --git a/models/evm/silver/tests/transactions/test_silver_evm__transactions_recent.yml b/models/evm/silver/tests/transactions/test_silver_evm__transactions_recent.yml new file mode 100644 index 0000000..42a6bd3 --- /dev/null +++ b/models/evm/silver/tests/transactions/test_silver_evm__transactions_recent.yml @@ -0,0 +1,53 @@ +version: 2 +models: + - name: test_silver_evm__transactions_recent + description: "This is a view used to test the last three days of transactions." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TRANSACTIONS_ID + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: TX_POSITION + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_POSITION + tests: + - not_null + - name: TRANSACTION_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/streamline/_evm_block_lookback.sql b/models/evm/streamline/core/_evm_block_lookback.sql similarity index 86% rename from models/evm/streamline/_evm_block_lookback.sql rename to models/evm/streamline/core/_evm_block_lookback.sql index 1ee4149..36fa0d0 100644 --- a/models/evm/streamline/_evm_block_lookback.sql +++ b/models/evm/streamline/core/_evm_block_lookback.sql @@ -5,7 +5,7 @@ SELECT MIN(block_number) AS block_number FROM - {{ ref("silver_evm__blocks") }} + {{ ref("core_evm__fact_blocks") }} WHERE block_timestamp >= DATEADD('hour', -72, TRUNCATE(SYSDATE(), 'HOUR')) AND block_timestamp < DATEADD('hour', -71, TRUNCATE(SYSDATE(), 'HOUR')) diff --git a/models/evm/streamline/complete/streamline__complete_get_evm_blocks.sql b/models/evm/streamline/core/complete/streamline__complete_get_evm_blocks.sql similarity index 100% rename from models/evm/streamline/complete/streamline__complete_get_evm_blocks.sql rename to models/evm/streamline/core/complete/streamline__complete_get_evm_blocks.sql diff --git a/models/evm/streamline/complete/streamline__complete_get_evm_receipts.sql b/models/evm/streamline/core/complete/streamline__complete_get_evm_receipts.sql similarity index 100% rename from models/evm/streamline/complete/streamline__complete_get_evm_receipts.sql rename to models/evm/streamline/core/complete/streamline__complete_get_evm_receipts.sql diff --git a/models/evm/streamline/complete/streamline__complete_get_evm_traces.sql b/models/evm/streamline/core/complete/streamline__complete_get_evm_traces.sql similarity index 100% rename from models/evm/streamline/complete/streamline__complete_get_evm_traces.sql rename to models/evm/streamline/core/complete/streamline__complete_get_evm_traces.sql diff --git a/models/evm/streamline/external/flowscan/bronze_api__FR_contract_abis.sql b/models/evm/streamline/core/flowscan/bronze_api__FR_contract_abis.sql similarity index 100% rename from models/evm/streamline/external/flowscan/bronze_api__FR_contract_abis.sql rename to models/evm/streamline/core/flowscan/bronze_api__FR_contract_abis.sql diff --git a/models/evm/streamline/external/flowscan/bronze_api__contract_abis.sql b/models/evm/streamline/core/flowscan/bronze_api__contract_abis.sql similarity index 100% rename from models/evm/streamline/external/flowscan/bronze_api__contract_abis.sql rename to models/evm/streamline/core/flowscan/bronze_api__contract_abis.sql diff --git a/models/evm/streamline/external/flowscan/silver_api__contract_abis.sql b/models/evm/streamline/core/flowscan/silver_api__contract_abis.sql similarity index 100% rename from models/evm/streamline/external/flowscan/silver_api__contract_abis.sql rename to models/evm/streamline/core/flowscan/silver_api__contract_abis.sql diff --git a/models/evm/streamline/external/flowscan/streamline__complete_contract_abis.sql b/models/evm/streamline/core/flowscan/streamline__complete_contract_abis.sql similarity index 100% rename from models/evm/streamline/external/flowscan/streamline__complete_contract_abis.sql rename to models/evm/streamline/core/flowscan/streamline__complete_contract_abis.sql diff --git a/models/evm/streamline/external/flowscan/streamline__contract_abis_realtime.sql b/models/evm/streamline/core/flowscan/streamline__contract_abis_realtime.sql similarity index 100% rename from models/evm/streamline/external/flowscan/streamline__contract_abis_realtime.sql rename to models/evm/streamline/core/flowscan/streamline__contract_abis_realtime.sql diff --git a/models/evm/streamline/history/streamline__get_evm_traces_history.sql b/models/evm/streamline/core/history/streamline__get_evm_traces_history.sql similarity index 100% rename from models/evm/streamline/history/streamline__get_evm_traces_history.sql rename to models/evm/streamline/core/history/streamline__get_evm_traces_history.sql diff --git a/models/evm/streamline/realtime/streamline__get_evm_blocks_realtime.sql b/models/evm/streamline/core/realtime/streamline__get_evm_blocks_realtime.sql similarity index 100% rename from models/evm/streamline/realtime/streamline__get_evm_blocks_realtime.sql rename to models/evm/streamline/core/realtime/streamline__get_evm_blocks_realtime.sql diff --git a/models/evm/streamline/realtime/streamline__get_evm_receipts_realtime.sql b/models/evm/streamline/core/realtime/streamline__get_evm_receipts_realtime.sql similarity index 100% rename from models/evm/streamline/realtime/streamline__get_evm_receipts_realtime.sql rename to models/evm/streamline/core/realtime/streamline__get_evm_receipts_realtime.sql diff --git a/models/evm/streamline/realtime/streamline__get_evm_traces_realtime.sql b/models/evm/streamline/core/realtime/streamline__get_evm_traces_realtime.sql similarity index 100% rename from models/evm/streamline/realtime/streamline__get_evm_traces_realtime.sql rename to models/evm/streamline/core/realtime/streamline__get_evm_traces_realtime.sql diff --git a/models/evm/streamline/retry/_missing_receipts.sql b/models/evm/streamline/core/retry/_missing_receipts.sql similarity index 100% rename from models/evm/streamline/retry/_missing_receipts.sql rename to models/evm/streamline/core/retry/_missing_receipts.sql diff --git a/models/evm/streamline/streamline__evm_blocks.sql b/models/evm/streamline/core/streamline__evm_blocks.sql similarity index 100% rename from models/evm/streamline/streamline__evm_blocks.sql rename to models/evm/streamline/core/streamline__evm_blocks.sql diff --git a/models/evm/streamline/decoder/streamline__decoded_logs_complete.sql b/models/evm/streamline/decoder/streamline__decoded_logs_complete.sql new file mode 100644 index 0000000..02e5267 --- /dev/null +++ b/models/evm/streamline/decoder/streamline__decoded_logs_complete.sql @@ -0,0 +1,35 @@ +-- depends_on: {{ ref('bronze__decoded_logs') }} +{{ config ( + materialized = "incremental", + unique_key = "_log_id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["_log_id"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(_log_id)", + tags = ['streamline_decoded_logs_complete'] +) }} + +{# Main query starts here #} +SELECT + block_number, + file_name, + id AS _log_id, + {{ dbt_utils.generate_surrogate_key(['id']) }} AS complete_decoded_logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {% if is_incremental() %} + {{ ref('bronze__decoded_logs') }} + WHERE + _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1970-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM + {{ this }} + ) + {% else %} + {{ ref('bronze__decoded_logs_fr') }} + {% endif %} + +QUALIFY (ROW_NUMBER() OVER (PARTITION BY id ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/evm/streamline/decoder/streamline__decoded_logs_realtime.sql b/models/evm/streamline/decoder/streamline__decoded_logs_realtime.sql new file mode 100644 index 0000000..4ffc27f --- /dev/null +++ b/models/evm/streamline/decoder/streamline__decoded_logs_realtime.sql @@ -0,0 +1,88 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_decode_logs_v2', + target = "{{this.schema}}.{{this.identifier}}", + params = { + "external_table": "evm_decoded_logs", + "sql_limit": 10000000, + "producer_batch_size": 400000, + "worker_batch_size": 200000, + "sql_source": "decoded_logs_realtime" + } + ), + tags = ['streamline_decoded_logs_realtime'] +) }} + +WITH target_blocks AS ( + SELECT + block_number + FROM + {{ ref('core_evm__fact_blocks') }} + WHERE + block_number >= ( + SELECT + block_number + FROM + {{ ref('_evm_block_lookback') }} + ) +), +existing_logs_to_exclude AS ( + SELECT + _log_id + FROM + {{ ref('streamline__decoded_logs_complete') }} + l + INNER JOIN target_blocks b USING (block_number) + WHERE + l.inserted_timestamp :: DATE >= DATEADD('day', -2, SYSDATE()) +), +candidate_logs AS ( + SELECT + l.block_number, + l.tx_hash, + l.event_index, + l.contract_address, + l.topics, + l.data, + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id + FROM + target_blocks b + INNER JOIN {{ ref('core_evm__fact_event_logs') }} + l USING (block_number) + WHERE + l.tx_succeeded + AND l.inserted_timestamp :: DATE >= DATEADD('day', -2, SYSDATE()) +) +SELECT + l.block_number, + l._log_id, + A.abi, + OBJECT_CONSTRUCT( + 'topics', + l.topics, + 'data', + l.data, + 'address', + l.contract_address + ) AS DATA +FROM + candidate_logs l + INNER JOIN {{ ref('silver_evm__complete_event_abis') }} A + ON A.parent_contract_address = l.contract_address + AND A.event_signature = l.topics [0] :: STRING + AND l.block_number BETWEEN A.start_block + AND A.end_block +WHERE + NOT EXISTS ( + SELECT + 1 + FROM + existing_logs_to_exclude e + WHERE + e._log_id = l._log_id + ) \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index c48aecf..85995ee 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -131,6 +131,7 @@ sources: - name: minting_assets - name: contract_abis - name: evm_traces_v2 + - name: evm_decoded_logs - name: crosschain_silver database: crosschain @@ -167,3 +168,9 @@ sources: - name: ufc_strike_metadata - name: espn_nfl_teams - name: espn_nfl_athletes + + - name: crosschain_public + database: crosschain + schema: bronze_public + tables: + - name: user_abis