From 7d7873eac0a2052a01b565259ae5e759d51bc8c0 Mon Sep 17 00:00:00 2001 From: Sam <110511194+sam-xyz@users.noreply.github.com> Date: Thu, 30 Jan 2025 01:28:56 +0800 Subject: [PATCH] An 5714/traces2 hotswap polygon (#414) * add * add * add * fix curve * add tests * overflow * custom tests --------- Co-authored-by: Austin --- .../workflows/dbt_run_overflowed_traces2.yml | 46 -- .github/workflows/dbt_run_traces_fix.yml | 50 -- .../bronze__potential_overflowed_traces.sql | 60 ++- .../v2/bronze__overflowed_traces2.sql | 80 --- .../bronze__potential_overflowed_traces2.sql | 77 --- models/gold/core/core__fact_traces.sql | 500 +++++++++++++++++- models/gold/core/core__fact_traces.yml | 60 ++- .../traces/test_gold__fact_traces_full.sql | 9 + .../traces/test_gold__fact_traces_full.yml | 120 +++++ .../traces/test_gold__fact_traces_recent.sql | 16 + .../traces/test_gold__fact_traces_recent.yml | 120 +++++ .../NFT/sales/silver__element_sales.sql | 8 +- .../NFT/sales/silver__tofunft_sales.sql | 6 +- ...ver_observability__traces_completeness.sql | 5 +- models/silver/abis/silver__proxies.sql | 11 +- .../abis/silver__user_verified_abis.sql | 2 +- .../silver__overflowed_traces.sql | 162 ++---- .../silver__run_overflowed_traces.sql | 18 +- .../v2/silver__overflowed_traces2.sql | 115 ---- .../silver/core/silver__created_contracts.sql | 4 +- models/silver/core/silver__fact_traces2.sql | 491 ----------------- .../silver/core/silver__native_transfers.sql | 21 +- .../core/silver__relevant_contracts.sql | 6 +- models/silver/core/silver__traces.sql | 484 ++--------------- models/silver/core/silver__traces2.sql | 15 - .../tests/traces/test_silver__traces_full.yml | 74 ++- .../traces/test_silver__traces_recent.yml | 56 +- .../traces2_fix/silver__fact_traces2_fix.sql | 13 +- .../silver_bridge__stargate_createpool.sql | 15 +- .../dex/curve/silver_dex__curve_pools.sql | 17 +- .../hashflow/silver_dex__hashflow_pools.sql | 19 +- models/sources.yml | 8 +- .../streamline__matic_balances_history.sql | 4 +- .../streamline__matic_balances_realtime.sql | 4 +- tests/polygon/test_silver__missing_traces.sql | 2 +- .../test_silver__recent_missing_traces.sql | 2 +- 36 files changed, 1076 insertions(+), 1624 deletions(-) delete mode 100644 .github/workflows/dbt_run_overflowed_traces2.yml delete mode 100644 .github/workflows/dbt_run_traces_fix.yml delete mode 100644 models/bronze/overflow/v2/bronze__overflowed_traces2.sql delete mode 100644 models/bronze/overflow/v2/bronze__potential_overflowed_traces2.sql create mode 100644 models/gold/core/tests/traces/test_gold__fact_traces_full.sql create mode 100644 models/gold/core/tests/traces/test_gold__fact_traces_full.yml create mode 100644 models/gold/core/tests/traces/test_gold__fact_traces_recent.sql create mode 100644 models/gold/core/tests/traces/test_gold__fact_traces_recent.yml delete mode 100644 models/silver/core/overflow/v2/silver__overflowed_traces2.sql delete mode 100644 models/silver/core/silver__fact_traces2.sql delete mode 100644 models/silver/core/silver__traces2.sql diff --git a/.github/workflows/dbt_run_overflowed_traces2.yml b/.github/workflows/dbt_run_overflowed_traces2.yml deleted file mode 100644 index 48c1d52..0000000 --- a/.github/workflows/dbt_run_overflowed_traces2.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: dbt_run_overflowed_traces2 -run-name: dbt_run_overflowed_traces2 - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - - - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -m "polygon_models,tag:overflowed_traces2" --vars '{"OVERFLOWED_TRACES":True}' \ No newline at end of file diff --git a/.github/workflows/dbt_run_traces_fix.yml b/.github/workflows/dbt_run_traces_fix.yml deleted file mode 100644 index 49b32ed..0000000 --- a/.github/workflows/dbt_run_traces_fix.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: dbt_run_traces_fix -run-name: dbt_run_traces_fix - -on: - workflow_dispatch: - inputs: - use_backfill_env: - description: "Use the backfill environment" - type: boolean - schedule: - # every 15 minutes (see https://crontab.guru) - - cron: "*/15 * * * *" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: ${{ github.event_name == 'workflow_dispatch' && inputs.use_backfill_env && 'workflow_prod_backfill' || 'workflow_prod_2xl' }} - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: run traces fix model - run: | - dbt run -m "polygon_models,tag:traces_fix" diff --git a/models/bronze/overflow/bronze__potential_overflowed_traces.sql b/models/bronze/overflow/bronze__potential_overflowed_traces.sql index b19f804..338c9e2 100644 --- a/models/bronze/overflow/bronze__potential_overflowed_traces.sql +++ b/models/bronze/overflow/bronze__potential_overflowed_traces.sql @@ -6,50 +6,62 @@ WITH impacted_blocks AS ( SELECT - VALUE :: INT AS block_number + blocks_impacted_array FROM - ( - SELECT - blocks_impacted_array - FROM - {{ ref("silver_observability__traces_completeness") }} - ORDER BY - test_timestamp DESC - LIMIT - 1 - ), LATERAL FLATTEN ( + {{ ref("silver_observability__traces_completeness") }} + ORDER BY + test_timestamp DESC + LIMIT + 1 +), all_missing AS ( + SELECT + DISTINCT VALUE :: INT AS block_number + FROM + impacted_blocks, + LATERAL FLATTEN ( input => blocks_impacted_array ) ), all_txs AS ( SELECT - t.block_number, - t.position, - t.tx_hash + block_number, + POSITION AS tx_position, + tx_hash FROM {{ ref("silver__transactions") }} - t - JOIN impacted_blocks USING (block_number) + JOIN all_missing USING (block_number) ), missing_txs AS ( SELECT - DISTINCT block_number, - POSITION, + DISTINCT txs.block_number, + txs.tx_position, file_name FROM - all_txs - LEFT JOIN {{ ref("silver__traces") }} - tr USING ( + all_txs txs + LEFT JOIN {{ source( + "polygon_gold", + "fact_traces" + ) }} + tr2 USING ( block_number, - tx_hash + tx_position ) JOIN {{ ref("streamline__traces_complete") }} USING (block_number) + LEFT JOIN {{ source( + 'polygon_silver', + 'overflowed_traces' + ) }} + ot USING ( + block_number, + tx_position + ) WHERE - tr.tx_hash IS NULL + tr2.block_number IS NULL + AND ot.block_number IS NULL ) SELECT block_number, - POSITION, + tx_position AS POSITION, file_name, build_scoped_file_url( @streamline.bronze.external_tables, diff --git a/models/bronze/overflow/v2/bronze__overflowed_traces2.sql b/models/bronze/overflow/v2/bronze__overflowed_traces2.sql deleted file mode 100644 index 2be73b9..0000000 --- a/models/bronze/overflow/v2/bronze__overflowed_traces2.sql +++ /dev/null @@ -1,80 +0,0 @@ -{{ config ( - materialized = "view", - tags = ['overflowed_traces2'] -) }} - -{% for item in range( - 1, - 11 - ) %} - - SELECT - o.file_name, - f.block_number, - f.index_vals, - f.path, - f.key, - f.value_ - FROM - ( - SELECT - file_name, - file_url, - index_cols, - [overflowed_block, overflowed_tx] AS index_vals - FROM - ( - SELECT - block_number, - POSITION, - file_name, - file_url, - index_cols, - VALUE [0] AS overflowed_block, - VALUE [1] AS overflowed_tx, - block_number = overflowed_block - AND POSITION = overflowed_tx AS missing - FROM - ( - SELECT - block_number, - POSITION, - file_name, - file_url, - index_cols, - utils.udf_detect_overflowed_responses( - file_url, - index_cols - ) AS index_vals - FROM - {{ ref("bronze__potential_overflowed_traces2") }} - WHERE - row_no = {{ item }} - ), - LATERAL FLATTEN ( - input => index_vals - ) - ) - WHERE - missing = TRUE - ) o, - TABLE( - utils.udtf_flatten_overflowed_responses( - o.file_url, - o.index_cols, - [o.index_vals] - ) - ) f - WHERE - NOT IS_OBJECT( - f.value_ - ) - AND NOT IS_ARRAY( - f.value_ - ) - AND NOT IS_NULL_VALUE( - f.value_ - ) {% if not loop.last %} - UNION ALL - {% endif %} - {% endfor %} diff --git a/models/bronze/overflow/v2/bronze__potential_overflowed_traces2.sql b/models/bronze/overflow/v2/bronze__potential_overflowed_traces2.sql deleted file mode 100644 index f01ff6d..0000000 --- a/models/bronze/overflow/v2/bronze__potential_overflowed_traces2.sql +++ /dev/null @@ -1,77 +0,0 @@ -{{ config ( - materialized = "view", - tags = ['overflowed_traces2'] -) }} - -WITH impacted_blocks AS ( - - SELECT - blocks_impacted_array - FROM - {{ ref("silver_observability__traces_completeness") }} - ORDER BY - test_timestamp DESC - LIMIT - 1 -), all_missing AS ( - SELECT - DISTINCT VALUE :: INT AS block_number - FROM - impacted_blocks, - LATERAL FLATTEN ( - input => blocks_impacted_array - ) -), -all_txs AS ( - SELECT - block_number, - POSITION AS tx_position, - tx_hash - FROM - {{ ref("silver__transactions") }} - JOIN all_missing USING (block_number) -), -missing_txs AS ( - SELECT - DISTINCT txs.block_number, - txs.tx_position, - file_name - FROM - all_txs txs - LEFT JOIN {{ ref("silver__traces2") }} - tr2 USING ( - block_number, - tx_position - ) - JOIN {{ ref("streamline__traces_complete") }} USING (block_number) - LEFT JOIN {{ source( - 'polygon_silver', - 'overflowed_traces2' - ) }} - ot USING ( - block_number, - tx_position - ) - WHERE - tr2.block_number IS NULL - AND ot.block_number IS NULL -) -SELECT - block_number, - tx_position AS POSITION, - file_name, - build_scoped_file_url( - @streamline.bronze.external_tables, - file_name - ) AS file_url, - ['block_number', 'array_index'] AS index_cols, - ROW_NUMBER() over ( - ORDER BY - block_number ASC, - POSITION ASC - ) AS row_no -FROM - missing_txs -ORDER BY - block_number ASC, - POSITION ASC diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql index 7c9c5a7..3353a65 100644 --- a/models/gold/core/core__fact_traces.sql +++ b/models/gold/core/core__fact_traces.sql @@ -1,43 +1,491 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, - "columns": true } +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + incremental_predicates = [fsc_evm.standard_predicate()], + cluster_by = "block_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['non_realtime'], + full_refresh = false ) }} +{# {{ fsc_evm.gold_traces_v1( +full_reload_start_block = 5000000, +full_reload_blocks = 1000000, +uses_overflow_steps = TRUE +) }} +#} +WITH silver_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + parent_trace_address, + trace_address_array, + trace_json, + traces_id, + 'regular' AS source + FROM + {{ ref('silver__traces') }} + WHERE + 1 = 1 + +{% if is_incremental() and not full_reload_mode %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) {% elif is_incremental() and full_reload_mode %} +AND block_number BETWEEN ( + SELECT + MAX( + block_number + ) + FROM + {{ this }} +) +AND ( + SELECT + MAX( + block_number + ) + 1000000 + FROM + {{ this }} +) +{% else %} + AND block_number <= 5000000 +{% endif %} +), +sub_traces AS ( + SELECT + block_number, + tx_position, + parent_trace_address, + COUNT(*) AS sub_traces + FROM + silver_traces + GROUP BY + block_number, + tx_position, + parent_trace_address +), +trace_index_array AS ( + SELECT + block_number, + tx_position, + trace_address, + ARRAY_AGG(flat_value) AS number_array + FROM + ( + SELECT + block_number, + tx_position, + trace_address, + IFF( + VALUE :: STRING = 'ORIGIN', + -1, + VALUE :: INT + ) AS flat_value + FROM + silver_traces, + LATERAL FLATTEN ( + input => trace_address_array + ) + ) + GROUP BY + block_number, + tx_position, + trace_address +), +trace_index_sub_traces AS ( + SELECT + b.block_number, + b.tx_position, + b.trace_address, + IFNULL( + sub_traces, + 0 + ) AS sub_traces, + number_array, + ROW_NUMBER() over ( + PARTITION BY b.block_number, + b.tx_position + ORDER BY + number_array ASC + ) - 1 AS trace_index, + b.trace_json, + b.traces_id, + b.source + FROM + silver_traces b + LEFT JOIN sub_traces s + ON b.block_number = s.block_number + AND b.tx_position = s.tx_position + AND b.trace_address = s.parent_trace_address + JOIN trace_index_array n + ON b.block_number = n.block_number + AND b.tx_position = n.tx_position + AND b.trace_address = n.trace_address +), +errored_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + trace_json + FROM + trace_index_sub_traces + WHERE + trace_json :error :: STRING IS NOT NULL +), +error_logic AS ( + SELECT + b0.block_number, + b0.tx_position, + b0.trace_address, + b0.trace_json :error :: STRING AS error, + b1.trace_json :error :: STRING AS any_error, + b2.trace_json :error :: STRING AS origin_error + FROM + trace_index_sub_traces b0 + LEFT JOIN errored_traces b1 + ON b0.block_number = b1.block_number + AND b0.tx_position = b1.tx_position + AND b0.trace_address RLIKE CONCAT( + '^', + b1.trace_address, + '(_[0-9]+)*$' + ) + LEFT JOIN errored_traces b2 + ON b0.block_number = b2.block_number + AND b0.tx_position = b2.tx_position + AND b2.trace_address = 'ORIGIN' +), +aggregated_errors AS ( + SELECT + block_number, + tx_position, + trace_address, + error, + IFF(MAX(any_error) IS NULL + AND error IS NULL + AND origin_error IS NULL, TRUE, FALSE) AS trace_succeeded + FROM + error_logic + GROUP BY + block_number, + tx_position, + trace_address, + error, + origin_error), + json_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + sub_traces, + number_array, + trace_index, + trace_json AS DATA, + trace_succeeded, + trace_json :error :: STRING AS error_reason, + trace_json :revertReason :: STRING AS revert_reason, + trace_json :from :: STRING AS from_address, + trace_json :to :: STRING AS to_address, + IFNULL( + trace_json :value :: STRING, + '0x0' + ) AS value_hex, + IFNULL( + utils.udf_hex_to_int( + trace_json :value :: STRING + ), + '0' + ) AS value_precise_raw, + utils.udf_decimal_adjust( + value_precise_raw, + 18 + ) AS value_precise, + value_precise :: FLOAT AS VALUE, + utils.udf_hex_to_int( + trace_json :gas :: STRING + ) :: INT AS gas, + utils.udf_hex_to_int( + trace_json :gasUsed :: STRING + ) :: INT AS gas_used, + trace_json :input :: STRING AS input, + trace_json :output :: STRING AS output, + trace_json :type :: STRING AS TYPE, + concat_ws( + '_', + TYPE, + trace_address + ) AS identifier, + IFF( + trace_succeeded, + 'SUCCESS', + 'FAIL' + ) AS trace_status, + traces_id + FROM + trace_index_sub_traces + JOIN aggregated_errors USING ( + block_number, + tx_position, + trace_address + ) + ), + incremental_traces AS ( + SELECT + f.block_number, + t.tx_hash, + t.block_timestamp, + t.origin_function_signature, + t.from_address AS origin_from_address, + t.to_address AS origin_to_address, + t.tx_status, + f.tx_position, + f.trace_index, + f.from_address AS from_address, + f.to_address AS to_address, + f.value_hex, + f.value_precise_raw, + f.value_precise, + f.value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.identifier, + f.sub_traces, + f.error_reason, + f.revert_reason, + f.trace_status, + f.data, + f.traces_id, + f.trace_succeeded, + f.trace_address, + IFF( + t.tx_status = 'SUCCESS', + TRUE, + FALSE + ) AS tx_succeeded + FROM + json_traces f + LEFT OUTER JOIN {{ ref('silver__transactions') }} + t + ON f.tx_position = t.position + AND f.block_number = t.block_number + +{% if is_incremental() and not full_reload_mode %} +AND t.modified_timestamp >= ( + SELECT + DATEADD('hour', -24, MAX(modified_timestamp)) + FROM + {{ this }}) + {% endif %} +) + +{% if is_incremental() %}, +overflow_blocks AS ( + SELECT + DISTINCT block_number + FROM + silver_traces + WHERE + source = 'overflow' +), +heal_missing_data AS ( + SELECT + t.block_number, + txs.tx_hash, + txs.block_timestamp, + txs.origin_function_signature, + txs.from_address AS origin_from_address, + txs.to_address AS origin_to_address, + txs.tx_status, + t.tx_position, + t.trace_index, + t.from_address, + t.to_address, + t.value_hex, + t.value_precise_raw, + t.value_precise, + t.value, + t.gas, + t.gas_used, + t.input, + t.output, + t.type, + t.identifier, + t.sub_traces, + t.error_reason, + t.revert_reason, + t.trace_status, + t.data, + t.fact_traces_id AS traces_id, + t.trace_succeeded, + t.trace_address, + IFF( + txs.tx_status = 'SUCCESS', + TRUE, + FALSE + ) AS tx_succeeded + FROM + {{ this }} + t + JOIN {{ ref('silver__transactions') }} + txs + ON t.tx_position = txs.position + AND t.block_number = txs.block_number + WHERE + t.tx_hash IS NULL + OR t.block_timestamp IS NULL + OR t.tx_status IS NULL +) +{% endif %}, +all_traces AS ( + SELECT + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + VALUE, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + revert_reason, + trace_status, + DATA, + trace_succeeded, + trace_address, + tx_succeeded + FROM + incremental_traces + +{% if is_incremental() %} +UNION ALL SELECT - tx_hash, block_number, + tx_hash, block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_status, + tx_position, + trace_index, from_address, to_address, - matic_value AS VALUE, - matic_value_precise_raw AS value_precise_raw, - matic_value_precise AS value_precise, + value_hex, + value_precise_raw, + value_precise, + VALUE, gas, gas_used, input, output, TYPE, identifier, + sub_traces, + error_reason, + revert_reason, + trace_status, + DATA, + trace_succeeded, + trace_address, + tx_succeeded +FROM + heal_missing_data +UNION ALL +SELECT + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + VALUE, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + revert_reason, + trace_status, + DATA, + trace_succeeded, + trace_address, + tx_succeeded +FROM + {{ this }} + JOIN overflow_blocks USING (block_number) +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + tx_position, + trace_index, + from_address, + to_address, + input, + output, + TYPE, + trace_address, + sub_traces, + VALUE, + value_precise_raw, + value_precise, + value_hex, + gas, + gas_used, + origin_from_address, + origin_to_address, + origin_function_signature, + trace_succeeded, + error_reason, + revert_reason, + tx_succeeded, + identifier, DATA, tx_status, - sub_traces, trace_status, - error_reason, - trace_index, - COALESCE ( - traces_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'trace_index'] - ) }} - ) AS fact_traces_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} AS fact_traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp FROM - {{ ref('silver__traces') }} + all_traces qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index +ORDER BY + modified_timestamp DESC, block_timestamp DESC nulls last)) = 1 diff --git a/models/gold/core/core__fact_traces.yml b/models/gold/core/core__fact_traces.yml index 83a65cb..c0e0a00 100644 --- a/models/gold/core/core__fact_traces.yml +++ b/models/gold/core/core__fact_traces.yml @@ -1,19 +1,37 @@ version: 2 models: - name: core__fact_traces - description: '{{ doc("poly_traces_table_doc") }}' + description: '{{ doc("poly_traces_table_doc") }}' columns: - name: BLOCK_NUMBER - description: '{{ doc("poly_traces_block_no") }}' + description: '{{ doc("poly_traces_block_no") }}' - name: BLOCK_TIMESTAMP description: '{{ doc("poly_traces_blocktime") }}' - name: TX_HASH - description: '{{ doc("poly_traces_tx_hash") }}' + description: '{{ doc("poly_traces_tx_hash") }}' + - name: TX_POSITION + description: The position of the transaction within the block. + - name: TRACE_INDEX + description: The index of the trace within the transaction. - name: FROM_ADDRESS description: '{{ doc("poly_traces_from") }}' - name: TO_ADDRESS description: '{{ doc("poly_traces_to") }}' + - name: INPUT + description: '{{ doc("poly_traces_input") }}' + - name: OUTPUT + description: '{{ doc("poly_traces_output") }}' + - name: TYPE + description: '{{ doc("poly_traces_type") }}' + - name: TRACE_ADDRESS + description: This field represents the position of the trace within the transaction. + - name: SUB_TRACES + description: '{{ doc("poly_traces_sub") }}' + - name: IDENTIFIER + description: '{{ doc("poly_traces_identifier") }}' + - name: DATA + description: '{{ doc("poly_traces_call_data") }}' - name: VALUE description: '{{ doc("poly_traces_value") }}' - name: VALUE_PRECISE_RAW @@ -24,29 +42,27 @@ models: description: '{{ doc("poly_traces_gas") }}' - name: GAS_USED description: '{{ doc("poly_traces_gas_used") }}' - - name: INPUT - description: '{{ doc("poly_traces_input") }}' - - name: OUTPUT - description: '{{ doc("poly_traces_output") }}' - - name: TYPE - description: '{{ doc("poly_traces_type") }}' - - name: IDENTIFIER - description: '{{ doc("poly_traces_identifier") }}' - - name: DATA - description: '{{ doc("poly_traces_call_data") }}' - - name: TX_STATUS - description: '{{ doc("poly_tx_status") }}' - - name: SUB_TRACES - description: '{{ doc("poly_traces_sub") }}' + - name: ORIGIN_FROM_ADDRESS + description: The from address at the transaction level. + - name: ORIGIN_TO_ADDRESS + description: The to address at the transaction level. + - name: ORIGIN_FUNCTION_SIGNATURE + description: The function signature at the transaction level. - name: TRACE_STATUS description: The status of the trace, either `SUCCESS` or `FAIL` + - name: TRACE_SUCCEEDED + description: Whether the trace succeeded or failed - name: ERROR_REASON description: The reason for the trace failure, if any. - - name: TRACE_INDEX - description: The index of the trace within the transaction. + - name: REVERT_REASON + description: The reason for the trace revert, if any. + - name: TX_STATUS + description: '{{ doc("poly_tx_status") }}' + - name: TX_SUCCEEDED + description: Whether the transaction succeeded or failed - name: FACT_TRACES_ID - description: '{{ doc("pk") }}' + description: '{{ doc("pk") }}' - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' + description: '{{ doc("inserted_timestamp") }}' - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' \ No newline at end of file + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/tests/traces/test_gold__fact_traces_full.sql b/models/gold/core/tests/traces/test_gold__fact_traces_full.sql new file mode 100644 index 0000000..ced1fba --- /dev/null +++ b/models/gold/core/tests/traces/test_gold__fact_traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('core__fact_traces') }} diff --git a/models/gold/core/tests/traces/test_gold__fact_traces_full.yml b/models/gold/core/tests/traces/test_gold__fact_traces_full.yml new file mode 100644 index 0000000..6677117 --- /dev/null +++ b/models/gold/core/tests/traces/test_gold__fact_traces_full.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold__fact_traces_full + description: "This is a view used to test all of the gold fact traces model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - fsc_utils.sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 AND TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 diff --git a/models/gold/core/tests/traces/test_gold__fact_traces_recent.sql b/models/gold/core/tests/traces/test_gold__fact_traces_recent.sql new file mode 100644 index 0000000..4bed00f --- /dev/null +++ b/models/gold/core/tests/traces/test_gold__fact_traces_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_test'] +) }} + +SELECT + * +FROM + {{ ref('core__fact_traces') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_block_lookback') }} + ) diff --git a/models/gold/core/tests/traces/test_gold__fact_traces_recent.yml b/models/gold/core/tests/traces/test_gold__fact_traces_recent.yml new file mode 100644 index 0000000..4206f50 --- /dev/null +++ b/models/gold/core/tests/traces/test_gold__fact_traces_recent.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold__fact_traces_recent + description: "This is a view used to test the last three days of fact traces." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - fsc_utils.sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 diff --git a/models/silver/NFT/sales/silver__element_sales.sql b/models/silver/NFT/sales/silver__element_sales.sql index 1c4b86a..867574a 100644 --- a/models/silver/NFT/sales/silver__element_sales.sql +++ b/models/silver/NFT/sales/silver__element_sales.sql @@ -184,7 +184,7 @@ old_native_transfers AS ( trace_index, from_address, to_address, - matic_value, + VALUE AS matic_value, matic_value * pow( 10, 18 @@ -200,7 +200,7 @@ old_native_transfers AS ( 0 ) AS intra_grouping FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE block_timestamp :: DATE >= ( SELECT @@ -250,13 +250,13 @@ old_native_transfers AS ( AND trace_status = 'SUCCESS' {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} ) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' +AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' {% endif %} ), old_native_labels AS ( diff --git a/models/silver/NFT/sales/silver__tofunft_sales.sql b/models/silver/NFT/sales/silver__tofunft_sales.sql index e8906e7..0502b6c 100644 --- a/models/silver/NFT/sales/silver__tofunft_sales.sql +++ b/models/silver/NFT/sales/silver__tofunft_sales.sql @@ -148,7 +148,7 @@ traces_raw AS ( ) ) :: INT AS bundle_array_size FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE block_timestamp :: DATE >= '2021-10-01' AND to_address = '0x7bc8b1b5aba4df3be9f9a32dae501214dc0e4f3f' @@ -159,13 +159,13 @@ traces_raw AS ( AND trace_status = 'SUCCESS' {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} ) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' +AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' {% endif %} ), traces_raw_rn AS ( diff --git a/models/silver/_observability/silver_observability__traces_completeness.sql b/models/silver/_observability/silver_observability__traces_completeness.sql index d3d1b24..4681f1e 100644 --- a/models/silver/_observability/silver_observability__traces_completeness.sql +++ b/models/silver/_observability/silver_observability__traces_completeness.sql @@ -81,7 +81,10 @@ broken_blocks AS ( FROM {{ ref("silver__transactions") }} tx - LEFT JOIN {{ ref("silver__traces") }} + LEFT JOIN {{ source( + "polygon_gold", + "fact_traces" + ) }} tr USING ( block_number, tx_hash diff --git a/models/silver/abis/silver__proxies.sql b/models/silver/abis/silver__proxies.sql index bb35fa3..3cceeaf 100644 --- a/models/silver/abis/silver__proxies.sql +++ b/models/silver/abis/silver__proxies.sql @@ -11,9 +11,9 @@ WITH base AS ( from_address, to_address, MIN(block_number) AS start_block, - MAX(_inserted_timestamp) AS _inserted_timestamp + MAX(modified_timestamp) AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE TYPE = 'DELEGATECALL' AND trace_status = 'SUCCESS' @@ -21,7 +21,7 @@ WITH base AS ( AND from_address != to_address -- exclude self-calls {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT MAX(_inserted_timestamp) - INTERVAL '24 hours' FROM @@ -99,5 +99,6 @@ FROM FINAL f JOIN {{ ref('silver__created_contracts') }} C ON f.contract_address = C.created_contract_address - JOIN {{ ref('silver__created_contracts') }} p - ON f.proxy_address = p.created_contract_address \ No newline at end of file + JOIN {{ ref('silver__created_contracts') }} + p + ON f.proxy_address = p.created_contract_address diff --git a/models/silver/abis/silver__user_verified_abis.sql b/models/silver/abis/silver__user_verified_abis.sql index e2f07c5..08a6fc1 100644 --- a/models/silver/abis/silver__user_verified_abis.sql +++ b/models/silver/abis/silver__user_verified_abis.sql @@ -473,7 +473,7 @@ valid_traces AS ( SELECT base_address FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} JOIN function_mapping ON function_sig = LEFT( input, diff --git a/models/silver/core/overflow/formatted_views/silver__overflowed_traces.sql b/models/silver/core/overflow/formatted_views/silver__overflowed_traces.sql index 7345141..e598cbd 100644 --- a/models/silver/core/overflow/formatted_views/silver__overflowed_traces.sql +++ b/models/silver/core/overflow/formatted_views/silver__overflowed_traces.sql @@ -1,12 +1,24 @@ +-- depends_on: {{ ref('bronze__overflowed_traces') }} +{% set warehouse = 'DBT_SNOWPARK' if var('OVERFLOWED_TRACES') else target.warehouse %} {{ config ( - materialized = 'view', - tags = ['overflowed_traces'] + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = ['block_number','tx_position'], + cluster_by = ['modified_timestamp::DATE','partition_key'], + tags = ['overflowed_traces'], + full_refresh = false, + snowflake_warehouse = warehouse ) }} +{% if is_incremental() %} WITH bronze_overflowed_traces AS ( SELECT block_number :: INT AS block_number, + ROUND( + block_number, + -3 + ) AS partition_key, index_vals [1] :: INT AS tx_position, IFF( path IN ( @@ -30,9 +42,7 @@ WITH bronze_overflowed_traces AS ( 'input', 'error', 'output', - 'revertReason', - 'txHash', - 'result.txHash' + 'revertReason' ), 'ORIGIN', REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') @@ -58,7 +68,7 @@ WITH bronze_overflowed_traces AS ( SPLIT( trace_address, '_' - ) AS str_array + ) AS trace_address_array FROM {{ ref("bronze__overflowed_traces") }} GROUP BY @@ -66,118 +76,40 @@ WITH bronze_overflowed_traces AS ( tx_position, trace_address, _inserted_timestamp -), -sub_traces AS ( - SELECT - block_number, - tx_position, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - bronze_overflowed_traces - GROUP BY - block_number, - tx_position, - parent_trace_address -), -num_array AS ( - SELECT - block_number, - tx_position, - trace_address, - ARRAY_AGG(flat_value) AS num_array - FROM - ( - SELECT - block_number, - tx_position, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - bronze_overflowed_traces, - LATERAL FLATTEN ( - input => str_array - ) - ) - GROUP BY - block_number, - tx_position, - trace_address -), -cleaned_traces AS ( - SELECT - b.block_number, - b.tx_position, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - num_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, - b.tx_position - ORDER BY - num_array ASC - ) - 1 AS trace_index, - trace_json, - b._inserted_timestamp - FROM - bronze_overflowed_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_position = s.tx_position - AND b.trace_address = s.parent_trace_address - JOIN num_array n - ON b.block_number = n.block_number - AND b.tx_position = n.tx_position - AND b.trace_address = n.trace_address ) SELECT - tx_position, - trace_index, block_number, + tx_position, trace_address, - trace_json :error :: STRING AS error_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS matic_value_precise_raw, - utils.udf_decimal_adjust( - matic_value_precise_raw, - 18 - ) AS matic_value_precise, - matic_value_precise :: FLOAT AS matic_value, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - concat_ws( - '-', - block_number, - tx_position, - identifier - ) AS _call_id, + parent_trace_address, + trace_address_array, + trace_json, + partition_key, _inserted_timestamp, - trace_json AS DATA, - sub_traces + {{ dbt_utils.generate_surrogate_key( + ['block_number', 'tx_position', 'trace_address'] + ) }} AS traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM - cleaned_traces + bronze_overflowed_traces qualify(ROW_NUMBER() over(PARTITION BY traces_id +ORDER BY + _inserted_timestamp DESC)) = 1 +{% else %} +SELECT + NULL :: INT AS block_number, + NULL :: INT tx_position, + NULL :: text AS trace_address, + NULL :: text AS parent_trace_address, + NULL :: ARRAY AS trace_address_array, + NULL :: OBJECT AS trace_json, + NULL :: INT AS partition_key, + NULL :: timestamp_ltz AS _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number', 'tx_position', 'trace_address'] + ) }} AS traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +{% endif %} diff --git a/models/silver/core/overflow/run_models/silver__run_overflowed_traces.sql b/models/silver/core/overflow/run_models/silver__run_overflowed_traces.sql index e90390d..cf45811 100644 --- a/models/silver/core/overflow/run_models/silver__run_overflowed_traces.sql +++ b/models/silver/core/overflow/run_models/silver__run_overflowed_traces.sql @@ -23,13 +23,7 @@ WITH base AS ( 'polygon-models', 'dbt_run_overflowed_traces.yml', NULL - ) AS run_overflow_models, - github_actions.workflow_dispatches( - 'FlipsideCrypto', - 'polygon-models', - 'dbt_run_overflowed_traces2.yml', - NULL - ) AS run_overflow_models2 + ) AS run_overflow_models FROM base WHERE @@ -47,15 +41,7 @@ WITH base AS ( 'status', 'skipped' ) - ) AS run_overflow_models, - COALESCE( - run_overflow_models2, - OBJECT_CONSTRUCT( - 'status', - 'skipped' - ) - ) AS run_overflow_models2, - SYSDATE() AS test_timestamp + ) AS run_overflow_models SYSDATE() AS test_timestamp FROM ( SELECT diff --git a/models/silver/core/overflow/v2/silver__overflowed_traces2.sql b/models/silver/core/overflow/v2/silver__overflowed_traces2.sql deleted file mode 100644 index f612da7..0000000 --- a/models/silver/core/overflow/v2/silver__overflowed_traces2.sql +++ /dev/null @@ -1,115 +0,0 @@ --- depends_on: {{ ref('bronze__overflowed_traces2') }} -{% set warehouse = 'DBT_SNOWPARK' if var('OVERFLOWED_TRACES') else target.warehouse %} -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = ['block_number','tx_position'], - cluster_by = ['modified_timestamp::DATE','partition_key'], - tags = ['overflowed_traces2'], - full_refresh = false, - snowflake_warehouse = warehouse -) }} - -{% if is_incremental() %} -WITH bronze_overflowed_traces AS ( - - SELECT - block_number :: INT AS block_number, - ROUND( - block_number, - -3 - ) AS partition_key, - index_vals [1] :: INT AS tx_position, - IFF( - path IN ( - 'result', - 'result.value', - 'result.type', - 'result.to', - 'result.input', - 'result.gasUsed', - 'result.gas', - 'result.from', - 'result.output', - 'result.error', - 'result.revertReason', - 'gasUsed', - 'gas', - 'type', - 'to', - 'from', - 'value', - 'input', - 'error', - 'output', - 'revertReason' - ), - 'ORIGIN', - REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') - ) AS trace_address, - SYSDATE() :: timestamp_ltz AS _inserted_timestamp, - OBJECT_AGG( - key, - value_ - ) AS trace_json, - CASE - WHEN trace_address = 'ORIGIN' THEN NULL - WHEN POSITION( - '_' IN trace_address - ) = 0 THEN 'ORIGIN' - ELSE REGEXP_REPLACE( - trace_address, - '_[0-9]+$', - '', - 1, - 1 - ) - END AS parent_trace_address, - SPLIT( - trace_address, - '_' - ) AS trace_address_array - FROM - {{ ref("bronze__overflowed_traces2") }} - GROUP BY - block_number, - tx_position, - trace_address, - _inserted_timestamp -) -SELECT - block_number, - tx_position, - trace_address, - parent_trace_address, - trace_address_array, - trace_json, - partition_key, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'tx_position', 'trace_address'] - ) }} AS traces_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - bronze_overflowed_traces qualify(ROW_NUMBER() over(PARTITION BY traces_id -ORDER BY - _inserted_timestamp DESC)) = 1 -{% else %} -SELECT - NULL :: INT AS block_number, - NULL :: INT tx_position, - NULL :: text AS trace_address, - NULL :: text AS parent_trace_address, - NULL :: ARRAY AS trace_address_array, - NULL :: OBJECT AS trace_json, - NULL :: INT AS partition_key, - NULL :: timestamp_ltz AS _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'tx_position', 'trace_address'] - ) }} AS traces_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -{% endif %} diff --git a/models/silver/core/silver__created_contracts.sql b/models/silver/core/silver__created_contracts.sql index c6baa29..acbd37d 100644 --- a/models/silver/core/silver__created_contracts.sql +++ b/models/silver/core/silver__created_contracts.sql @@ -13,7 +13,7 @@ SELECT to_address AS created_contract_address, from_address AS creator_address, input AS created_contract_input, - _inserted_timestamp, + modified_timestamp AS _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['to_address'] ) }} AS created_contracts_id, @@ -21,7 +21,7 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE TYPE ILIKE 'create%' AND to_address IS NOT NULL diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql deleted file mode 100644 index 5d6f2c5..0000000 --- a/models/silver/core/silver__fact_traces2.sql +++ /dev/null @@ -1,491 +0,0 @@ -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = "block_number", - incremental_predicates = [fsc_evm.standard_predicate()], - cluster_by = "block_timestamp::date", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['non_realtime'], - full_refresh = false -) }} -{# {{ fsc_evm.gold_traces_v1( -full_reload_start_block = 5000000, -full_reload_blocks = 1000000, -uses_overflow_steps = TRUE -) }} -#} -WITH silver_traces AS ( - - SELECT - block_number, - tx_position, - trace_address, - parent_trace_address, - trace_address_array, - trace_json, - traces_id, - 'regular' AS source - FROM - {{ ref('silver__traces2') }} - WHERE - 1 = 1 - -{% if is_incremental() and not full_reload_mode %} -AND modified_timestamp > ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} -) {% elif is_incremental() and full_reload_mode %} -AND block_number BETWEEN ( - SELECT - MAX( - block_number - ) - FROM - {{ this }} -) -AND ( - SELECT - MAX( - block_number - ) + 1000000 - FROM - {{ this }} -) -{% else %} - AND block_number <= 5000000 -{% endif %} -), -sub_traces AS ( - SELECT - block_number, - tx_position, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - silver_traces - GROUP BY - block_number, - tx_position, - parent_trace_address -), -trace_index_array AS ( - SELECT - block_number, - tx_position, - trace_address, - ARRAY_AGG(flat_value) AS number_array - FROM - ( - SELECT - block_number, - tx_position, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - silver_traces, - LATERAL FLATTEN ( - input => trace_address_array - ) - ) - GROUP BY - block_number, - tx_position, - trace_address -), -trace_index_sub_traces AS ( - SELECT - b.block_number, - b.tx_position, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - number_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, - b.tx_position - ORDER BY - number_array ASC - ) - 1 AS trace_index, - b.trace_json, - b.traces_id, - b.source - FROM - silver_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_position = s.tx_position - AND b.trace_address = s.parent_trace_address - JOIN trace_index_array n - ON b.block_number = n.block_number - AND b.tx_position = n.tx_position - AND b.trace_address = n.trace_address -), -errored_traces AS ( - SELECT - block_number, - tx_position, - trace_address, - trace_json - FROM - trace_index_sub_traces - WHERE - trace_json :error :: STRING IS NOT NULL -), -error_logic AS ( - SELECT - b0.block_number, - b0.tx_position, - b0.trace_address, - b0.trace_json :error :: STRING AS error, - b1.trace_json :error :: STRING AS any_error, - b2.trace_json :error :: STRING AS origin_error - FROM - trace_index_sub_traces b0 - LEFT JOIN errored_traces b1 - ON b0.block_number = b1.block_number - AND b0.tx_position = b1.tx_position - AND b0.trace_address RLIKE CONCAT( - '^', - b1.trace_address, - '(_[0-9]+)*$' - ) - LEFT JOIN errored_traces b2 - ON b0.block_number = b2.block_number - AND b0.tx_position = b2.tx_position - AND b2.trace_address = 'ORIGIN' -), -aggregated_errors AS ( - SELECT - block_number, - tx_position, - trace_address, - error, - IFF(MAX(any_error) IS NULL - AND error IS NULL - AND origin_error IS NULL, TRUE, FALSE) AS trace_succeeded - FROM - error_logic - GROUP BY - block_number, - tx_position, - trace_address, - error, - origin_error), - json_traces AS ( - SELECT - block_number, - tx_position, - trace_address, - sub_traces, - number_array, - trace_index, - trace_json AS DATA, - trace_succeeded, - trace_json :error :: STRING AS error_reason, - trace_json :revertReason :: STRING AS revert_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - trace_json :value :: STRING, - '0x0' - ) AS value_hex, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS value_precise_raw, - utils.udf_decimal_adjust( - value_precise_raw, - 18 - ) AS value_precise, - value_precise :: FLOAT AS VALUE, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - IFF( - trace_succeeded, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - traces_id - FROM - trace_index_sub_traces - JOIN aggregated_errors USING ( - block_number, - tx_position, - trace_address - ) - ), - incremental_traces AS ( - SELECT - f.block_number, - t.tx_hash, - t.block_timestamp, - t.origin_function_signature, - t.from_address AS origin_from_address, - t.to_address AS origin_to_address, - t.tx_status, - f.tx_position, - f.trace_index, - f.from_address AS from_address, - f.to_address AS to_address, - f.value_hex, - f.value_precise_raw, - f.value_precise, - f.value, - f.gas, - f.gas_used, - f.input, - f.output, - f.type, - f.identifier, - f.sub_traces, - f.error_reason, - f.revert_reason, - f.trace_status, - f.data, - f.traces_id, - f.trace_succeeded, - f.trace_address, - IFF( - t.tx_status = 'SUCCESS', - TRUE, - FALSE - ) AS tx_succeeded - FROM - json_traces f - LEFT OUTER JOIN {{ ref('silver__transactions') }} - t - ON f.tx_position = t.position - AND f.block_number = t.block_number - -{% if is_incremental() and not full_reload_mode %} -AND t.modified_timestamp >= ( - SELECT - DATEADD('hour', -24, MAX(modified_timestamp)) - FROM - {{ this }}) - {% endif %} -) - -{% if is_incremental() %}, -overflow_blocks AS ( - SELECT - DISTINCT block_number - FROM - silver_traces - WHERE - source = 'overflow' -), -heal_missing_data AS ( - SELECT - t.block_number, - txs.tx_hash, - txs.block_timestamp, - txs.origin_function_signature, - txs.from_address AS origin_from_address, - txs.to_address AS origin_to_address, - txs.tx_status, - t.tx_position, - t.trace_index, - t.from_address, - t.to_address, - t.value_hex, - t.value_precise_raw, - t.value_precise, - t.value, - t.gas, - t.gas_used, - t.input, - t.output, - t.type, - t.identifier, - t.sub_traces, - t.error_reason, - t.revert_reason, - t.trace_status, - t.data, - t.fact_traces_id AS traces_id, - t.trace_succeeded, - t.trace_address, - IFF( - txs.tx_status = 'SUCCESS', - TRUE, - FALSE - ) AS tx_succeeded - FROM - {{ this }} - t - JOIN {{ ref('silver__transactions') }} - txs - ON t.tx_position = txs.position - AND t.block_number = txs.block_number - WHERE - t.tx_hash IS NULL - OR t.block_timestamp IS NULL - OR t.tx_status IS NULL -) -{% endif %}, -all_traces AS ( - SELECT - block_number, - tx_hash, - block_timestamp, - origin_function_signature, - origin_from_address, - origin_to_address, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_hex, - value_precise_raw, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - revert_reason, - trace_status, - DATA, - trace_succeeded, - trace_address, - tx_succeeded - FROM - incremental_traces - -{% if is_incremental() %} -UNION ALL -SELECT - block_number, - tx_hash, - block_timestamp, - origin_function_signature, - origin_from_address, - origin_to_address, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_hex, - value_precise_raw, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - revert_reason, - trace_status, - DATA, - trace_succeeded, - trace_address, - tx_succeeded -FROM - heal_missing_data -UNION ALL -SELECT - block_number, - tx_hash, - block_timestamp, - origin_function_signature, - origin_from_address, - origin_to_address, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_hex, - value_precise_raw, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - revert_reason, - trace_status, - DATA, - trace_succeeded, - trace_address, - tx_succeeded -FROM - {{ this }} - JOIN overflow_blocks USING (block_number) -{% endif %} -) -SELECT - block_number, - block_timestamp, - tx_hash, - tx_position, - trace_index, - from_address, - to_address, - input, - output, - TYPE, - trace_address, - sub_traces, - VALUE, - value_precise_raw, - value_precise, - value_hex, - gas, - gas_used, - origin_from_address, - origin_to_address, - origin_function_signature, - trace_succeeded, - error_reason, - revert_reason, - tx_succeeded, - identifier, - DATA, - tx_status, - trace_status, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'trace_index'] - ) }} AS fact_traces_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM - all_traces qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index -ORDER BY - modified_timestamp DESC, block_timestamp DESC nulls last)) = 1 diff --git a/models/silver/core/silver__native_transfers.sql b/models/silver/core/silver__native_transfers.sql index 8f725ba..5d94a3b 100644 --- a/models/silver/core/silver__native_transfers.sql +++ b/models/silver/core/silver__native_transfers.sql @@ -16,15 +16,24 @@ WITH matic_base AS ( identifier, from_address, to_address, - matic_value, - _call_id, - _inserted_timestamp, - matic_value_precise_raw, - matic_value_precise, + VALUE AS matic_value, + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp, + value_precise_raw AS matic_value_precise_raw, + value_precise AS matic_value_precise, tx_position, trace_index FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE matic_value > 0 AND tx_status = 'SUCCESS' diff --git a/models/silver/core/silver__relevant_contracts.sql b/models/silver/core/silver__relevant_contracts.sql index 85b5b2b..23aefb7 100644 --- a/models/silver/core/silver__relevant_contracts.sql +++ b/models/silver/core/silver__relevant_contracts.sql @@ -31,10 +31,10 @@ function_calls AS ( SELECT to_address AS contract_address, COUNT(*) AS function_call_count, - MAX(_inserted_timestamp) AS max_inserted_timestamp_traces, + MAX(modified_timestamp) AS max_inserted_timestamp_traces, MAX(block_number) AS latest_call_block FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE tx_status = 'SUCCESS' AND trace_status = 'SUCCESS' @@ -43,7 +43,7 @@ function_calls AS ( AND input <> '0x' {% if is_incremental() %} -AND _inserted_timestamp > ( +AND modified_timestamp > ( SELECT MAX(max_inserted_timestamp_traces) FROM diff --git a/models/silver/core/silver__traces.sql b/models/silver/core/silver__traces.sql index bc48a57..d52cf23 100644 --- a/models/silver/core/silver__traces.sql +++ b/models/silver/core/silver__traces.sql @@ -1,27 +1,33 @@ -- depends_on: {{ ref('bronze__traces') }} -{% set warehouse = 'DBT_SNOWPARK' if var('OVERFLOWED_TRACES') else target.warehouse %} {{ config ( materialized = "incremental", incremental_strategy = 'delete+insert', unique_key = "block_number", - cluster_by = "block_timestamp::date, _inserted_timestamp::date", + cluster_by = ['modified_timestamp::DATE','partition_key'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", full_refresh = false, - tags = ['non_realtime', 'overflowed_traces'], - snowflake_warehouse = warehouse + tags = ['non_realtime'] ) }} - +{# {{ silver_traces_v1( +full_reload_start_block = 5000000, +full_reload_blocks = 1000000, +use_partition_key = TRUE +) }} +#} WITH bronze_traces AS ( SELECT block_number, + partition_key, VALUE :array_index :: INT AS tx_position, DATA :result AS full_traces, _inserted_timestamp FROM -{% if is_incremental() %} -{{ ref('bronze__traces') }} +{% if is_incremental() and not full_reload_mode %} +{{ ref( + 'bronze__traces' +) }} WHERE _inserted_timestamp >= ( SELECT @@ -29,12 +35,29 @@ WHERE FROM {{ this }} ) - AND DATA :result IS NOT NULL -{% else %} - {{ ref('bronze__traces_fr') }} + AND DATA :result IS NOT NULL {% elif is_incremental() and full_reload_mode %} + {{ ref( + 'bronze__traces_fr' + ) }} WHERE - _partition_by_block_id <= 2300000 - AND DATA :result IS NOT NULL + partition_key BETWEEN ( + SELECT + MAX(partition_key) - 100000 + FROM + {{ this }} + ) + AND ( + SELECT + MAX(partition_key) + 1000000 + FROM + {{ this }} + ) +{% else %} + {{ ref( + 'bronze__traces_fr' + ) }} +WHERE + partition_key <= 5000000 {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position @@ -45,6 +68,7 @@ flatten_traces AS ( SELECT block_number, tx_position, + partition_key, IFF( path IN ( 'result', @@ -58,6 +82,7 @@ flatten_traces AS ( 'result.output', 'result.error', 'result.revertReason', + 'result.time', 'gasUsed', 'gas', 'type', @@ -67,6 +92,7 @@ flatten_traces AS ( 'input', 'error', 'output', + 'time', 'revertReason' ), 'ORIGIN', @@ -93,7 +119,7 @@ flatten_traces AS ( SPLIT( trace_address, '_' - ) AS str_array + ) AS trace_address_array FROM bronze_traces txs, TABLE( @@ -111,440 +137,26 @@ flatten_traces AS ( GROUP BY block_number, tx_position, + partition_key, trace_address, _inserted_timestamp -), -sub_traces AS ( - SELECT - block_number, - tx_position, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - flatten_traces - GROUP BY - block_number, - tx_position, - parent_trace_address -), -num_array AS ( - SELECT - block_number, - tx_position, - trace_address, - ARRAY_AGG(flat_value) AS num_array - FROM - ( - SELECT - block_number, - tx_position, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - flatten_traces, - LATERAL FLATTEN ( - input => str_array - ) - ) - GROUP BY - block_number, - tx_position, - trace_address -), -cleaned_traces AS ( - SELECT - b.block_number, - b.tx_position, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - num_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, - b.tx_position - ORDER BY - num_array ASC - ) - 1 AS trace_index, - trace_json, - b._inserted_timestamp - FROM - flatten_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_position = s.tx_position - AND b.trace_address = s.parent_trace_address - JOIN num_array n - ON b.block_number = n.block_number - AND b.tx_position = n.tx_position - AND b.trace_address = n.trace_address -), -final_traces AS ( - SELECT - tx_position, - trace_index, - block_number, - trace_address, - trace_json :error :: STRING AS error_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS matic_value_precise_raw, - utils.udf_decimal_adjust( - matic_value_precise_raw, - 18 - ) AS matic_value_precise, - matic_value_precise :: FLOAT AS matic_value, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - concat_ws( - '-', - block_number, - tx_position, - identifier - ) AS _call_id, - _inserted_timestamp, - trace_json AS DATA, - sub_traces - FROM - cleaned_traces -), -new_records AS ( - SELECT - f.block_number, - t.tx_hash, - t.block_timestamp, - t.tx_status, - f.tx_position, - f.trace_index, - f.from_address, - f.to_address, - f.matic_value_precise_raw, - f.matic_value_precise, - f.matic_value, - f.gas, - f.gas_used, - f.input, - f.output, - f.type, - f.identifier, - f.sub_traces, - f.error_reason, - IFF( - f.error_reason IS NULL, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - f.data, - IFF( - t.tx_hash IS NULL - OR t.block_timestamp IS NULL - OR t.tx_status IS NULL, - TRUE, - FALSE - ) AS is_pending, - f._call_id, - f._inserted_timestamp - FROM - final_traces f - LEFT OUTER JOIN {{ ref('silver__transactions') }} - t - ON f.tx_position = t.position - AND f.block_number = t.block_number - -{% if is_incremental() %} -AND t._INSERTED_TIMESTAMP >= ( - SELECT - DATEADD('hour', -24, MAX(_inserted_timestamp)) - FROM - {{ this }}) - {% endif %} -) - -{% if is_incremental() %}, -missing_data AS ( - SELECT - t.block_number, - txs.tx_hash, - txs.block_timestamp, - txs.tx_status, - t.tx_position, - t.trace_index, - t.from_address, - t.to_address, - t.matic_value_precise_raw, - t.matic_value_precise, - t.matic_value, - t.gas, - t.gas_used, - t.input, - t.output, - t.type, - t.identifier, - t.sub_traces, - t.error_reason, - t.trace_status, - t.data, - FALSE AS is_pending, - t._call_id, - GREATEST( - t._inserted_timestamp, - txs._inserted_timestamp - ) AS _inserted_timestamp - FROM - {{ this }} - t - INNER JOIN {{ ref('silver__transactions') }} - txs - ON t.tx_position = txs.position - AND t.block_number = txs.block_number - WHERE - t.is_pending -) -{% endif %}, - -{% if is_incremental() and var( - 'OVERFLOWED_TRACES', -) %} -overflowed_traces AS ( - SELECT - t.block_number, - txs.tx_hash, - txs.block_timestamp, - txs.tx_status, - t.tx_position, - t.trace_index, - t.from_address, - t.to_address, - t.matic_value_precise_raw, - t.matic_value_precise, - t.matic_value, - t.gas, - t.gas_used, - t.input, - t.output, - t.type, - t.identifier, - t.sub_traces, - t.error_reason, - IFF( - t.error_reason IS NULL, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - t.data, - IFF( - txs.tx_hash IS NULL - OR txs.block_timestamp IS NULL - OR txs.tx_status IS NULL, - TRUE, - FALSE - ) AS is_pending, - t._call_id, - txs._inserted_timestamp AS _inserted_timestamp - FROM - {{ source( - 'polygon_silver', - 'overflowed_traces' - ) }} - t - LEFT JOIN {{ ref('silver__transactions') }} - txs - ON t.tx_position = txs.position - AND t.block_number = txs.block_number -), -{% endif %} - -FINAL AS ( - SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - matic_value_precise_raw, - matic_value_precise, - matic_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp - FROM - new_records - -{% if is_incremental() %} -UNION -SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - matic_value_precise_raw, - matic_value_precise, - matic_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp -FROM - missing_data -UNION ALL -SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - matic_value_precise_raw, - matic_value_precise, - matic_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp -FROM - {{ this }} - INNER JOIN ( - SELECT - DISTINCT block_number - FROM - missing_data - -{% if is_incremental() and var( - 'OVERFLOWED_TRACES', -) %} -UNION -SELECT - DISTINCT block_number -FROM - overflowed_traces -{% endif %} -) USING (block_number) -{% endif %} - -{% if is_incremental() and var( - 'OVERFLOWED_TRACES', -) %} -UNION ALL -SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - matic_value_precise_raw, - matic_value_precise, - matic_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp -FROM - overflowed_traces -{% endif %} ) SELECT block_number, - tx_hash, - block_timestamp, - tx_status, tx_position, - trace_index, - from_address, - to_address, - matic_value_precise, - matic_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, + trace_address, + parent_trace_address, + trace_address_array, + trace_json, + partition_key, _inserted_timestamp, - matic_value_precise_raw, {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'trace_index'] + ['block_number', 'tx_position', 'trace_address'] ) }} AS traces_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index + flatten_traces qualify(ROW_NUMBER() over(PARTITION BY traces_id ORDER BY - _inserted_timestamp DESC, is_pending ASC)) = 1 + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql deleted file mode 100644 index ae15b13..0000000 --- a/models/silver/core/silver__traces2.sql +++ /dev/null @@ -1,15 +0,0 @@ --- depends_on: {{ ref('bronze__traces') }} -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = "block_number", - cluster_by = ['modified_timestamp::DATE','partition_key'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - full_refresh = false, - tags = ['non_realtime'] -) }} -{{ silver_traces_v1( - full_reload_start_block = 5000000, - full_reload_blocks = 1000000, - use_partition_key = true -) }} diff --git a/models/silver/core/tests/traces/test_silver__traces_full.yml b/models/silver/core/tests/traces/test_silver__traces_full.yml index 55902f3..f21a7e1 100644 --- a/models/silver/core/tests/traces/test_silver__traces_full.yml +++ b/models/silver/core/tests/traces/test_silver__traces_full.yml @@ -1,58 +1,52 @@ version: 2 models: - name: test_silver__traces_full + description: "This is a view used to test all of the silver traces model." tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - BLOCK_NUMBER - - TX_POSITION - - TRACE_INDEX + - TRACES_ID + columns: - name: BLOCK_NUMBER tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: BLOCK_TIMESTAMP + - not_null + - name: TX_POSITION tests: - - not_null: - where: NOT IS_PENDING + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: TRACE_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + datepart: hour + interval: 2 - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: + - TIMESTAMP_LTZ - TIMESTAMP_NTZ - - name: TX_HASH - tests: - - not_null: - where: NOT IS_PENDING - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - - name: FROM_ADDRESS - tests: - - not_null: - where: TYPE <> 'SELFDESTRUCT' - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - - name: TO_ADDRESS - tests: - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - where: TO_ADDRESS IS NOT NULL - - name: IDENTIFIER + - name: INSERTED_TIMESTAMP tests: - not_null - - name: MATIC_VALUE + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP tests: - not_null - - name: GAS - tests: - - not_null - - name: GAS_USED - tests: - - not_null - - + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.yml b/models/silver/core/tests/traces/test_silver__traces_recent.yml index 2e8b3d5..10e6629 100644 --- a/models/silver/core/tests/traces/test_silver__traces_recent.yml +++ b/models/silver/core/tests/traces/test_silver__traces_recent.yml @@ -1,34 +1,52 @@ version: 2 models: - name: test_silver__traces_recent + description: "This is a view used to test the last three days of traces." tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - BLOCK_NUMBER - - TX_POSITION - - TRACE_INDEX + - TRACES_ID + columns: - name: BLOCK_NUMBER tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: BLOCK_TIMESTAMP + - not_null + - name: TX_POSITION tests: - - not_null: - where: NOT IS_PENDING + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: TRACE_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + datepart: hour + interval: 2 - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: + - TIMESTAMP_LTZ - TIMESTAMP_NTZ - - name: TX_HASH + - name: INSERTED_TIMESTAMP tests: - - not_null: - where: NOT IS_PENDING - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - \ No newline at end of file + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ diff --git a/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql b/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql index 2aa7c77..899d5fa 100644 --- a/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql +++ b/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql @@ -1,12 +1,11 @@ -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = ["block_number", "tx_position", "trace_address"], - tags = ['traces_fix'] +{# {{ config ( +materialized = "incremental", +incremental_strategy = 'delete+insert', +unique_key = ["block_number", "tx_position", "trace_address"], +tags = ['traces_fix'] ) }} {% set batch_query %} - SELECT MAX(next_batch_id) AS next_batch_id FROM @@ -161,4 +160,4 @@ aggregated_errors AS ( prod_trace_succeeded FROM batch - CROSS JOIN final_errors + CROSS JOIN final_errors #} diff --git a/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql b/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql index 784cc05..4874cb0 100644 --- a/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql +++ b/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql @@ -13,10 +13,19 @@ WITH base_contracts AS ( block_timestamp, from_address, to_address AS contract_address, - _call_id, - _inserted_timestamp + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE from_address = LOWER('0x808d7c71ad2ba3FA531b068a2417C63106BC0949') AND TYPE ILIKE 'create%' diff --git a/models/silver/defi/dex/curve/silver_dex__curve_pools.sql b/models/silver/defi/dex/curve/silver_dex__curve_pools.sql index e85ab25..09dd4a0 100644 --- a/models/silver/defi/dex/curve/silver_dex__curve_pools.sql +++ b/models/silver/defi/dex/curve/silver_dex__curve_pools.sql @@ -14,15 +14,24 @@ WITH contract_deployments AS ( block_timestamp, from_address AS deployer_address, to_address AS contract_address, - _call_id, - _inserted_timestamp, + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp, ROW_NUMBER() over ( ORDER BY contract_address ) AS row_num FROM {{ ref( - 'silver__traces' + 'core__fact_traces' ) }} WHERE -- curve contract deployers @@ -372,4 +381,4 @@ SELECT token_address ASC ) AS token_num FROM - FINAL + FINAL \ No newline at end of file diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql index fb6def1..540996a 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql @@ -13,10 +13,19 @@ WITH contract_deployments AS ( block_timestamp, from_address AS deployer_address, to_address AS contract_address, - _call_id, - _inserted_timestamp + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE from_address IN ( '0x63ae536fec0b57bdeb1fd6a893191b4239f61bff', @@ -41,7 +50,6 @@ qualify(ROW_NUMBER() over(PARTITION BY to_address ORDER BY block_timestamp ASC)) = 1 ) - SELECT tx_hash, block_number, @@ -50,4 +58,5 @@ SELECT contract_address AS pool_address, _call_id, _inserted_timestamp -FROM contract_deployments +FROM + contract_deployments diff --git a/models/sources.yml b/models/sources.yml index 31b53ea..288730c 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -60,7 +60,6 @@ sources: tables: - name: verified_abis - name: overflowed_traces - - name: overflowed_traces2 - name: polygon_bronze_api database: polygon schema: bronze_api @@ -80,4 +79,9 @@ sources: database: external schema: polymarket tables: - - name: dim_markets \ No newline at end of file + - name: dim_markets + - name: polygon_gold + database: "{{ 'polygon' if target.database == 'POLYGON' else 'polygon_dev' }}" + schema: core + tables: + - name: fact_traces diff --git a/models/streamline/silver/balances/history/streamline__matic_balances_history.sql b/models/streamline/silver/balances/history/streamline__matic_balances_history.sql index 71910bc..f6612c5 100644 --- a/models/streamline/silver/balances/history/streamline__matic_balances_history.sql +++ b/models/streamline/silver/balances/history/streamline__matic_balances_history.sql @@ -20,9 +20,9 @@ traces AS ( from_address, to_address FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE - matic_value > 0 + VALUE > 0 AND trace_status = 'SUCCESS' AND tx_status = 'SUCCESS' AND block_number < ( diff --git a/models/streamline/silver/balances/realtime/streamline__matic_balances_realtime.sql b/models/streamline/silver/balances/realtime/streamline__matic_balances_realtime.sql index 4ecb9df..4bb3805 100644 --- a/models/streamline/silver/balances/realtime/streamline__matic_balances_realtime.sql +++ b/models/streamline/silver/balances/realtime/streamline__matic_balances_realtime.sql @@ -24,9 +24,9 @@ traces AS ( from_address, to_address FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE - matic_value > 0 + VALUE > 0 AND trace_status = 'SUCCESS' AND tx_status = 'SUCCESS' AND block_number >= ( diff --git a/tests/polygon/test_silver__missing_traces.sql b/tests/polygon/test_silver__missing_traces.sql index 90060b9..c842996 100644 --- a/tests/polygon/test_silver__missing_traces.sql +++ b/tests/polygon/test_silver__missing_traces.sql @@ -1,2 +1,2 @@ -- depends_on: {{ ref('test_silver__transactions_full') }} -{{ missing_traces(ref("test_silver__traces_full")) }} +{{ missing_traces(ref("test_gold__fact_traces_full")) }} diff --git a/tests/polygon/test_silver__recent_missing_traces.sql b/tests/polygon/test_silver__recent_missing_traces.sql index 537d8f1..70e60af 100644 --- a/tests/polygon/test_silver__recent_missing_traces.sql +++ b/tests/polygon/test_silver__recent_missing_traces.sql @@ -1,2 +1,2 @@ -- depends_on: {{ ref('test_silver__transactions_recent') }} -{{ recent_missing_traces(ref("test_silver__traces_recent"), 10000) }} +{{ recent_missing_traces(ref("test_gold__fact_traces_recent"), 10000) }}