diff --git a/models/evm/gold/core/tests/traces/test_gold__fact_traces_full.sql b/models/evm/gold/core/tests/traces/test_gold__fact_traces_full.sql new file mode 100644 index 0000000..1d24175 --- /dev/null +++ b/models/evm/gold/core/tests/traces/test_gold__fact_traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('core_evm__fact_traces') }} diff --git a/models/evm/gold/core/tests/traces/test_gold__fact_traces_full.yml b/models/evm/gold/core/tests/traces/test_gold__fact_traces_full.yml new file mode 100644 index 0000000..6677117 --- /dev/null +++ b/models/evm/gold/core/tests/traces/test_gold__fact_traces_full.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold__fact_traces_full + description: "This is a view used to test all of the gold fact traces model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - fsc_utils.sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 AND TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 diff --git a/models/evm/gold/core/tests/traces/test_gold__fact_traces_recent.sql b/models/evm/gold/core/tests/traces/test_gold__fact_traces_recent.sql new file mode 100644 index 0000000..398992d --- /dev/null +++ b/models/evm/gold/core/tests/traces/test_gold__fact_traces_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_test'] +) }} + +SELECT + * +FROM + {{ ref('core_evm__fact_traces') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_block_lookback') }} + ) diff --git a/models/evm/gold/core/tests/traces/test_gold__fact_traces_recent.yml b/models/evm/gold/core/tests/traces/test_gold__fact_traces_recent.yml new file mode 100644 index 0000000..4206f50 --- /dev/null +++ b/models/evm/gold/core/tests/traces/test_gold__fact_traces_recent.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold__fact_traces_recent + description: "This is a view used to test the last three days of fact traces." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - fsc_utils.sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 diff --git a/models/evm/silver/core/silver_evm__traces_old.sql b/models/evm/silver/core/silver_evm__traces_old.sql deleted file mode 100644 index e788bf1..0000000 --- a/models/evm/silver/core/silver_evm__traces_old.sql +++ /dev/null @@ -1,429 +0,0 @@ --- depends_on: {{ ref('bronze_evm__streamline_traces') }} -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = "block_number", - cluster_by = "block_timestamp::date, _inserted_timestamp::date", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - full_refresh = false, - tags = ['core'] -) }} - -{% if execute %} - {% set query_traces %} - CREATE - OR REPLACE temporary TABLE silver_evm.traces__traces_intermediate_tmp AS - - SELECT - VALUE :BLOCK_NUMBER :: INT AS block_number, - VALUE :array_index :: INT AS tx_position, - DATA :result AS full_traces, - DATA :txHash :: STRING AS tx_hash, - _inserted_timestamp - FROM - -{% if is_incremental() %} -{{ ref('bronze_evm__streamline_traces') }} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) _inserted_timestamp - FROM - {{ this }} - ) - AND DATA :result IS NOT NULL -{% else %} - {{ ref('bronze_evm__streamline_fr_traces') }} -WHERE - DATA :result IS NOT NULL -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position -ORDER BY - _inserted_timestamp DESC)) = 1 {% endset %} - {% do run_query( - query_traces - ) %} -{% endif %} - -WITH flatten_traces AS ( - SELECT - block_number, - tx_hash, - IFF( - path IN ( - 'result', - 'result.value', - 'result.type', - 'result.to', - 'result.input', - 'result.gasUsed', - 'result.gas', - 'result.from', - 'result.output', - 'result.error', - 'result.revertReason', - 'gasUsed', - 'gas', - 'type', - 'to', - 'from', - 'value', - 'input', - 'error', - 'output', - 'revertReason' - ), - 'ORIGIN', - REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') - ) AS trace_address, - _inserted_timestamp, - OBJECT_AGG( - key, - VALUE - ) AS trace_json, - CASE - WHEN trace_address = 'ORIGIN' THEN NULL - WHEN POSITION( - '_' IN trace_address - ) = 0 THEN 'ORIGIN' - ELSE REGEXP_REPLACE( - trace_address, - '_[0-9]+$', - '', - 1, - 1 - ) - END AS parent_trace_address, - SPLIT( - trace_address, - '_' - ) AS str_array - FROM - silver_evm.traces__traces_intermediate_tmp txs, - TABLE( - FLATTEN( - input => PARSE_JSON( - txs.full_traces - ), - recursive => TRUE - ) - ) f - WHERE - f.index IS NULL - AND f.key != 'calls' - AND f.path != 'result' - GROUP BY - block_number, - tx_hash, - trace_address, - _inserted_timestamp -), -sub_traces AS ( - SELECT - block_number, - tx_hash, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - flatten_traces - GROUP BY - block_number, - tx_hash, - parent_trace_address -), -num_array AS ( - SELECT - block_number, - tx_hash, - trace_address, - ARRAY_AGG(flat_value) AS num_array - FROM - ( - SELECT - block_number, - tx_hash, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - flatten_traces, - LATERAL FLATTEN ( - input => str_array - ) - ) - GROUP BY - block_number, - tx_hash, - trace_address -), -cleaned_traces AS ( - SELECT - b.block_number, - b.tx_hash, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - num_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, - b.tx_hash - ORDER BY - num_array ASC - ) - 1 AS trace_index, - trace_json, - b._inserted_timestamp - FROM - flatten_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_hash = s.tx_hash - AND b.trace_address = s.parent_trace_address - JOIN num_array n - ON b.block_number = n.block_number - AND b.tx_hash = n.tx_hash - AND b.trace_address = n.trace_address -), -final_traces AS ( - SELECT - tx_hash, - trace_index, - block_number, - trace_address, - trace_json :error :: STRING AS error_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS value_precise_raw, - utils.udf_decimal_adjust( - value_precise_raw, - 18 - ) AS value_precise, - value_precise :: FLOAT AS VALUE, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - _inserted_timestamp, - trace_json AS DATA, - sub_traces - FROM - cleaned_traces -), -new_records AS ( - SELECT - f.block_number, - f.tx_hash, - t.block_timestamp, - t.tx_status, - t.position AS tx_position, - f.trace_index, - f.from_address, - f.to_address, - f.value_precise_raw, - f.value_precise, - f.value, - f.gas, - f.gas_used, - f.input, - f.output, - f.type, - f.identifier, - f.sub_traces, - f.error_reason, - IFF( - f.error_reason IS NULL, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - f.data, - IFF( - t.tx_hash IS NULL - OR t.block_timestamp IS NULL - OR t.tx_status IS NULL, - TRUE, - FALSE - ) AS is_pending, - concat_ws( - '-', - f.block_number, - t.position, - f.identifier - ) AS _call_id, - f._inserted_timestamp - FROM - final_traces f - LEFT OUTER JOIN {{ ref('silver_evm__transactions') }} - t - ON f.tx_hash = t.tx_hash - AND f.block_number = t.block_number - -{% if is_incremental() %} -AND t._INSERTED_TIMESTAMP >= ( - SELECT - DATEADD('hour', -24, MAX(_inserted_timestamp)) - FROM - {{ this }}) - {% endif %} -) - -{% if is_incremental() %}, -missing_data AS ( - SELECT - t.block_number, - t.tx_hash, - txs.block_timestamp, - txs.tx_status, - txs.position AS tx_position, - t.trace_index, - t.from_address, - t.to_address, - t.value_precise_raw, - t.value_precise, - t.value, - t.gas, - t.gas_used, - t.input, - t.output, - t.type, - t.identifier, - t.sub_traces, - t.error_reason, - t.trace_status, - t.data, - FALSE AS is_pending, - concat_ws( - '-', - t.block_number, - txs.position, - t.identifier - ) AS _call_id, - GREATEST( - t._inserted_timestamp, - txs._inserted_timestamp - ) AS _inserted_timestamp - FROM - {{ this }} - t - INNER JOIN {{ ref('silver_evm__transactions') }} - txs - ON t.tx_hash = txs.tx_hash - AND t.block_number = txs.block_number - WHERE - t.is_pending -) -{% endif %}, -FINAL AS ( - SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_precise_raw, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp - FROM - new_records - -{% if is_incremental() %} -UNION -SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_precise_raw, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp -FROM - missing_data -{% endif %} -) -SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp, - value_precise_raw, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'trace_index'] - ) }} AS traces_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_hash, trace_index -ORDER BY - _inserted_timestamp DESC, is_pending ASC)) = 1 diff --git a/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.sql b/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.sql index ad564d9..212bf1d 100644 --- a/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.sql +++ b/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.sql @@ -6,6 +6,6 @@ SELECT * FROM - {{ ref('core_evm__fact_traces') }} + {{ ref('silver_evm__traces') }} WHERE block_timestamp < DATEADD('hour', -1, SYSDATE()) diff --git a/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.yml b/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.yml index 4a080bb..f0d5d08 100644 --- a/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.yml +++ b/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.yml @@ -1,59 +1,52 @@ version: 2 models: - - name: test_silver_evm__traces_full + - name: test_silver__traces_full + description: "This is a view used to test all of the silver traces model." tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - BLOCK_NUMBER - - TX_POSITION - - TRACE_INDEX + - TRACES_ID + columns: - name: BLOCK_NUMBER tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: BLOCK_TIMESTAMP + - not_null + - name: TX_HASH tests: - - not_null: - where: NOT IS_PENDING + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: TRACE_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + datepart: hour + interval: 2 - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_LTZ - TIMESTAMP_NTZ - - name: TX_HASH - tests: - - not_null: - where: NOT IS_PENDING - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - - name: FROM_ADDRESS - tests: - - not_null: - where: TYPE <> 'SELFDESTRUCT' - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - - name: TO_ADDRESS - tests: - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - where: TO_ADDRESS IS NOT NULL - - name: IDENTIFIER + - name: INSERTED_TIMESTAMP tests: - not_null - - name: VALUE + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP tests: - not_null - - name: GAS - tests: - - not_null - - name: GAS_USED - tests: - - not_null - - + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ diff --git a/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.sql b/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.sql index a4b0e79..3bec4ed 100644 --- a/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.sql +++ b/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.sql @@ -13,7 +13,7 @@ WITH last_3_days AS ( SELECT * FROM - {{ ref('core_evm__fact_traces') }} + {{ ref('silver_evm__traces') }} WHERE block_number >= ( SELECT diff --git a/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.yml b/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.yml index 058449a..ed5acc4 100644 --- a/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.yml +++ b/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.yml @@ -1,35 +1,52 @@ version: 2 models: - - name: test_silver_evm__traces_recent + - name: test_silver__traces_recent + description: "This is a view used to test the last three days of traces." tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - BLOCK_NUMBER - - TX_POSITION - - TRACE_INDEX + - TRACES_ID + columns: - name: BLOCK_NUMBER tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: BLOCK_TIMESTAMP + - not_null + - name: TX_HASH tests: - - not_null: - where: NOT IS_PENDING + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: TRACE_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + datepart: hour + interval: 2 - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_LTZ - TIMESTAMP_NTZ - - name: TX_HASH + - name: INSERTED_TIMESTAMP tests: - - not_null: - where: NOT IS_PENDING - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - \ No newline at end of file + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ