From c930126c5a2f9300a49a6d5dad2c7fd55cbb887f Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Tue, 2 May 2023 10:33:29 -0400 Subject: [PATCH] tests (#498) --- .github/workflows/dbt_test.yml | 2 +- macros/tests/missing_txs.sql | 63 +++ models/silver/core/silver__traces2.sql | 468 +++++++++--------- models/silver/core/silver__transactions2.sql | 9 +- .../tests/blocks/test_silver__blocks_full.sql | 9 + .../tests/blocks/test_silver__blocks_full.yml | 94 ++++ .../blocks/test_silver__blocks_recent.sql | 27 + .../blocks/test_silver__blocks_recent.yml | 26 + .../event_logs/test_silver__logs_full.sql | 9 + .../event_logs/test_silver__logs_full.yml | 76 +++ .../event_logs/test_silver__logs_recent.sql | 27 + .../event_logs/test_silver__logs_recent.yml | 32 ++ .../receipts/test_silver__receipts_full.sql | 9 + .../receipts/test_silver__receipts_full.yml | 82 +++ .../receipts/test_silver__receipts_recent.sql | 27 + .../receipts/test_silver__receipts_recent.yml | 28 ++ .../tests/traces/test_silver__traces_full.sql | 9 + .../tests/traces/test_silver__traces_full.yml | 56 +++ .../traces/test_silver__traces_recent.sql | 27 + .../traces/test_silver__traces_recent.yml | 32 ++ .../test_silver__transactions_full.sql | 9 + .../test_silver__transactions_full.yml | 118 +++++ .../test_silver__transactions_recent.sql | 27 + .../test_silver__transactions_recent.yml | 22 + .../test_silver__missing_receipts.sql | 2 + .../ethereum/test_silver__missing_traces.sql | 2 + .../test_silver__recent_missing_receipts.sql | 2 + .../test_silver__recent_missing_traces.sql | 2 + .../test_silver__transactions_tx_gaps.sql | 2 - 29 files changed, 1055 insertions(+), 243 deletions(-) create mode 100644 macros/tests/missing_txs.sql create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_full.sql create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_full.yml create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_recent.sql create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_recent.yml create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_full.sql create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_full.yml create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_recent.sql create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_recent.yml create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_full.sql create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_full.yml create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_recent.sql create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_recent.yml create mode 100644 models/silver/core/tests/traces/test_silver__traces_full.sql create mode 100644 models/silver/core/tests/traces/test_silver__traces_full.yml create mode 100644 models/silver/core/tests/traces/test_silver__traces_recent.sql create mode 100644 models/silver/core/tests/traces/test_silver__traces_recent.yml create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_full.sql create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_full.yml create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_recent.sql create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_recent.yml create mode 100644 tests/ethereum/test_silver__missing_receipts.sql create mode 100644 tests/ethereum/test_silver__missing_traces.sql create mode 100644 tests/ethereum/test_silver__recent_missing_receipts.sql create mode 100644 tests/ethereum/test_silver__recent_missing_traces.sql delete mode 100644 tests/ethereum/test_silver__transactions_tx_gaps.sql diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index ed2f2699..f99941ae 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -41,7 +41,7 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt test + dbt test --exclude tag:full_test tag:recent_test diff --git a/macros/tests/missing_txs.sql b/macros/tests/missing_txs.sql new file mode 100644 index 00000000..8abc4624 --- /dev/null +++ b/macros/tests/missing_txs.sql @@ -0,0 +1,63 @@ +{% macro missing_txs( + model + ) %} + WITH txs_base AS ( + SELECT + block_number AS base_block_number, + tx_hash AS base_tx_hash + FROM + {{ ref('test_silver__transactions_full') }} + ), + model_name AS ( + SELECT + block_number AS model_block_number, + tx_hash AS model_tx_hash + FROM + {{ model }} + ) +SELECT + base_block_number, + base_tx_hash, + model_block_number, + model_tx_hash +FROM + txs_base + LEFT JOIN model_name + ON base_block_number = model_block_number + AND base_tx_hash = model_tx_hash +WHERE + model_tx_hash IS NULL + OR model_block_number IS NULL +{% endmacro %} + +{% macro recent_missing_txs( + model + ) %} + WITH txs_base AS ( + SELECT + block_number AS base_block_number, + tx_hash AS base_tx_hash + FROM + {{ ref('test_silver__transactions_recent') }} + ), + model_name AS ( + SELECT + block_number AS model_block_number, + tx_hash AS model_tx_hash + FROM + {{ model }} + ) +SELECT + base_block_number, + base_tx_hash, + model_block_number, + model_tx_hash +FROM + txs_base + LEFT JOIN model_name + ON base_block_number = model_block_number + AND base_tx_hash = model_tx_hash +WHERE + model_tx_hash IS NULL + OR model_block_number IS NULL +{% endmacro %} diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index 031b083a..3e9bcb2c 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -17,245 +17,243 @@ WITH traces_txs AS ( FROM {% if is_incremental() %} -{{ ref('bronze__streamline_FR_traces') }} +{{ ref('bronze__streamline_traces') }} WHERE - _partition_by_block_id BETWEEN ( + _inserted_timestamp >= ( SELECT - ROUND(MAX(block_number), -4) + MAX(_inserted_timestamp) _inserted_timestamp FROM - {{ this }}) - AND ( - SELECT - ROUND(MAX(block_number), -4) + 1000000 - FROM - {{ this }}) - {% else %} - {{ ref('bronze__streamline_FR_traces') }} - WHERE - _partition_by_block_id <= 2500000 - {% endif %} + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_traces') }} +WHERE + _partition_by_block_id <= 2500000 +{% endif %} - qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position - ORDER BY - _inserted_timestamp DESC)) = 1 - ), - base_table AS ( - SELECT - CASE - WHEN POSITION( - '.', - path :: STRING - ) > 0 THEN REPLACE( - REPLACE( - path :: STRING, - SUBSTR(path :: STRING, len(path :: STRING) - POSITION('.', REVERSE(path :: STRING)) + 1, POSITION('.', REVERSE(path :: STRING))), - '' - ), - '.', - '__' - ) - ELSE '__' - END AS id, - OBJECT_AGG( - DISTINCT key, - VALUE - ) AS DATA, - txs.tx_position AS tx_position, - txs.block_number AS block_number, - txs._inserted_timestamp AS _inserted_timestamp - FROM - traces_txs txs, - TABLE( - FLATTEN( - input => PARSE_JSON( - txs.full_traces - ), - recursive => TRUE - ) - ) f - WHERE - f.index IS NULL - AND f.key != 'calls' - GROUP BY - tx_position, - id, - block_number, - _inserted_timestamp - ), - flattened_traces AS ( - SELECT - DATA :from :: STRING AS from_address, - PUBLIC.udf_hex_to_int( - DATA :gas :: STRING - ) AS gas, - PUBLIC.udf_hex_to_int( - DATA :gasUsed :: STRING - ) AS gas_used, - DATA :input :: STRING AS input, - DATA :output :: STRING AS output, - DATA :error :: STRING AS error_reason, - DATA :to :: STRING AS to_address, - DATA :type :: STRING AS TYPE, - CASE - WHEN DATA :type :: STRING = 'CALL' THEN PUBLIC.udf_hex_to_int( - DATA :value :: STRING - ) / pow( - 10, - 18 - ) - ELSE 0 - END AS eth_value, - CASE - WHEN id = '__' THEN CONCAT( - DATA :type :: STRING, - '_ORIGIN' - ) - ELSE CONCAT( - DATA :type :: STRING, - '_', - REPLACE( - REPLACE(REPLACE(REPLACE(id, 'calls', ''), '[', ''), ']', ''), - '__', - '_' - ) - ) - END AS identifier, - concat_ws( - '-', - block_number, - tx_position, - identifier - ) AS _call_id, - SPLIT( - identifier, - '_' - ) AS id_split, - ARRAY_SLICE(id_split, 1, ARRAY_SIZE(id_split)) AS levels, - ARRAY_TO_STRING( - levels, - '_' - ) AS LEVEL, - CASE - WHEN ARRAY_SIZE(levels) = 1 - AND levels [0] :: STRING = 'ORIGIN' THEN NULL - WHEN ARRAY_SIZE(levels) = 1 THEN 'ORIGIN' - ELSE ARRAY_TO_STRING(ARRAY_SLICE(levels, 0, ARRAY_SIZE(levels) -1), '_')END AS parent_level, - COUNT(parent_level) over ( - PARTITION BY block_number, - tx_position, - parent_level - ) AS sub_traces,* - FROM - base_table - ), - group_sub_traces AS ( - SELECT - tx_position, - block_number, - parent_level, - sub_traces - FROM - flattened_traces - GROUP BY - tx_position, - block_number, - parent_level, - sub_traces - ), - add_sub_traces AS ( - SELECT - flattened_traces.tx_position AS tx_position, - flattened_traces.block_number :: INTEGER AS block_number, - flattened_traces.error_reason AS error_reason, - flattened_traces.from_address AS from_address, - flattened_traces.to_address AS to_address, - flattened_traces.eth_value :: FLOAT AS eth_value, - flattened_traces.gas :: FLOAT AS gas, - flattened_traces.gas_used :: FLOAT AS gas_used, - flattened_traces.input AS input, - flattened_traces.output AS output, - flattened_traces.type AS TYPE, - flattened_traces.identifier AS identifier, - flattened_traces._call_id AS _call_id, - flattened_traces.data AS DATA, - group_sub_traces.sub_traces AS sub_traces, - ROW_NUMBER() over( - PARTITION BY flattened_traces.block_number, - flattened_traces.tx_position - ORDER BY - flattened_traces.gas :: FLOAT DESC - ) AS trace_index, - flattened_traces._inserted_timestamp AS _inserted_timestamp - FROM - flattened_traces - LEFT OUTER JOIN group_sub_traces - ON flattened_traces.tx_position = group_sub_traces.tx_position - AND flattened_traces.level = group_sub_traces.parent_level - AND flattened_traces.block_number = group_sub_traces.block_number - ), - final_traces AS ( - SELECT - tx_position, - trace_index, - block_number, - error_reason, - from_address, - to_address, - eth_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - _call_id, - _inserted_timestamp, - DATA, - sub_traces - FROM - add_sub_traces - WHERE - identifier IS NOT NULL - ), - new_records AS ( - SELECT - f.block_number, - t.tx_hash, - t.block_timestamp, - t.tx_status, - f.tx_position, - f.trace_index, - f.from_address, - f.to_address, - f.eth_value, - f.gas, - f.gas_used, - f.input, - f.output, - f.type, - f.identifier, - f.sub_traces, - f.error_reason, - CASE - WHEN f.error_reason IS NULL THEN 'SUCCESS' - ELSE 'FAIL' - END AS trace_status, - f.data, - CASE - WHEN t.tx_hash IS NULL - OR t.block_timestamp IS NULL - OR t.tx_status IS NULL THEN TRUE - ELSE FALSE - END AS is_pending, - f._call_id, - f._inserted_timestamp - FROM - final_traces f - LEFT OUTER JOIN {{ ref('silver__transactions2') }} - t - ON f.tx_position = t.position - AND f.block_number = t.block_number - ) +qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position +ORDER BY + _inserted_timestamp DESC)) = 1 +), +base_table AS ( + SELECT + CASE + WHEN POSITION( + '.', + path :: STRING + ) > 0 THEN REPLACE( + REPLACE( + path :: STRING, + SUBSTR(path :: STRING, len(path :: STRING) - POSITION('.', REVERSE(path :: STRING)) + 1, POSITION('.', REVERSE(path :: STRING))), + '' + ), + '.', + '__' + ) + ELSE '__' + END AS id, + OBJECT_AGG( + DISTINCT key, + VALUE + ) AS DATA, + txs.tx_position AS tx_position, + txs.block_number AS block_number, + txs._inserted_timestamp AS _inserted_timestamp + FROM + traces_txs txs, + TABLE( + FLATTEN( + input => PARSE_JSON( + txs.full_traces + ), + recursive => TRUE + ) + ) f + WHERE + f.index IS NULL + AND f.key != 'calls' + GROUP BY + tx_position, + id, + block_number, + _inserted_timestamp +), +flattened_traces AS ( + SELECT + DATA :from :: STRING AS from_address, + PUBLIC.udf_hex_to_int( + DATA :gas :: STRING + ) AS gas, + PUBLIC.udf_hex_to_int( + DATA :gasUsed :: STRING + ) AS gas_used, + DATA :input :: STRING AS input, + DATA :output :: STRING AS output, + DATA :error :: STRING AS error_reason, + DATA :to :: STRING AS to_address, + DATA :type :: STRING AS TYPE, + CASE + WHEN DATA :type :: STRING = 'CALL' THEN PUBLIC.udf_hex_to_int( + DATA :value :: STRING + ) / pow( + 10, + 18 + ) + ELSE 0 + END AS eth_value, + CASE + WHEN id = '__' THEN CONCAT( + DATA :type :: STRING, + '_ORIGIN' + ) + ELSE CONCAT( + DATA :type :: STRING, + '_', + REPLACE( + REPLACE(REPLACE(REPLACE(id, 'calls', ''), '[', ''), ']', ''), + '__', + '_' + ) + ) + END AS identifier, + concat_ws( + '-', + block_number, + tx_position, + identifier + ) AS _call_id, + SPLIT( + identifier, + '_' + ) AS id_split, + ARRAY_SLICE(id_split, 1, ARRAY_SIZE(id_split)) AS levels, + ARRAY_TO_STRING( + levels, + '_' + ) AS LEVEL, + CASE + WHEN ARRAY_SIZE(levels) = 1 + AND levels [0] :: STRING = 'ORIGIN' THEN NULL + WHEN ARRAY_SIZE(levels) = 1 THEN 'ORIGIN' + ELSE ARRAY_TO_STRING(ARRAY_SLICE(levels, 0, ARRAY_SIZE(levels) -1), '_')END AS parent_level, + COUNT(parent_level) over ( + PARTITION BY block_number, + tx_position, + parent_level + ) AS sub_traces,* + FROM + base_table + ), + group_sub_traces AS ( + SELECT + tx_position, + block_number, + parent_level, + sub_traces + FROM + flattened_traces + GROUP BY + tx_position, + block_number, + parent_level, + sub_traces + ), + add_sub_traces AS ( + SELECT + flattened_traces.tx_position AS tx_position, + flattened_traces.block_number :: INTEGER AS block_number, + flattened_traces.error_reason AS error_reason, + flattened_traces.from_address AS from_address, + flattened_traces.to_address AS to_address, + flattened_traces.eth_value :: FLOAT AS eth_value, + flattened_traces.gas :: FLOAT AS gas, + flattened_traces.gas_used :: FLOAT AS gas_used, + flattened_traces.input AS input, + flattened_traces.output AS output, + flattened_traces.type AS TYPE, + flattened_traces.identifier AS identifier, + flattened_traces._call_id AS _call_id, + flattened_traces.data AS DATA, + group_sub_traces.sub_traces AS sub_traces, + ROW_NUMBER() over( + PARTITION BY flattened_traces.block_number, + flattened_traces.tx_position + ORDER BY + flattened_traces.gas :: FLOAT DESC, + flattened_traces.eth_value :: FLOAT ASC, + flattened_traces.to_address + ) AS trace_index, + flattened_traces._inserted_timestamp AS _inserted_timestamp + FROM + flattened_traces + LEFT OUTER JOIN group_sub_traces + ON flattened_traces.tx_position = group_sub_traces.tx_position + AND flattened_traces.level = group_sub_traces.parent_level + AND flattened_traces.block_number = group_sub_traces.block_number + ), + final_traces AS ( + SELECT + tx_position, + trace_index, + block_number, + error_reason, + from_address, + to_address, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + _call_id, + _inserted_timestamp, + DATA, + sub_traces + FROM + add_sub_traces + WHERE + identifier IS NOT NULL + ), + new_records AS ( + SELECT + f.block_number, + t.tx_hash, + t.block_timestamp, + t.tx_status, + f.tx_position, + f.trace_index, + f.from_address, + f.to_address, + f.eth_value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.identifier, + f.sub_traces, + f.error_reason, + CASE + WHEN f.error_reason IS NULL THEN 'SUCCESS' + ELSE 'FAIL' + END AS trace_status, + f.data, + CASE + WHEN t.tx_hash IS NULL + OR t.block_timestamp IS NULL + OR t.tx_status IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + f._call_id, + f._inserted_timestamp + FROM + final_traces f + LEFT OUTER JOIN {{ ref('silver__transactions2') }} + t + ON f.tx_position = t.position + AND f.block_number = t.block_number + ) {% if is_incremental() %}, missing_data AS ( diff --git a/models/silver/core/silver__transactions2.sql b/models/silver/core/silver__transactions2.sql index a27c707d..21b96669 100644 --- a/models/silver/core/silver__transactions2.sql +++ b/models/silver/core/silver__transactions2.sql @@ -3,11 +3,10 @@ materialized = 'incremental', unique_key = "tx_hash", cluster_by = "block_timestamp::date, _inserted_timestamp::date", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - full_refresh = false + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION" ) }} -- -- add configs back and lookback macro --- -- incremental_predicates = ["dynamic_range", "block_timestamp::date"], +-- -- incremental_predicates = ["dynamic_range", "block_timestamp::date"], full_refresh = false WITH base AS ( SELECT @@ -118,7 +117,7 @@ new_records AS ( r ON A.block_number = r.block_number AND A.data :hash :: STRING = r.tx_hash - LEFT OUTER JOIN {{ ref('silver__blocks') }} + LEFT OUTER JOIN {{ ref('silver__blocks2') }} b ON A.block_number = b.block_number ) @@ -168,7 +167,7 @@ missing_data AS ( FROM {{ this }} t - INNER JOIN {{ ref('silver__blocks') }} + INNER JOIN {{ ref('silver__blocks2') }} b ON t.block_number = b.block_number INNER JOIN {{ ref('silver__receipts') }} diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.sql b/models/silver/core/tests/blocks/test_silver__blocks_full.sql new file mode 100644 index 00000000..e91850e5 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__blocks2') }} diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.yml b/models/silver/core/tests/blocks/test_silver__blocks_full.yml new file mode 100644 index 00000000..aef2a66c --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.yml @@ -0,0 +1,94 @@ +version: 2 +models: + - name: test_silver__blocks_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TOTAL_DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EXTRA_DATA + tests: + - not_null + - name: GAS_LIMIT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: PARENT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: MINER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: RECEIPTS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SHA3_UNCLES + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SIZE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.sql b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql new file mode 100644 index 00000000..67e25154 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__blocks2') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.yml b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml new file mode 100644 index 00000000..9ad3f04a --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: test_silver__blocks_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">10" + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + \ No newline at end of file diff --git a/models/silver/core/tests/event_logs/test_silver__logs_full.sql b/models/silver/core/tests/event_logs/test_silver__logs_full.sql new file mode 100644 index 00000000..a5179599 --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__logs2') }} diff --git a/models/silver/core/tests/event_logs/test_silver__logs_full.yml b/models/silver/core/tests/event_logs/test_silver__logs_full.yml new file mode 100644 index 00000000..264df6a1 --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_full.yml @@ -0,0 +1,76 @@ +version: 2 +models: + - name: test_silver__logs_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - tx_block_count: + config: + severity: error + error_if: "!=0" + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TOPICS + tests: + - not_null + - name: DATA + tests: + - not_null + - name: EVENT_REMOVED + tests: + - not_null + - name: _LOG_ID + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null: + where: NOT IS_PENDING + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ \ No newline at end of file diff --git a/models/silver/core/tests/event_logs/test_silver__logs_recent.sql b/models/silver/core/tests/event_logs/test_silver__logs_recent.sql new file mode 100644 index 00000000..b7755187 --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__logs2') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/event_logs/test_silver__logs_recent.yml b/models/silver/core/tests/event_logs/test_silver__logs_recent.yml new file mode 100644 index 00000000..ae46f47e --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_recent.yml @@ -0,0 +1,32 @@ +version: 2 +models: + - name: test_silver__logs_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null diff --git a/models/silver/core/tests/receipts/test_silver__receipts_full.sql b/models/silver/core/tests/receipts/test_silver__receipts_full.sql new file mode 100644 index 00000000..56f39ae1 --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__receipts') }} diff --git a/models/silver/core/tests/receipts/test_silver__receipts_full.yml b/models/silver/core/tests/receipts/test_silver__receipts_full.yml new file mode 100644 index 00000000..22ca7cca --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_full.yml @@ -0,0 +1,82 @@ +version: 2 +models: + - name: test_silver__receipts_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: CUMULATIVE_GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_STATUS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['SUCCESS', 'FAIL'] + - name: TYPE + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + + diff --git a/models/silver/core/tests/receipts/test_silver__receipts_recent.sql b/models/silver/core/tests/receipts/test_silver__receipts_recent.sql new file mode 100644 index 00000000..0b5a9017 --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__receipts') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/receipts/test_silver__receipts_recent.yml b/models/silver/core/tests/receipts/test_silver__receipts_recent.yml new file mode 100644 index 00000000..828793a6 --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_recent.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: test_silver__receipts_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + + diff --git a/models/silver/core/tests/traces/test_silver__traces_full.sql b/models/silver/core/tests/traces/test_silver__traces_full.sql new file mode 100644 index 00000000..eaf9db09 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__traces2') }} diff --git a/models/silver/core/tests/traces/test_silver__traces_full.yml b/models/silver/core/tests/traces/test_silver__traces_full.yml new file mode 100644 index 00000000..d42c96e8 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_full.yml @@ -0,0 +1,56 @@ +version: 2 +models: + - name: test_silver__traces_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _CALL_ID + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: IDENTIFIER + tests: + - not_null + - name: ETH_VALUE + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + + diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.sql b/models/silver/core/tests/traces/test_silver__traces_recent.sql new file mode 100644 index 00000000..acc55ecf --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__traces2') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.yml b/models/silver/core/tests/traces/test_silver__traces_recent.yml new file mode 100644 index 00000000..7915a850 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_recent.yml @@ -0,0 +1,32 @@ +version: 2 +models: + - name: test_silver__traces_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _CALL_ID + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + \ No newline at end of file diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.sql b/models/silver/core/tests/transactions/test_silver__transactions_full.sql new file mode 100644 index 00000000..636f9635 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__transactions2') }} diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.yml b/models/silver/core/tests/transactions/test_silver__transactions_full.yml new file mode 100644 index 00000000..746e2e32 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.yml @@ -0,0 +1,118 @@ +version: 2 +models: + - name: test_silver__transactions_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: VALUE + tests: + - not_null + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS + tests: + - not_null + - name: INPUT_DATA + tests: + - not_null + - name: TX_STATUS + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['SUCCESS', 'FAIL'] + where: NOT IS_PENDING + - name: GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CUMULATIVE_GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_FEE + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null: + where: NOT IS_PENDING + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + + diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.sql b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql new file mode 100644 index 00000000..1f751594 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__transactions2') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.yml b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml new file mode 100644 index 00000000..aa2881b0 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml @@ -0,0 +1,22 @@ +version: 2 +models: + - name: test_silver__transactions_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + \ No newline at end of file diff --git a/tests/ethereum/test_silver__missing_receipts.sql b/tests/ethereum/test_silver__missing_receipts.sql new file mode 100644 index 00000000..8481599b --- /dev/null +++ b/tests/ethereum/test_silver__missing_receipts.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_full') }} +{{ missing_txs(ref("test_silver__receipts_full")) }} diff --git a/tests/ethereum/test_silver__missing_traces.sql b/tests/ethereum/test_silver__missing_traces.sql new file mode 100644 index 00000000..c96256f3 --- /dev/null +++ b/tests/ethereum/test_silver__missing_traces.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_full') }} +{{ missing_txs(ref("test_silver__traces_full")) }} diff --git a/tests/ethereum/test_silver__recent_missing_receipts.sql b/tests/ethereum/test_silver__recent_missing_receipts.sql new file mode 100644 index 00000000..f657fd34 --- /dev/null +++ b/tests/ethereum/test_silver__recent_missing_receipts.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_recent') }} +{{ recent_missing_txs(ref("test_silver__receipts_recent")) }} diff --git a/tests/ethereum/test_silver__recent_missing_traces.sql b/tests/ethereum/test_silver__recent_missing_traces.sql new file mode 100644 index 00000000..26a69d4b --- /dev/null +++ b/tests/ethereum/test_silver__recent_missing_traces.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_recent') }} +{{ recent_missing_txs(ref("test_silver__traces_recent")) }} diff --git a/tests/ethereum/test_silver__transactions_tx_gaps.sql b/tests/ethereum/test_silver__transactions_tx_gaps.sql deleted file mode 100644 index 7f84b8f0..00000000 --- a/tests/ethereum/test_silver__transactions_tx_gaps.sql +++ /dev/null @@ -1,2 +0,0 @@ --- depends_on: {{ ref('silver__blocks') }} -{{ tx_gaps(ref("silver__transactions")) }}