From 6d5d0709d33b6bd423dc280f547762780e6b45bb Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Tue, 30 May 2023 10:53:20 -0400 Subject: [PATCH] tests (#81) * tests * traces tweaks * tweak test and lookback * recent test yml * schedule * incremental * move decoder test * fix exclude * increase refresh --- .../workflows/dbt_run_streamline_realtime.yml | 4 +- .github/workflows/dbt_run_temp_backfill.yml | 4 +- .github/workflows/dbt_test.yml | 2 +- .github/workflows/dbt_test_intraday.yml | 47 +++++++ macros/tests/missing_txs.sql | 63 ++++++++++ macros/tests/tx_gaps.sql | 50 +++++++- models/silver/core/silver__logs2.sql | 10 +- models/silver/core/silver__traces2.sql | 3 +- models/silver/core/silver__transactions2.sql | 9 +- .../tests/blocks/test_silver__blocks_full.sql | 9 ++ .../tests/blocks/test_silver__blocks_full.yml | 95 ++++++++++++++ .../blocks/test_silver__blocks_recent.sql | 27 ++++ .../blocks/test_silver__blocks_recent.yml | 27 ++++ .../event_logs/test_silver__logs_full.sql | 9 ++ .../event_logs/test_silver__logs_full.yml | 77 ++++++++++++ .../event_logs/test_silver__logs_recent.sql | 27 ++++ .../event_logs/test_silver__logs_recent.yml | 33 +++++ .../receipts/test_silver__receipts_full.sql | 9 ++ .../receipts/test_silver__receipts_full.yml | 82 ++++++++++++ .../receipts/test_silver__receipts_recent.sql | 27 ++++ .../receipts/test_silver__receipts_recent.yml | 28 +++++ .../tests/traces/test_silver__traces_full.sql | 9 ++ .../tests/traces/test_silver__traces_full.yml | 59 +++++++++ .../traces/test_silver__traces_recent.sql | 27 ++++ .../traces/test_silver__traces_recent.yml | 35 ++++++ .../test_silver__transactions_full.sql | 9 ++ .../test_silver__transactions_full.yml | 119 ++++++++++++++++++ .../test_silver__transactions_recent.sql | 27 ++++ .../test_silver__transactions_recent.yml | 22 ++++ .../tests/test_silver__decoded_logs_full.sql | 9 ++ .../test_silver__decoded_logs_full.yml} | 2 +- .../test_silver__decoded_logs_recent.sql | 27 ++++ .../test_silver__decoded_logs_recent.yml | 52 ++++++++ tests/bsc/test_silver__missing_receipts.sql | 2 + tests/bsc/test_silver__missing_traces.sql | 2 + tests/bsc/test_silver__missing_txs.sql | 2 + .../test_silver__recent_missing_receipts.sql | 2 + .../test_silver__recent_missing_traces.sql | 2 + tests/bsc/test_silver__recent_missing_txs.sql | 2 + .../bsc/test_silver__transactions_tx_gap.sql | 2 - 40 files changed, 1030 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/dbt_test_intraday.yml create mode 100644 macros/tests/missing_txs.sql create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_full.sql create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_full.yml create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_recent.sql create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_recent.yml create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_full.sql create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_full.yml create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_recent.sql create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_recent.yml create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_full.sql create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_full.yml create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_recent.sql create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_recent.yml create mode 100644 models/silver/core/tests/traces/test_silver__traces_full.sql create mode 100644 models/silver/core/tests/traces/test_silver__traces_full.yml create mode 100644 models/silver/core/tests/traces/test_silver__traces_recent.sql create mode 100644 models/silver/core/tests/traces/test_silver__traces_recent.yml create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_full.sql create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_full.yml create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_recent.sql create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_recent.yml create mode 100644 models/silver/tests/test_silver__decoded_logs_full.sql rename models/silver/{silver__decoded_logs.yml => tests/test_silver__decoded_logs_full.yml} (96%) create mode 100644 models/silver/tests/test_silver__decoded_logs_recent.sql create mode 100644 models/silver/tests/test_silver__decoded_logs_recent.yml create mode 100644 tests/bsc/test_silver__missing_receipts.sql create mode 100644 tests/bsc/test_silver__missing_traces.sql create mode 100644 tests/bsc/test_silver__missing_txs.sql create mode 100644 tests/bsc/test_silver__recent_missing_receipts.sql create mode 100644 tests/bsc/test_silver__recent_missing_traces.sql create mode 100644 tests/bsc/test_silver__recent_missing_txs.sql delete mode 100644 tests/bsc/test_silver__transactions_tx_gap.sql diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml index c065762..a025dad 100644 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -4,8 +4,8 @@ run-name: dbt_run_streamline_realtime on: workflow_dispatch: schedule: - # Runs "every 2 hours" (see https://crontab.guru) - - cron: '0 1-23/2 * * *' + # Runs “At every 30th minute.” (see https://crontab.guru) + - cron: '*/30 * * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_run_temp_backfill.yml b/.github/workflows/dbt_run_temp_backfill.yml index 45ae1ba..aec2706 100644 --- a/.github/workflows/dbt_run_temp_backfill.yml +++ b/.github/workflows/dbt_run_temp_backfill.yml @@ -4,8 +4,8 @@ run-name: dbt_run_temp_backfill on: workflow_dispatch: schedule: - # Runs "every 2 hours at minute 20" (see https://crontab.guru) - - cron: '20 */2 * * *' + # Runs every "At minute 10.” (see https://crontab.guru) + - cron: '10 * * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index ed2f269..f99941a 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -41,7 +41,7 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt test + dbt test --exclude tag:full_test tag:recent_test diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml new file mode 100644 index 0000000..b977062 --- /dev/null +++ b/.github/workflows/dbt_test_intraday.yml @@ -0,0 +1,47 @@ +name: dbt_test_intraday +run-name: dbt_test_intraday + +on: + workflow_dispatch: + schedule: + # Runs “At minute 15 past every 4th hour.” (see https://crontab.guru) + - cron: '15 */4 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt test -m tag:recent_test + + + diff --git a/macros/tests/missing_txs.sql b/macros/tests/missing_txs.sql new file mode 100644 index 0000000..8abc462 --- /dev/null +++ b/macros/tests/missing_txs.sql @@ -0,0 +1,63 @@ +{% macro missing_txs( + model + ) %} + WITH txs_base AS ( + SELECT + block_number AS base_block_number, + tx_hash AS base_tx_hash + FROM + {{ ref('test_silver__transactions_full') }} + ), + model_name AS ( + SELECT + block_number AS model_block_number, + tx_hash AS model_tx_hash + FROM + {{ model }} + ) +SELECT + base_block_number, + base_tx_hash, + model_block_number, + model_tx_hash +FROM + txs_base + LEFT JOIN model_name + ON base_block_number = model_block_number + AND base_tx_hash = model_tx_hash +WHERE + model_tx_hash IS NULL + OR model_block_number IS NULL +{% endmacro %} + +{% macro recent_missing_txs( + model + ) %} + WITH txs_base AS ( + SELECT + block_number AS base_block_number, + tx_hash AS base_tx_hash + FROM + {{ ref('test_silver__transactions_recent') }} + ), + model_name AS ( + SELECT + block_number AS model_block_number, + tx_hash AS model_tx_hash + FROM + {{ model }} + ) +SELECT + base_block_number, + base_tx_hash, + model_block_number, + model_tx_hash +FROM + txs_base + LEFT JOIN model_name + ON base_block_number = model_block_number + AND base_tx_hash = model_tx_hash +WHERE + model_tx_hash IS NULL + OR model_block_number IS NULL +{% endmacro %} diff --git a/macros/tests/tx_gaps.sql b/macros/tests/tx_gaps.sql index a9d93df..ed52562 100644 --- a/macros/tests/tx_gaps.sql +++ b/macros/tests/tx_gaps.sql @@ -6,7 +6,7 @@ block_number, tx_count FROM - {{ ref('silver__blocks') }} + {{ ref('test_silver__blocks_full') }} ), model_name AS ( SELECT @@ -29,5 +29,51 @@ FROM LEFT JOIN model_name ON block_base.block_number = model_name.block_number WHERE - tx_count <> model_tx_count + ( + tx_count <> model_tx_count + ) + OR ( + model_tx_count IS NULL + AND tx_count <> 0 + ) +{% endmacro %} + +{% macro recent_tx_gaps( + model + ) %} + WITH block_base AS ( + SELECT + block_number, + tx_count + FROM + {{ ref('test_silver__blocks_recent') }} + ), + model_name AS ( + SELECT + block_number, + COUNT( + DISTINCT tx_hash + ) AS model_tx_count + FROM + {{ model }} + GROUP BY + block_number + ) +SELECT + block_base.block_number, + tx_count, + model_name.block_number AS model_block_number, + model_tx_count +FROM + block_base + LEFT JOIN model_name + ON block_base.block_number = model_name.block_number +WHERE + ( + tx_count <> model_tx_count + ) + OR ( + model_tx_count IS NULL + AND tx_count <> 0 + ) {% endmacro %} diff --git a/models/silver/core/silver__logs2.sql b/models/silver/core/silver__logs2.sql index 1c3a10e..3ac42dc 100644 --- a/models/silver/core/silver__logs2.sql +++ b/models/silver/core/silver__logs2.sql @@ -83,14 +83,12 @@ new_records AS ( FROM flat_logs l LEFT OUTER JOIN {{ ref('silver__transactions2') }} - txs USING ( - block_number, - tx_hash - ) + txs + ON l.block_number = txs.block_number + AND l.tx_hash = txs.tx_hash {% if is_incremental() %} -WHERE - txs._INSERTED_TIMESTAMP >= '{{ lookback() }}' +AND txs._INSERTED_TIMESTAMP >= '{{ lookback() }}' {% endif %} ) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index fb8df19..5c7efdf 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -257,8 +257,7 @@ flattened_traces AS ( AND f.block_number = t.block_number {% if is_incremental() %} -WHERE - t._INSERTED_TIMESTAMP >= '{{ lookback() }}' +AND t._INSERTED_TIMESTAMP >= '{{ lookback() }}' {% endif %} ) diff --git a/models/silver/core/silver__transactions2.sql b/models/silver/core/silver__transactions2.sql index 8f4bbc2..accf7fd 100644 --- a/models/silver/core/silver__transactions2.sql +++ b/models/silver/core/silver__transactions2.sql @@ -103,14 +103,13 @@ new_records AS ( r ON A.block_number = r.block_number AND A.data :hash :: STRING = r.tx_hash - LEFT OUTER JOIN {{ ref('silver__blocks2') }} - b - ON A.block_number = b.block_number {% if is_incremental() %} -WHERE - r._INSERTED_TIMESTAMP >= '{{ lookback() }}' +AND r._INSERTED_TIMESTAMP >= '{{ lookback() }}' {% endif %} +LEFT OUTER JOIN {{ ref('silver__blocks2') }} +b +ON A.block_number = b.block_number ) {% if is_incremental() %}, diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.sql b/models/silver/core/tests/blocks/test_silver__blocks_full.sql new file mode 100644 index 0000000..e91850e --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__blocks2') }} diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.yml b/models/silver/core/tests/blocks/test_silver__blocks_full.yml new file mode 100644 index 0000000..acdee83 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.yml @@ -0,0 +1,95 @@ +version: 2 +models: + - name: test_silver__blocks_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TOTAL_DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EXTRA_DATA + tests: + - not_null + - name: GAS_LIMIT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: PARENT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: MINER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: RECEIPTS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SHA3_UNCLES + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SIZE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.sql b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql new file mode 100644 index 0000000..67e2515 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__blocks2') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.yml b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml new file mode 100644 index 0000000..a13423a --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml @@ -0,0 +1,27 @@ +version: 2 +models: + - name: test_silver__blocks_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">10" + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + \ No newline at end of file diff --git a/models/silver/core/tests/event_logs/test_silver__logs_full.sql b/models/silver/core/tests/event_logs/test_silver__logs_full.sql new file mode 100644 index 0000000..a517959 --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__logs2') }} diff --git a/models/silver/core/tests/event_logs/test_silver__logs_full.yml b/models/silver/core/tests/event_logs/test_silver__logs_full.yml new file mode 100644 index 0000000..0108182 --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_full.yml @@ -0,0 +1,77 @@ +version: 2 +models: + - name: test_silver__logs_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - tx_block_count: + config: + severity: error + error_if: "!=0" + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TOPICS + tests: + - not_null + - name: DATA + tests: + - not_null + - name: EVENT_REMOVED + tests: + - not_null + - name: _LOG_ID + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null: + where: NOT IS_PENDING + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ \ No newline at end of file diff --git a/models/silver/core/tests/event_logs/test_silver__logs_recent.sql b/models/silver/core/tests/event_logs/test_silver__logs_recent.sql new file mode 100644 index 0000000..b775518 --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__logs2') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/event_logs/test_silver__logs_recent.yml b/models/silver/core/tests/event_logs/test_silver__logs_recent.yml new file mode 100644 index 0000000..e43492a --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_recent.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: test_silver__logs_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null diff --git a/models/silver/core/tests/receipts/test_silver__receipts_full.sql b/models/silver/core/tests/receipts/test_silver__receipts_full.sql new file mode 100644 index 0000000..56f39ae --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__receipts') }} diff --git a/models/silver/core/tests/receipts/test_silver__receipts_full.yml b/models/silver/core/tests/receipts/test_silver__receipts_full.yml new file mode 100644 index 0000000..22ca7cc --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_full.yml @@ -0,0 +1,82 @@ +version: 2 +models: + - name: test_silver__receipts_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: CUMULATIVE_GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_STATUS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['SUCCESS', 'FAIL'] + - name: TYPE + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + + diff --git a/models/silver/core/tests/receipts/test_silver__receipts_recent.sql b/models/silver/core/tests/receipts/test_silver__receipts_recent.sql new file mode 100644 index 0000000..0b5a901 --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__receipts') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/receipts/test_silver__receipts_recent.yml b/models/silver/core/tests/receipts/test_silver__receipts_recent.yml new file mode 100644 index 0000000..828793a --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_recent.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: test_silver__receipts_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + + diff --git a/models/silver/core/tests/traces/test_silver__traces_full.sql b/models/silver/core/tests/traces/test_silver__traces_full.sql new file mode 100644 index 0000000..eaf9db0 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__traces2') }} diff --git a/models/silver/core/tests/traces/test_silver__traces_full.yml b/models/silver/core/tests/traces/test_silver__traces_full.yml new file mode 100644 index 0000000..f3209ec --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_full.yml @@ -0,0 +1,59 @@ +version: 2 +models: + - name: test_silver__traces_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - TX_POSITION + - TRACE_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: IDENTIFIER + tests: + - not_null + - name: BNB_VALUE + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + + diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.sql b/models/silver/core/tests/traces/test_silver__traces_recent.sql new file mode 100644 index 0000000..acc55ec --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__traces2') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.yml b/models/silver/core/tests/traces/test_silver__traces_recent.yml new file mode 100644 index 0000000..51db5b0 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_recent.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: test_silver__traces_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - TX_POSITION + - TRACE_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + \ No newline at end of file diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.sql b/models/silver/core/tests/transactions/test_silver__transactions_full.sql new file mode 100644 index 0000000..636f963 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__transactions2') }} diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.yml b/models/silver/core/tests/transactions/test_silver__transactions_full.yml new file mode 100644 index 0000000..da4acd6 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.yml @@ -0,0 +1,119 @@ +version: 2 +models: + - name: test_silver__transactions_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: VALUE + tests: + - not_null + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS + tests: + - not_null + - name: INPUT_DATA + tests: + - not_null + - name: TX_STATUS + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['SUCCESS', 'FAIL'] + where: NOT IS_PENDING + - name: GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CUMULATIVE_GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_FEE + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null: + where: NOT IS_PENDING + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + + diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.sql b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql new file mode 100644 index 0000000..1f75159 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__transactions2') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.yml b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml new file mode 100644 index 0000000..aa2881b --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml @@ -0,0 +1,22 @@ +version: 2 +models: + - name: test_silver__transactions_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + \ No newline at end of file diff --git a/models/silver/tests/test_silver__decoded_logs_full.sql b/models/silver/tests/test_silver__decoded_logs_full.sql new file mode 100644 index 0000000..1dcc890 --- /dev/null +++ b/models/silver/tests/test_silver__decoded_logs_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__decoded_logs') }} diff --git a/models/silver/silver__decoded_logs.yml b/models/silver/tests/test_silver__decoded_logs_full.yml similarity index 96% rename from models/silver/silver__decoded_logs.yml rename to models/silver/tests/test_silver__decoded_logs_full.yml index 19e9c1c..436c8e7 100644 --- a/models/silver/silver__decoded_logs.yml +++ b/models/silver/tests/test_silver__decoded_logs_full.yml @@ -1,6 +1,6 @@ version: 2 models: - - name: silver__decoded_logs + - name: test_silver__decoded_logs_full tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: diff --git a/models/silver/tests/test_silver__decoded_logs_recent.sql b/models/silver/tests/test_silver__decoded_logs_recent.sql new file mode 100644 index 0000000..ff24729 --- /dev/null +++ b/models/silver/tests/test_silver__decoded_logs_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__decoded_logs') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/tests/test_silver__decoded_logs_recent.yml b/models/silver/tests/test_silver__decoded_logs_recent.yml new file mode 100644 index 0000000..a72e60b --- /dev/null +++ b/models/silver/tests/test_silver__decoded_logs_recent.yml @@ -0,0 +1,52 @@ +version: 2 +models: + - name: test_silver__decoded_logs_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - dbt_utils.recency: + datepart: day + field: _INSERTED_TIMESTAMP + interval: 1 + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: EVENT_NAME + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + + + + + diff --git a/tests/bsc/test_silver__missing_receipts.sql b/tests/bsc/test_silver__missing_receipts.sql new file mode 100644 index 0000000..8481599 --- /dev/null +++ b/tests/bsc/test_silver__missing_receipts.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_full') }} +{{ missing_txs(ref("test_silver__receipts_full")) }} diff --git a/tests/bsc/test_silver__missing_traces.sql b/tests/bsc/test_silver__missing_traces.sql new file mode 100644 index 0000000..c96256f --- /dev/null +++ b/tests/bsc/test_silver__missing_traces.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_full') }} +{{ missing_txs(ref("test_silver__traces_full")) }} diff --git a/tests/bsc/test_silver__missing_txs.sql b/tests/bsc/test_silver__missing_txs.sql new file mode 100644 index 0000000..499466f --- /dev/null +++ b/tests/bsc/test_silver__missing_txs.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__blocks_full') }} +{{ tx_gaps(ref("test_silver__transactions_full")) }} diff --git a/tests/bsc/test_silver__recent_missing_receipts.sql b/tests/bsc/test_silver__recent_missing_receipts.sql new file mode 100644 index 0000000..f657fd3 --- /dev/null +++ b/tests/bsc/test_silver__recent_missing_receipts.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_recent') }} +{{ recent_missing_txs(ref("test_silver__receipts_recent")) }} diff --git a/tests/bsc/test_silver__recent_missing_traces.sql b/tests/bsc/test_silver__recent_missing_traces.sql new file mode 100644 index 0000000..26a69d4 --- /dev/null +++ b/tests/bsc/test_silver__recent_missing_traces.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_recent') }} +{{ recent_missing_txs(ref("test_silver__traces_recent")) }} diff --git a/tests/bsc/test_silver__recent_missing_txs.sql b/tests/bsc/test_silver__recent_missing_txs.sql new file mode 100644 index 0000000..3d02c92 --- /dev/null +++ b/tests/bsc/test_silver__recent_missing_txs.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__blocks_recent') }} +{{ recent_tx_gaps(ref("test_silver__transactions_recent")) }} diff --git a/tests/bsc/test_silver__transactions_tx_gap.sql b/tests/bsc/test_silver__transactions_tx_gap.sql deleted file mode 100644 index 7f84b8f..0000000 --- a/tests/bsc/test_silver__transactions_tx_gap.sql +++ /dev/null @@ -1,2 +0,0 @@ --- depends_on: {{ ref('silver__blocks') }} -{{ tx_gaps(ref("silver__transactions")) }}