diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index cee4fb3..5fccabe 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -44,7 +44,7 @@ jobs: - name: Run DBT Jobs run: | - dbt test -s models/gold tag:observability --vars '{"TEST_RANGE": True, "start_height": ${{ vars.STREAMLINE_START_BLOCK }}, "end_height": 100000000}' + dbt test -s models/gold tag:observability --vars '{"TEST_RANGE": True, "START_HEIGHT": ${{ vars.STREAMLINE_START_BLOCK }}, "END_HEIGHT": 100000000}' continue-on-error: true - name: Log test results diff --git a/models/descriptions/pending_result_response.md b/models/descriptions/pending_result_response.md new file mode 100644 index 0000000..661948e --- /dev/null +++ b/models/descriptions/pending_result_response.md @@ -0,0 +1,5 @@ +{% docs pending_result_response %} + +Internal column tracking if the events response has been received yet, to re-sync the tx_id and append events. + +{% enddocs %} diff --git a/models/silver/core/silver__streamline_transactions_final.yml b/models/silver/core/silver__streamline_transactions_final.yml index 5fa0672..d6c6677 100644 --- a/models/silver/core/silver__streamline_transactions_final.yml +++ b/models/silver/core/silver__streamline_transactions_final.yml @@ -21,6 +21,19 @@ models: - not_null - unique + - name: pending_result_response + description: "{{ doc('pending_result_response') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: BOOLEAN + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: [false] + row_condition: "date_trunc('day', block_timestamp) <= SYSDATE() - interval '1 day' AND block_height >= {{ var('STREAMLINE_START_BLOCK' )}}" + config: + severity: error + error_if: ">50" + - name: block_timestamp description: "{{ doc('block_timestamp') }}" tests: diff --git a/tests/tests__events_final_gap.sql b/tests/tests__events_final_gap.sql new file mode 100644 index 0000000..48f6324 --- /dev/null +++ b/tests/tests__events_final_gap.sql @@ -0,0 +1,51 @@ +{{ config( + severity = 'error', + tags = ['streamline_test'] +) }} + +WITH txs AS ( + + SELECT + block_height, + tx_id, + ARRAY_SIZE(events) AS event_ct + FROM + {{ ref('silver__streamline_transactions_final') }} + WHERE + block_height >= {{ var('STREAMLINE_START_BLOCK') }} + AND (_inserted_timestamp BETWEEN SYSDATE() - INTERVAL '3 days' + AND SYSDATE() - INTERVAL '2 hours') +), +events AS ( + SELECT + block_height, + tx_id, + COUNT(1) AS event_ct + FROM + {{ ref('silver__streamline_events') }} + WHERE + block_height >= {{ var('STREAMLINE_START_BLOCK') }} + AND(_inserted_timestamp BETWEEN SYSDATE() - INTERVAL '3 days' + AND SYSDATE() - INTERVAL '2 hours') + GROUP BY + 1, + 2 +), +compare AS ( + SELECT + txs.block_height, + txs.tx_id, + txs.event_ct AS tx_event_ct, + events.event_ct AS event_event_ct + FROM + txs + LEFT JOIN events + ON txs.block_height = events.block_height + AND txs.tx_id = events.tx_id + WHERE + txs.event_ct != events.event_ct + ) +SELECT + * +FROM + compare diff --git a/tests/tests__streamline_blocks_gap.sql b/tests/tests__streamline_blocks_gap.sql index 4225eff..4be56f2 100644 --- a/tests/tests__streamline_blocks_gap.sql +++ b/tests/tests__streamline_blocks_gap.sql @@ -22,9 +22,9 @@ WITH streamline_blocks AS ( determine_prior_block AS ( SELECT block_height, - block_id, + id, parent_id, - LAG(block_id) over ( + LAG(id) over ( ORDER BY block_height ) AS prev_block_id, diff --git a/tests/tests__streamline_transaction_results_gap.sql b/tests/tests__streamline_transaction_results_gap.sql index 46e455f..0715256 100644 --- a/tests/tests__streamline_transaction_results_gap.sql +++ b/tests/tests__streamline_transaction_results_gap.sql @@ -1,6 +1,7 @@ {{ config( severity = 'error', - tags = ['streamline_test'] + tags = ['streamline_test'], + error_if= ">=100" ) }} WITH results_expected AS (