From 45f72e71d857116da129001e7fc0e19c4cceb781 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 23 Aug 2023 13:37:16 -0600 Subject: [PATCH] An 3660/observability models (#155) * blocks completeness * block tx count external api model * txs completeness model * add col alias * test result agg cte and temp test range * curr range test * revert source change - other pr * xchain silver * nuke api key & move to secrets, create workflow * add scheduled tag * update workflow w cache * add obs tag * rem testing range from txs complete * comment out schedule and del double comman * add default null for key var --- .github/workflows/dbt_get_block_tx_count.yml | 52 +++++ .github/workflows/dbt_run_hourly.yml | 2 +- dbt_project.yml | 3 +- models/bronze/bronze__blocks.sql | 3 +- models/bronze/bronze__labels.sql | 3 +- models/bronze/bronze__moments_metadata.sql | 3 +- models/bronze/bronze__prices_v2.sql | 7 +- models/bronze/bronze__transactions.sql | 3 +- models/gold/core__dim_allday_metadata.sql | 3 +- models/gold/core__dim_contract_labels.sql | 3 +- models/gold/core__dim_moment_metadata.sql | 2 +- models/gold/core__dim_swap_pool_labels.sql | 3 +- models/gold/core__dim_topshot_metadata.sql | 3 +- models/gold/core__dim_validator_labels.sql | 3 +- models/gold/core__ez_bridge_transactions.sql | 2 +- models/gold/core__ez_nft_sales.sql | 2 +- models/gold/core__ez_staking_actions.sql | 2 +- models/gold/core__ez_swaps.sql | 2 +- models/gold/core__ez_token_transfers.sql | 2 +- models/gold/core__fact_blocks.sql | 3 +- models/gold/core__fact_events.sql | 3 +- models/gold/core__fact_hourly_prices.sql | 3 +- models/gold/core__fact_prices.sql | 3 +- models/gold/core__fact_transactions.sql | 3 +- .../silver_observability__block_tx_count.sql | 66 +++++++ ...ver_observability__blocks_completeness.sql | 187 ++++++++++++++++++ ...silver_observability__txs_completeness.sql | 152 ++++++++++++++ models/silver/base/silver__blocks.sql | 3 +- .../silver/base/silver__event_attributes.sql | 3 +- .../base/silver__event_attributes_https.sql | 3 +- models/silver/base/silver__events.sql | 3 +- models/silver/base/silver__events_final.sql | 3 +- models/silver/base/silver__transactions.sql | 3 +- .../silver/labels/silver__contract_labels.sql | 3 +- models/silver/labels/silver__labels.sql | 1 + models/silver/labels/silver__labels_pools.sql | 3 +- .../labels/silver__labels_pools_metapier.sql | 3 +- .../labels/silver__validator_labels.sql | 3 +- .../silver__allday_moments_metadata_error.sql | 3 +- .../nft/silver__nft_allday_metadata.sql | 3 +- models/silver/nft/silver__nft_giglabs.sql | 2 +- .../nft/silver__nft_moment_editions.sql | 2 +- .../nft/silver__nft_moment_metadata.sql | 2 +- .../nft/silver__nft_moment_metadata_final.sql | 2 +- .../silver/nft/silver__nft_moment_minted.sql | 2 +- .../nft/silver__nft_moment_minted_2.sql | 2 +- .../silver/nft/silver__nft_moment_series.sql | 2 +- models/silver/nft/silver__nft_moment_set.sql | 2 +- models/silver/nft/silver__nft_moments.sql | 2 +- models/silver/nft/silver__nft_sales.sql | 2 +- .../nft/silver__nft_topshot_metadata.sql | 3 +- .../nft/silver__nft_topshot_pack_sales.sql | 2 +- .../silver/nft/silver__nft_topshot_sales.sql | 2 +- ...ver__nft_transactions_secondary_market.sql | 2 +- models/silver/prices/silver__prices.sql | 3 +- .../silver/prices/silver__prices_hourly.sql | 3 +- models/silver/prices/silver__prices_swaps.sql | 3 +- .../prices/silver__prices_swaps_hourly.sql | 3 +- .../staking/silver__staking_actions.sql | 3 +- models/silver/swaps/silver__swaps.sql | 3 +- models/silver/swaps/silver__swaps_events.sql | 3 +- .../swaps/silver__swaps_single_trade.sql | 3 +- .../transfers/silver__bridge_blocto.sql | 2 +- .../silver/transfers/silver__bridge_celer.sql | 2 +- .../transfers/silver__token_transfers.sql | 3 +- models/sources.yml | 4 +- 66 files changed, 560 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/dbt_get_block_tx_count.yml create mode 100644 models/silver/_observability/silver_observability__block_tx_count.sql create mode 100644 models/silver/_observability/silver_observability__blocks_completeness.sql create mode 100644 models/silver/_observability/silver_observability__txs_completeness.sql diff --git a/.github/workflows/dbt_get_block_tx_count.yml b/.github/workflows/dbt_get_block_tx_count.yml new file mode 100644 index 0000000..e0bbadd --- /dev/null +++ b/.github/workflows/dbt_get_block_tx_count.yml @@ -0,0 +1,52 @@ +name: dbt_get_block_tx_count +run-name: dbt_get_block_tx_count + +on: + workflow_dispatch: + # schedule: + # Runs "every minute" (see https://crontab.guru) + # TODO enable after manual test + # - cron: '* * * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + BITQUERY_API_KEY: "${{ secrets.BITQUERY_API_KEY }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -s tag:get_block_tx_count --vars 'BITQUERY_API_KEY: ${{ secrets.BITQUERY_API_KEY }}' + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_hourly.yml b/.github/workflows/dbt_run_hourly.yml index 2fd8f38..46f4d4d 100644 --- a/.github/workflows/dbt_run_hourly.yml +++ b/.github/workflows/dbt_run_hourly.yml @@ -30,7 +30,7 @@ jobs: dbt_command: > dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; dbt seed; - dbt run --exclude streamline__all_topshot_moments_minted_metadata_needed streamline__allday_moments_metadata_needed models/silver/streamline; + dbt run -s tag:scheduled environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} secrets: inherit diff --git a/dbt_project.yml b/dbt_project.yml index 9c806f9..02d1744 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -56,5 +56,6 @@ vars: "dbt_date:time_zone": GMT UPDATE_SNOWFLAKE_TAGS: True UPDATE_UDFS_AND_SPS: False + OBSERV_FULL_TEST: False STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False - STREAMLINE_INVOKE_STREAMS: True + STREAMLINE_INVOKE_STREAMS: False diff --git a/models/bronze/bronze__blocks.sql b/models/bronze/bronze__blocks.sql index 9e5090b..f5aa142 100644 --- a/models/bronze/bronze__blocks.sql +++ b/models/bronze/bronze__blocks.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} SELECT diff --git a/models/bronze/bronze__labels.sql b/models/bronze/bronze__labels.sql index def4be7..820ead5 100644 --- a/models/bronze/bronze__labels.sql +++ b/models/bronze/bronze__labels.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} WITH labels AS ( diff --git a/models/bronze/bronze__moments_metadata.sql b/models/bronze/bronze__moments_metadata.sql index d00229e..89a9bc3 100644 --- a/models/bronze/bronze__moments_metadata.sql +++ b/models/bronze/bronze__moments_metadata.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} SELECT diff --git a/models/bronze/bronze__prices_v2.sql b/models/bronze/bronze__prices_v2.sql index c2917d8..a6cb781 100644 --- a/models/bronze/bronze__prices_v2.sql +++ b/models/bronze/bronze__prices_v2.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} WITH coingecko AS ( @@ -15,7 +16,7 @@ WITH coingecko AS ( _inserted_timestamp FROM {{ source( - 'crosschain_v2', + 'crosschain_silver', 'hourly_prices_coin_gecko' ) }} ), @@ -31,7 +32,7 @@ coinmarketcap AS ( _inserted_timestamp FROM {{ source( - 'crosschain_v2', + 'crosschain_silver', 'hourly_prices_coin_market_cap' ) }} ), diff --git a/models/bronze/bronze__transactions.sql b/models/bronze/bronze__transactions.sql index 274de43..495e032 100644 --- a/models/bronze/bronze__transactions.sql +++ b/models/bronze/bronze__transactions.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} SELECT diff --git a/models/gold/core__dim_allday_metadata.sql b/models/gold/core__dim_allday_metadata.sql index b84a36a..fcfc157 100644 --- a/models/gold/core__dim_allday_metadata.sql +++ b/models/gold/core__dim_allday_metadata.sql @@ -6,7 +6,8 @@ 'PURPOSE': 'NFT, ALLDAY' } } - } + }, + tags = ['scheduled'] ) }} WITH allday AS ( diff --git a/models/gold/core__dim_contract_labels.sql b/models/gold/core__dim_contract_labels.sql index d191a49..d1bfdbf 100644 --- a/models/gold/core__dim_contract_labels.sql +++ b/models/gold/core__dim_contract_labels.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} WITH contract_labels AS ( diff --git a/models/gold/core__dim_moment_metadata.sql b/models/gold/core__dim_moment_metadata.sql index 0937f24..d1620f5 100644 --- a/models/gold/core__dim_moment_metadata.sql +++ b/models/gold/core__dim_moment_metadata.sql @@ -1,6 +1,6 @@ {{ config ( materialized = 'view', - tags = ['nft', 'dapper'], + tags = ['nft', 'dapper', 'scheduled'], meta = { 'database_tags':{ 'table': { diff --git a/models/gold/core__dim_swap_pool_labels.sql b/models/gold/core__dim_swap_pool_labels.sql index 0b034d7..8a5f75f 100644 --- a/models/gold/core__dim_swap_pool_labels.sql +++ b/models/gold/core__dim_swap_pool_labels.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} WITH pairs AS ( diff --git a/models/gold/core__dim_topshot_metadata.sql b/models/gold/core__dim_topshot_metadata.sql index 4cb24d7..545bad4 100644 --- a/models/gold/core__dim_topshot_metadata.sql +++ b/models/gold/core__dim_topshot_metadata.sql @@ -6,7 +6,8 @@ 'PURPOSE': 'NFT, TOPSHOT' } } - } + }, + tags = ['scheduled'] ) }} WITH topshot AS ( diff --git a/models/gold/core__dim_validator_labels.sql b/models/gold/core__dim_validator_labels.sql index 8d8c4ee..9883470 100644 --- a/models/gold/core__dim_validator_labels.sql +++ b/models/gold/core__dim_validator_labels.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} WITH validators AS ( diff --git a/models/gold/core__ez_bridge_transactions.sql b/models/gold/core__ez_bridge_transactions.sql index ec63135..3bdeb0c 100644 --- a/models/gold/core__ez_bridge_transactions.sql +++ b/models/gold/core__ez_bridge_transactions.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - tags = ['ez', 'bridge'], + tags = ['ez', 'bridge', 'scheduled'], meta={ 'database_tags':{ 'table': { diff --git a/models/gold/core__ez_nft_sales.sql b/models/gold/core__ez_nft_sales.sql index 1f3f5d3..31d557c 100644 --- a/models/gold/core__ez_nft_sales.sql +++ b/models/gold/core__ez_nft_sales.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - tags = ['nft', 'ez'], + tags = ['nft', 'ez', 'scheduled'], meta={ 'database_tags':{ 'table': { diff --git a/models/gold/core__ez_staking_actions.sql b/models/gold/core__ez_staking_actions.sql index 24a77b6..342ead0 100644 --- a/models/gold/core__ez_staking_actions.sql +++ b/models/gold/core__ez_staking_actions.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - tags = ['ez'], + tags = ['ez', 'scheduled'], meta={ 'database_tags':{ 'table': { diff --git a/models/gold/core__ez_swaps.sql b/models/gold/core__ez_swaps.sql index 34e0317..b01fae7 100644 --- a/models/gold/core__ez_swaps.sql +++ b/models/gold/core__ez_swaps.sql @@ -1,6 +1,6 @@ {{ config ( materialized = 'view', - tags = ['ez'], + tags = ['ez', 'scheduled'], meta={ 'database_tags':{ 'table': { diff --git a/models/gold/core__ez_token_transfers.sql b/models/gold/core__ez_token_transfers.sql index 85b7130..c0b7ae1 100644 --- a/models/gold/core__ez_token_transfers.sql +++ b/models/gold/core__ez_token_transfers.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - tags = ['ez'] + tags = ['ez', 'scheduled'] ) }} SELECT diff --git a/models/gold/core__fact_blocks.sql b/models/gold/core__fact_blocks.sql index d398e90..0615e83 100644 --- a/models/gold/core__fact_blocks.sql +++ b/models/gold/core__fact_blocks.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} WITH silver_blocks AS ( diff --git a/models/gold/core__fact_events.sql b/models/gold/core__fact_events.sql index 4261029..5068dc5 100644 --- a/models/gold/core__fact_events.sql +++ b/models/gold/core__fact_events.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} WITH events_final AS ( diff --git a/models/gold/core__fact_hourly_prices.sql b/models/gold/core__fact_hourly_prices.sql index a438750..844efe3 100644 --- a/models/gold/core__fact_hourly_prices.sql +++ b/models/gold/core__fact_hourly_prices.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} WITH api AS ( diff --git a/models/gold/core__fact_prices.sql b/models/gold/core__fact_prices.sql index 34d85a2..e8b397c 100644 --- a/models/gold/core__fact_prices.sql +++ b/models/gold/core__fact_prices.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} WITH token_labels AS ( diff --git a/models/gold/core__fact_transactions.sql b/models/gold/core__fact_transactions.sql index d85c64f..908bf71 100644 --- a/models/gold/core__fact_transactions.sql +++ b/models/gold/core__fact_transactions.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} WITH silver_txs AS ( diff --git a/models/silver/_observability/silver_observability__block_tx_count.sql b/models/silver/_observability/silver_observability__block_tx_count.sql new file mode 100644 index 0000000..4f33842 --- /dev/null +++ b/models/silver/_observability/silver_observability__block_tx_count.sql @@ -0,0 +1,66 @@ +{{ config( + materialized = 'incremental', + unique_key = 'block_height', + tags = ['observability', 'get_block_tx_count'] +) }} + +WITH starting_block AS ( + +{% if is_incremental() %} + +SELECT + MAX(block_height) AS block_height_start, {{ target.database }}.streamline.udf_get_chainhead() AS max_block_height +FROM + {{ this }} +{% else %} + -- Candidate 4 starts at 4132133 +SELECT + 4132133 AS block_height_start, 1000000000 AS max_block_height +{% endif %}), +params AS ( + SELECT + 'query ($network: FlowNetwork!, $block_height_start: Int!, $block_height_end: Int!) { flow(network: $network) { blocks(height: {gt: $block_height_start, lteq: $block_height_end}) { height transactionsCount } } }' AS query, + OBJECT_CONSTRUCT( + 'network', + 'flow', + 'block_height_start', + block_height_start, + 'block_height_end', + IFF( + block_height_start + 25000 > max_block_height, + max_block_height, + block_height_start + 25000 + ) + ) AS variables, + '{{ var('BITQUERY_API_KEY', Null) }}' AS api_key + FROM + starting_block +), +get_bitquery AS ( + SELECT + livequery_dev.live.udf_api( + 'POST', + 'https://graphql.bitquery.io', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'X-API-KEY', + api_key + ), + OBJECT_CONSTRUCT( + 'query', + query, + 'variables', + variables + ) + ) AS res, + res :data :data :flow :blocks :: ARRAY AS blocks_res + FROM + params +) +SELECT + VALUE :height :: INTEGER AS block_height, + VALUE :transactionsCount :: INTEGER AS transaction_ct +FROM + get_bitquery, + LATERAL FLATTEN(blocks_res) diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql new file mode 100644 index 0000000..5585296 --- /dev/null +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -0,0 +1,187 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = False, + tags = ['observability'] +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_height) AS min_block, + MAX(block_height) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, SYSDATE()) + +{% if is_incremental() %} +AND ( + block_height >= ( + SELECT + MIN(block_height) + FROM + ( + SELECT + MIN(block_height) AS block_height + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) + AND DATEADD('hour', -95, SYSDATE()) + UNION + SELECT + MIN(VALUE) - 1 AS block_height + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_height >= 7601063 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_height + FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +blocks AS ( + SELECT + l.block_height, + block_timestamp, + LAG( + l.block_height, + 1 + ) over ( + ORDER BY + l.block_height ASC + ) AS prev_BLOCK_HEIGHT + FROM + {{ ref("silver__blocks") }} + l + INNER JOIN block_range b + ON l.block_height = b.block_height + AND l.block_height >= ( + SELECT + MIN(block_height) + FROM + block_range + ) +), +block_gen AS ( + SELECT + _id AS block_height + FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + MIN(block_height) + FROM + blocks + ) + AND ( + SELECT + MAX(block_height) + FROM + blocks + ) +), +test_blocks AS ( + SELECT + 'blocks' AS test_name, + MIN( + b.block_height + ) AS min_block, + MAX( + b.block_height + ) AS max_block, + MIN( + b.block_timestamp + ) AS min_block_timestamp, + MAX( + b.block_timestamp + ) AS max_block_timestamp, + COUNT(1) AS blocks_tested, + COUNT( + CASE + WHEN C.block_height IS NOT NULL THEN A.block_height + END + ) AS blocks_impacted_count, + ARRAY_AGG( + CASE + WHEN C.block_height IS NOT NULL THEN A.block_height + END + ) within GROUP ( + ORDER BY + A.block_height + ) AS blocks_impacted_array, + SYSDATE() AS test_timestamp + FROM + block_gen A + LEFT JOIN blocks b + ON A.block_height = b.block_height + LEFT JOIN blocks C + ON A.block_height > C.prev_block_height + AND A.block_height < C.block_height + AND C.block_height - C.prev_block_height <> 1 + WHERE + COALESCE( + b.block_height, + C.block_height + ) IS NOT NULL +), +FINAL AS ( + SELECT + test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + test_timestamp + FROM + test_blocks +) +SELECT + * +FROM + FINAL diff --git a/models/silver/_observability/silver_observability__txs_completeness.sql b/models/silver/_observability/silver_observability__txs_completeness.sql new file mode 100644 index 0000000..4b58697 --- /dev/null +++ b/models/silver/_observability/silver_observability__txs_completeness.sql @@ -0,0 +1,152 @@ +{{ config( + materialized = 'incremental', + unique_key = '_test_timestamp', + tags = ['observability'] +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_height) AS min_block, + MAX(block_height) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, SYSDATE()) + +{% if is_incremental() %} +AND ( + block_height >= ( + SELECT + MIN(block_height) + FROM + ( + SELECT + MIN(block_height) AS block_height + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) + AND DATEADD('hour', -95, SYSDATE()) + UNION + SELECT + MIN(VALUE) - 1 AS block_height + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_height >= 7601063 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_height + FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} + WHERE + block_height BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +txs_per_block_actual AS ( + SELECT + block_height, + COUNT( + DISTINCT tx_id + ) AS txs + FROM + {{ ref('silver__transactions') }} + WHERE + block_height IN ( + SELECT + block_height + FROM + block_range + ) + GROUP BY + 1 +), +txs_per_block_expected AS ( + SELECT + block_height, + transaction_ct AS txs + FROM + {{ ref('silver_observability__block_tx_count') }} + WHERE + block_height IN ( + SELECT + block_height + FROM + block_range + ) +), +comparison AS ( + SELECT + id.block_height, + A.txs AS actual_tx_count, + e.txs AS expected_tx_count + FROM + block_range id + LEFT JOIN txs_per_block_actual A USING (block_height) + LEFT JOIN txs_per_block_expected e USING (block_height) +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_height) within GROUP ( + ORDER BY + block_height + ) AS blocks_impacted_array + FROM + comparison + WHERE + actual_tx_count != expected_tx_count +), +FINAL AS ( + SELECT + 'transactions' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + SYSDATE() AS _test_timestamp + FROM + summary_stats + JOIN impacted_blocks +) +SELECT + * +FROM + FINAL diff --git a/models/silver/base/silver__blocks.sql b/models/silver/base/silver__blocks.sql index fdd0113..aa94ab7 100644 --- a/models/silver/base/silver__blocks.sql +++ b/models/silver/base/silver__blocks.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'block_height', - incremental_strategy = 'delete+insert' + incremental_strategy = 'delete+insert', + tags = ['scheduled'] ) }} WITH bronze_blocks AS ( diff --git a/models/silver/base/silver__event_attributes.sql b/models/silver/base/silver__event_attributes.sql index 84b236a..0d2d489 100644 --- a/models/silver/base/silver__event_attributes.sql +++ b/models/silver/base/silver__event_attributes.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'attribute_id', - incremental_strategy = 'delete+insert' + incremental_strategy = 'delete+insert', + tags = ['scheduled'] ) }} WITH events AS ( diff --git a/models/silver/base/silver__event_attributes_https.sql b/models/silver/base/silver__event_attributes_https.sql index c05101d..92de007 100644 --- a/models/silver/base/silver__event_attributes_https.sql +++ b/models/silver/base/silver__event_attributes_https.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'attribute_id', - incremental_strategy = 'delete+insert' + incremental_strategy = 'delete+insert', + tags = ['scheduled'] ) }} WITH events AS ( diff --git a/models/silver/base/silver__events.sql b/models/silver/base/silver__events.sql index 52a1893..07ad3b2 100644 --- a/models/silver/base/silver__events.sql +++ b/models/silver/base/silver__events.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'event_id', - incremental_strategy = 'delete+insert' + incremental_strategy = 'delete+insert', + tags = ['scheduled'] ) }} WITH transactions AS ( diff --git a/models/silver/base/silver__events_final.sql b/models/silver/base/silver__events_final.sql index b7df8ac..44e933a 100644 --- a/models/silver/base/silver__events_final.sql +++ b/models/silver/base/silver__events_final.sql @@ -2,7 +2,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::date'], - unique_key = "CONCAT_WS('-', tx_id, event_index)" + unique_key = "CONCAT_WS('-', tx_id, event_index)", + tags = ['scheduled'] ) }} WITH silver_events AS ( diff --git a/models/silver/base/silver__transactions.sql b/models/silver/base/silver__transactions.sql index 3c05a18..2285785 100644 --- a/models/silver/base/silver__transactions.sql +++ b/models/silver/base/silver__transactions.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'tx_id', - incremental_strategy = 'delete+insert' + incremental_strategy = 'delete+insert', + tags = ['scheduled'] ) }} WITH bronze_txs AS ( diff --git a/models/silver/labels/silver__contract_labels.sql b/models/silver/labels/silver__contract_labels.sql index fed5cf2..845dca9 100644 --- a/models/silver/labels/silver__contract_labels.sql +++ b/models/silver/labels/silver__contract_labels.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'table', cluster_by = ['event_contract'], - unique_key = 'event_contract' + unique_key = 'event_contract', + tags = ['scheduled'] ) }} WITH splt AS ( diff --git a/models/silver/labels/silver__labels.sql b/models/silver/labels/silver__labels.sql index e886c28..7782ef5 100644 --- a/models/silver/labels/silver__labels.sql +++ b/models/silver/labels/silver__labels.sql @@ -2,6 +2,7 @@ materialized = 'table', cluster_by = ['address'], unique_key = 'event_id', + tags = ['scheduled'] ) }} WITH labels AS ( diff --git a/models/silver/labels/silver__labels_pools.sql b/models/silver/labels/silver__labels_pools.sql index 36a5cc7..bcc5f1b 100644 --- a/models/silver/labels/silver__labels_pools.sql +++ b/models/silver/labels/silver__labels_pools.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'tx_id', - incremental_strategy = 'delete+insert' + incremental_strategy = 'delete+insert', + tags = ['scheduled'] ) }} WITH events AS ( diff --git a/models/silver/labels/silver__labels_pools_metapier.sql b/models/silver/labels/silver__labels_pools_metapier.sql index 5ce229d..ca535cd 100644 --- a/models/silver/labels/silver__labels_pools_metapier.sql +++ b/models/silver/labels/silver__labels_pools_metapier.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'tx_id', - incremental_strategy = 'delete+insert' + incremental_strategy = 'delete+insert', + tags = ['scheduled'] ) }} WITH events AS ( diff --git a/models/silver/labels/silver__validator_labels.sql b/models/silver/labels/silver__validator_labels.sql index 0a986c7..8f0cfc7 100644 --- a/models/silver/labels/silver__validator_labels.sql +++ b/models/silver/labels/silver__validator_labels.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'table' + materialized = 'table', + tags = ['scheduled'] ) }} WITH labels AS ( diff --git a/models/silver/nft/silver__allday_moments_metadata_error.sql b/models/silver/nft/silver__allday_moments_metadata_error.sql index ffadeda..40e0b09 100644 --- a/models/silver/nft/silver__allday_moments_metadata_error.sql +++ b/models/silver/nft/silver__allday_moments_metadata_error.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'view' + materialized = 'view', + tags = ['scheduled'] ) }} SELECT diff --git a/models/silver/nft/silver__nft_allday_metadata.sql b/models/silver/nft/silver__nft_allday_metadata.sql index d4b8099..1b7341e 100644 --- a/models/silver/nft/silver__nft_allday_metadata.sql +++ b/models/silver/nft/silver__nft_allday_metadata.sql @@ -2,7 +2,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'nft_id' + unique_key = 'nft_id', + tags = ['scheduled'] ) }} WITH metadata AS ( diff --git a/models/silver/nft/silver__nft_giglabs.sql b/models/silver/nft/silver__nft_giglabs.sql index f34bcde..72f2bba 100644 --- a/models/silver/nft/silver__nft_giglabs.sql +++ b/models/silver/nft/silver__nft_giglabs.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'tx_id', - tags = ['nft'] + tags = ['nft', 'scheduled'] ) }} WITH events AS ( diff --git a/models/silver/nft/silver__nft_moment_editions.sql b/models/silver/nft/silver__nft_moment_editions.sql index 21f1be5..ddac7cb 100644 --- a/models/silver/nft/silver__nft_moment_editions.sql +++ b/models/silver/nft/silver__nft_moment_editions.sql @@ -3,7 +3,7 @@ cluster_by = ['_inserted_timestamp'], unique_key = "concat_ws('-', event_contract, edition_id)", incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper'] + tags = ['nft', 'dapper', 'scheduled'] ) }} WITH events AS ( diff --git a/models/silver/nft/silver__nft_moment_metadata.sql b/models/silver/nft/silver__nft_moment_metadata.sql index db21e98..14a10b7 100644 --- a/models/silver/nft/silver__nft_moment_metadata.sql +++ b/models/silver/nft/silver__nft_moment_metadata.sql @@ -3,7 +3,7 @@ cluster_by = ['play_id'], unique_key = "concat_ws('-', event_contract, play_id)", incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper'] + tags = ['nft', 'dapper', 'scheduled'] ) }} WITH play_creation AS ( diff --git a/models/silver/nft/silver__nft_moment_metadata_final.sql b/models/silver/nft/silver__nft_moment_metadata_final.sql index 8f849eb..bb08c37 100644 --- a/models/silver/nft/silver__nft_moment_metadata_final.sql +++ b/models/silver/nft/silver__nft_moment_metadata_final.sql @@ -2,7 +2,7 @@ materialized = 'table', cluster_by = ['_inserted_timestamp'], unique_key = "concat_ws('-',event_contract,edition_id,nft_id)", - tags = ['nft', 'dapper'] + tags = ['nft', 'dapper', 'scheduled'] ) }} WITH moments AS ( diff --git a/models/silver/nft/silver__nft_moment_minted.sql b/models/silver/nft/silver__nft_moment_minted.sql index 7434cf4..ca31a55 100644 --- a/models/silver/nft/silver__nft_moment_minted.sql +++ b/models/silver/nft/silver__nft_moment_minted.sql @@ -3,7 +3,7 @@ cluster_by = ['_inserted_timestamp'], unique_key = "concat_ws('-', event_contract, edition_id)", incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper'] + tags = ['nft', 'dapper', 'scheduled'] ) }} WITH events AS ( diff --git a/models/silver/nft/silver__nft_moment_minted_2.sql b/models/silver/nft/silver__nft_moment_minted_2.sql index d579469..ef12823 100644 --- a/models/silver/nft/silver__nft_moment_minted_2.sql +++ b/models/silver/nft/silver__nft_moment_minted_2.sql @@ -3,7 +3,7 @@ cluster_by = ['_inserted_timestamp'], unique_key = "concat_ws('-', event_contract, moment_id)", incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper'] + tags = ['nft', 'dapper', 'scheduled'] ) }} WITH events AS ( diff --git a/models/silver/nft/silver__nft_moment_series.sql b/models/silver/nft/silver__nft_moment_series.sql index aa9d21a..e82001a 100644 --- a/models/silver/nft/silver__nft_moment_series.sql +++ b/models/silver/nft/silver__nft_moment_series.sql @@ -3,7 +3,7 @@ cluster_by = ['_inserted_timestamp'], unique_key = "concat_ws('-', event_contract, series_id)", incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper'] + tags = ['nft', 'dapper', 'scheduled'] ) }} WITH events AS ( diff --git a/models/silver/nft/silver__nft_moment_set.sql b/models/silver/nft/silver__nft_moment_set.sql index 0fa60d8..4c57ab0 100644 --- a/models/silver/nft/silver__nft_moment_set.sql +++ b/models/silver/nft/silver__nft_moment_set.sql @@ -3,7 +3,7 @@ cluster_by = ['_inserted_timestamp'], unique_key = "concat_ws('-', event_contract, set_id)", incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper'] + tags = ['nft', 'dapper', 'scheduled'] ) }} WITH events AS ( diff --git a/models/silver/nft/silver__nft_moments.sql b/models/silver/nft/silver__nft_moments.sql index ce6ccba..cace3bc 100644 --- a/models/silver/nft/silver__nft_moments.sql +++ b/models/silver/nft/silver__nft_moments.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'], unique_key = "CONCAT_WS('-', tx_id, event_index)", - tags = ['nft'] + tags = ['nft', 'scheduled'] ) }} WITH events AS ( diff --git a/models/silver/nft/silver__nft_sales.sql b/models/silver/nft/silver__nft_sales.sql index 3fcd806..19c6797 100644 --- a/models/silver/nft/silver__nft_sales.sql +++ b/models/silver/nft/silver__nft_sales.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'tx_id', - tags = ['nft'] + tags = ['nft', 'scheduled'] ) }} WITH topshot AS ( diff --git a/models/silver/nft/silver__nft_topshot_metadata.sql b/models/silver/nft/silver__nft_topshot_metadata.sql index 7b5653b..56b4682 100644 --- a/models/silver/nft/silver__nft_topshot_metadata.sql +++ b/models/silver/nft/silver__nft_topshot_metadata.sql @@ -2,7 +2,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'nft_id' + unique_key = 'nft_id', + tags = ['scheduled'] ) }} WITH metadata AS ( diff --git a/models/silver/nft/silver__nft_topshot_pack_sales.sql b/models/silver/nft/silver__nft_topshot_pack_sales.sql index c53551c..c19243e 100644 --- a/models/silver/nft/silver__nft_topshot_pack_sales.sql +++ b/models/silver/nft/silver__nft_topshot_pack_sales.sql @@ -3,7 +3,7 @@ incremental_strategy = 'merge', cluster_by = ['_inserted_timestamp::DATE'], unique_key = ['tx_id','nft_id'], - tags = ['nft'] + tags = ['nft', 'scheduled'] ) }} WITH silver_events AS ( diff --git a/models/silver/nft/silver__nft_topshot_sales.sql b/models/silver/nft/silver__nft_topshot_sales.sql index 7e4bc2f..61e41df 100644 --- a/models/silver/nft/silver__nft_topshot_sales.sql +++ b/models/silver/nft/silver__nft_topshot_sales.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'tx_id', - tags = ['nft'] + tags = ['nft', 'scheduled'] ) }} WITH silver_events AS ( diff --git a/models/silver/nft/silver__nft_transactions_secondary_market.sql b/models/silver/nft/silver__nft_transactions_secondary_market.sql index 971d8ba..daabaa7 100644 --- a/models/silver/nft/silver__nft_transactions_secondary_market.sql +++ b/models/silver/nft/silver__nft_transactions_secondary_market.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'tx_id', - tags = ['nft'] + tags = ['nft', 'scheduled'] ) }} WITH silver_events AS ( diff --git a/models/silver/prices/silver__prices.sql b/models/silver/prices/silver__prices.sql index 37f353c..811393a 100644 --- a/models/silver/prices/silver__prices.sql +++ b/models/silver/prices/silver__prices.sql @@ -2,7 +2,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', cluster_by = ['recorded_at::DATE'], - unique_key = "concat_ws( '-', recorded_at, asset_id )" + unique_key = "concat_ws( '-', recorded_at, asset_id )", + tags = ['scheduled'] ) }} WITH token_prices AS ( diff --git a/models/silver/prices/silver__prices_hourly.sql b/models/silver/prices/silver__prices_hourly.sql index 8b1b1d4..2309f11 100644 --- a/models/silver/prices/silver__prices_hourly.sql +++ b/models/silver/prices/silver__prices_hourly.sql @@ -2,7 +2,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::date'], - unique_key = "concat_ws( '-', recorded_hour, id )" + unique_key = "concat_ws( '-', recorded_hour, id )", + tags = ['scheduled'] ) }} -- model named prices hourly but core view is hourly prices -- this is to organize the silver model with other prices models diff --git a/models/silver/prices/silver__prices_swaps.sql b/models/silver/prices/silver__prices_swaps.sql index 6781e0d..13ee9fc 100644 --- a/models/silver/prices/silver__prices_swaps.sql +++ b/models/silver/prices/silver__prices_swaps.sql @@ -2,7 +2,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::date'], - unique_key = "CONCAT_WS('-', block_timestamp, token_contract)" + unique_key = "CONCAT_WS('-', block_timestamp, token_contract)", + tags = ['scheduled'] ) }} WITH swaps AS ( diff --git a/models/silver/prices/silver__prices_swaps_hourly.sql b/models/silver/prices/silver__prices_swaps_hourly.sql index b1dad5c..8780eb8 100644 --- a/models/silver/prices/silver__prices_swaps_hourly.sql +++ b/models/silver/prices/silver__prices_swaps_hourly.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'table', cluster_by = ['recorded_hour::date'], - unique_key = "CONCAT_WS('-', recorded_hour, token)" + unique_key = "CONCAT_WS('-', recorded_hour, token)", + tags = ['scheduled'] ) }} WITH swap_prices AS ( diff --git a/models/silver/staking/silver__staking_actions.sql b/models/silver/staking/silver__staking_actions.sql index 0e7bf38..862b6bf 100644 --- a/models/silver/staking/silver__staking_actions.sql +++ b/models/silver/staking/silver__staking_actions.sql @@ -2,7 +2,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id' + unique_key = 'tx_id', + tags = ['scheduled'] ) }} WITH silver_events AS ( diff --git a/models/silver/swaps/silver__swaps.sql b/models/silver/swaps/silver__swaps.sql index 3be3a35..623157d 100644 --- a/models/silver/swaps/silver__swaps.sql +++ b/models/silver/swaps/silver__swaps.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['_inserted_timestamp::DATE'], unique_key = "CONCAT_WS('-', tx_id, swap_index)", - incremental_strategy = 'delete+insert' + incremental_strategy = 'delete+insert', + tags = ['scheduled'] ) }} WITH events AS ( diff --git a/models/silver/swaps/silver__swaps_events.sql b/models/silver/swaps/silver__swaps_events.sql index f915180..5f349d2 100644 --- a/models/silver/swaps/silver__swaps_events.sql +++ b/models/silver/swaps/silver__swaps_events.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['_inserted_timestamp::DATE'], unique_key = "CONCAT_WS('-', tx_id, event_index)", - incremental_strategy = 'delete+insert' + incremental_strategy = 'delete+insert', + tags = ['scheduled'] ) }} WITH swap_contracts AS ( diff --git a/models/silver/swaps/silver__swaps_single_trade.sql b/models/silver/swaps/silver__swaps_single_trade.sql index 2cd74ed..2486601 100644 --- a/models/silver/swaps/silver__swaps_single_trade.sql +++ b/models/silver/swaps/silver__swaps_single_trade.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'tx_id', - incremental_strategy = 'delete+insert' + incremental_strategy = 'delete+insert', + tags = ['scheduled'] ) }} WITH swaps_events AS ( diff --git a/models/silver/transfers/silver__bridge_blocto.sql b/models/silver/transfers/silver__bridge_blocto.sql index cfb6f2f..a40a53f 100644 --- a/models/silver/transfers/silver__bridge_blocto.sql +++ b/models/silver/transfers/silver__bridge_blocto.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::date'], unique_key = 'tx_id', - tags = ['bridge'] + tags = ['bridge', 'scheduled'] ) }} WITH events AS ( diff --git a/models/silver/transfers/silver__bridge_celer.sql b/models/silver/transfers/silver__bridge_celer.sql index 55aafe1..f280c2a 100644 --- a/models/silver/transfers/silver__bridge_celer.sql +++ b/models/silver/transfers/silver__bridge_celer.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::date'], unique_key = 'tx_id', - tags = ['bridge'] + tags = ['bridge', 'scheduled'] ) }} WITH events AS ( diff --git a/models/silver/transfers/silver__token_transfers.sql b/models/silver/transfers/silver__token_transfers.sql index eab09af..b5333ed 100644 --- a/models/silver/transfers/silver__token_transfers.sql +++ b/models/silver/transfers/silver__token_transfers.sql @@ -2,7 +2,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::date'], - unique_key = "CONCAT_WS('-', tx_id, sender, recipient, token_contract, amount)" + unique_key = "CONCAT_WS('-', tx_id, sender, recipient, token_contract, amount)", + tags = ['scheduled'] ) }} WITH events AS ( diff --git a/models/sources.yml b/models/sources.yml index 578952d..e97577d 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -33,10 +33,10 @@ sources: - name: transaction_results - - name: crosschain_v2 + - name: crosschain_silver database: crosschain schema: silver tables: - name: hourly_prices_coin_gecko - name: hourly_prices_coin_market_cap - + - name: number_sequence \ No newline at end of file