diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index e509a06..736e7cc 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -47,7 +47,7 @@ jobs: git checkout -B docs origin/main - name: generate dbt docs - run: dbt docs generate -t prod + run: dbt docs generate -t prod --no-compile - name: move files to docs directory run: | diff --git a/.github/workflows/dbt_run_scheduled_core.yml b/.github/workflows/dbt_run_scheduled_core.yml index 41c2fcd..cc1b922 100644 --- a/.github/workflows/dbt_run_scheduled_core.yml +++ b/.github/workflows/dbt_run_scheduled_core.yml @@ -42,10 +42,8 @@ jobs: - name: Run DBT Jobs run: | - dbt run-operation dispatch_github_workflow --args "{'repo_name': 'streamline-snowflake', 'workflow_name': 'dbt_run_near_external_table_update', 'gb_id': '${{ secrets.GB_ID}}'}"; dbt seed; - dbt run -s tag:scheduled_core --vars "{ 'RECEIPT_MAP_LOOKBACK_HOURS': 1.25}"; - dbt run -s tag:scheduled_non_core models/gold; + dbt run -s tag:scheduled_core tag:scheduled_non_core models/gold; - name: Store logs uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/dbt_run_scheduled_non_core.yml b/.github/workflows/dbt_run_scheduled_non_core.yml index 183a1a8..8367c60 100644 --- a/.github/workflows/dbt_run_scheduled_non_core.yml +++ b/.github/workflows/dbt_run_scheduled_non_core.yml @@ -42,7 +42,6 @@ jobs: - name: Run DBT Jobs run: | - dbt run-operation dispatch_github_workflow --args "{'repo_name': 'streamline-snowflake', 'workflow_name': 'dbt_run_near_external_table_update', 'gb_id': '${{ secrets.GB_ID}}'}"; dbt run -s tag:scheduled_non_core models/gold; - name: Store logs diff --git a/dbt_project.yml b/dbt_project.yml index eb57a4a..0cbe238 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ # and underscores. A good package name should reflect your organization's # name or the intended use of these models name: "near_models" -version: "1.3.0" +version: "2.0.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. @@ -66,9 +66,10 @@ vars: MANUAL_FIX: False OBSERV_FULL_TEST: False DBT_FULL_TEST: False - STREAMLINE_LOAD_LOOKBACK_HOURS: 3 - RECEIPT_MAP_LOOKBACK_HOURS: 6 - IS_MIGRATION: False + STREAMLINE_LOAD_LOOKBACK_HOURS: 3 # todo can deprecate + RECEIPT_MAP_LOOKBACK_HOURS: 6 # todo can deprecate + IS_MIGRATION: False # todo can deprecate + NEAR_MIGRATE_ARCHIVE: False HEAL_MODELS: [] core_folders: [ 'silver/streamline', diff --git a/macros/dbt/get_merge.sql b/macros/dbt/get_merge.sql index e3c75ef..7961ef2 100644 --- a/macros/dbt/get_merge.sql +++ b/macros/dbt/get_merge.sql @@ -1,4 +1,9 @@ {% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%} - {% set merge_sql = fsc_utils.get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %} + {% if incremental_predicates[0] == "dynamic_range_predicate_custom" %} + {% set predicates = [dynamic_range_predicate_custom(source, incremental_predicates[1], "DBT_INTERNAL_DEST")] %} + {% set merge_sql = fsc_utils.get_merge_sql(target, source, unique_key, dest_columns, predicates) %} + {% else %} + {% set merge_sql = fsc_utils.get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %} + {% endif %} {{ return(merge_sql) }} {% endmacro %} diff --git a/macros/dynamic_range_predicate_custom.sql b/macros/dynamic_range_predicate_custom.sql new file mode 100644 index 0000000..f11a8b7 --- /dev/null +++ b/macros/dynamic_range_predicate_custom.sql @@ -0,0 +1,117 @@ +{% macro dynamic_range_predicate_custom(source, predicate_column, output_alias="") -%} + {% set supported_data_types = ["INTEGER","DATE"] %} + {% set predicate_column_data_type_query %} + SELECT typeof({{predicate_column}}::variant) + FROM {{ source }} + WHERE {{predicate_column}} IS NOT NULL + LIMIT 1; + {% endset %} + {% set predicate_column_data_type_result = run_query(predicate_column_data_type_query) %} + {% if predicate_column_data_type_result.rows|length == 0 %} + {{ return('1=1') }} + {% endif %} + {% set predicate_column_data_type = predicate_column_data_type_result.columns[0].values()[0] %} + + + {% if predicate_column_data_type not in supported_data_types %} + {{ exceptions.raise_compiler_error("Data type of "~ predicate_column_data_type ~" is not supported, use one of "~ supported_data_types ~" column instead") }} + {% endif %} + + {% set get_start_end_query %} + SELECT + MIN( + {{ predicate_column }} + ) AS full_range_start, + MAX( + {{ predicate_column }} + ) AS full_range_end + FROM + {{ source }} + {% endset %} + {% set start_end_results = run_query(get_start_end_query).columns %} + {% set start_preciate_value = start_end_results[0].values()[0] %} + {% set end_predicate_value = start_end_results[1].values()[0] %} + + {% set get_limits_query %} + WITH block_range AS ( + {% if predicate_column_data_type == "INTEGER" %} + SELECT + SEQ4() + {{ start_preciate_value }} as predicate_value + FROM + TABLE(GENERATOR(rowcount => {{ end_predicate_value - start_preciate_value }}+1)) + {% else %} + SELECT + date_day as predicate_value + FROM + crosschain.core.dim_dates + WHERE + date_day BETWEEN '{{ start_preciate_value }}' AND '{{ end_predicate_value }}' + {% endif %} + ), + partition_block_counts AS ( + SELECT + b.predicate_value, + COUNT(r.{{ predicate_column }}) AS count_in_window + FROM + block_range b + LEFT OUTER JOIN {{ source }} + r + ON r.{{ predicate_column }} = b.predicate_value + GROUP BY + 1 + ), + range_groupings AS ( + SELECT + predicate_value, + count_in_window, + conditional_change_event( + count_in_window > 0 + ) over ( + ORDER BY + predicate_value + ) AS group_val + FROM + partition_block_counts + ), + contiguous_ranges AS ( + SELECT + MIN(predicate_value) AS start_value, + MAX(predicate_value) AS end_value + FROM + range_groupings + WHERE + count_in_window > 0 + GROUP BY + group_val + ), + between_stmts AS ( + SELECT + CONCAT( + '{{ output_alias~"." if output_alias else "" }}', + '{{ predicate_column }} between \'', + start_value, + '\' and \'', + end_value, + '\'' + ) AS b + FROM + contiguous_ranges + ) + SELECT + CONCAT('(', LISTAGG(b, ' OR '), ')', ' OR ', '(', '{{ output_alias~"." if output_alias else "" }}{{ predicate_column }} is null', ')') + FROM + between_stmts + {% endset %} + + {% set between_stmts = run_query(get_limits_query).columns[0].values()[0] %} + + {% if between_stmts != '()' %} + /* in case empty update array */ + {% set predicate_override = between_stmts %} + {% else %} + {% set predicate_override = '1=1' %} + /* need to have something or it will error since it expects at least 1 predicate */ + {% endif %} + + {{ return(predicate_override) }} +{% endmacro %} diff --git a/macros/manual_batch_refresh.sql b/macros/manual_batch_refresh.sql index 722f59e..ee06713 100644 --- a/macros/manual_batch_refresh.sql +++ b/macros/manual_batch_refresh.sql @@ -1,5 +1,6 @@ {% macro partition_load_manual( - scope = 'no_buffer' + scope = 'no_buffer', + partition_field = '_partition_by_block_number' ) %} {# if range_start and range_end not set in cli, default to earliest rpc data #} {% set range_start = var( @@ -19,17 +20,17 @@ 1 ) %} {% if scope == 'front' %} - _partition_by_block_number BETWEEN {{ range_start }} - ( + {{ partition_field }} BETWEEN {{ range_start }} - ( 10000 * {{ front_buffer }} ) AND {{ range_end }} {% elif scope == 'end' %} - _partition_by_block_number BETWEEN {{ range_start }} + {{ partition_field }} BETWEEN {{ range_start }} AND {{ range_end }} + ( 10000 * {{ end_buffer }} ) {% elif scope == 'no_buffer' %} - _partition_by_block_number BETWEEN {{ range_start }} + {{ partition_field }} BETWEEN {{ range_start }} AND {{ range_end }} {% else %} TRUE diff --git a/models/descriptions/chunks_json.md b/models/descriptions/chunks_json.md new file mode 100644 index 0000000..c35faaa --- /dev/null +++ b/models/descriptions/chunks_json.md @@ -0,0 +1,5 @@ +{% docs chunks_json %} + +A JSON containing an array of chunk headers contained in this block. + +{% enddocs %} diff --git a/models/descriptions/header_json.md b/models/descriptions/header_json.md new file mode 100644 index 0000000..19ce000 --- /dev/null +++ b/models/descriptions/header_json.md @@ -0,0 +1,5 @@ +{% docs header_json %} + +A JSON containing the block header. + +{% enddocs %} diff --git a/models/descriptions/outcome_json.md b/models/descriptions/outcome_json.md new file mode 100644 index 0000000..adc488d --- /dev/null +++ b/models/descriptions/outcome_json.md @@ -0,0 +1,3 @@ +{% docs outcome_json %} +JSON object containing the outcome of the receipt or transaction execution, including gas usage, status, and logs. +{% enddocs %} diff --git a/models/descriptions/receipt_json.md b/models/descriptions/receipt_json.md new file mode 100644 index 0000000..3758801 --- /dev/null +++ b/models/descriptions/receipt_json.md @@ -0,0 +1,3 @@ +{% docs receipt_json %} +JSON object containing the full receipt data including actions, predecessor, receiver, and other metadata. +{% enddocs %} diff --git a/models/descriptions/status_json.md b/models/descriptions/status_json.md new file mode 100644 index 0000000..03fe070 --- /dev/null +++ b/models/descriptions/status_json.md @@ -0,0 +1,3 @@ +{% docs status_json %} +JSON object containing the status of the transaction, including success or failure information. +{% enddocs %} diff --git a/models/descriptions/transaction_json.md b/models/descriptions/transaction_json.md new file mode 100644 index 0000000..9f5e28b --- /dev/null +++ b/models/descriptions/transaction_json.md @@ -0,0 +1,3 @@ +{% docs transaction_json %} +JSON object containing the full transaction data including actions, signer, receiver, and other metadata. +{% enddocs %} diff --git a/models/gold/atlas/atlas__ez_nft_contract_metrics.sql b/models/gold/atlas/atlas__ez_nft_contract_metrics.sql deleted file mode 100644 index 8212c37..0000000 --- a/models/gold/atlas/atlas__ez_nft_contract_metrics.sql +++ /dev/null @@ -1,34 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'ATLAS' - } - } - }, - tags = ['atlas'] -) }} - -WITH nft_data AS ( - - SELECT - atlas_nft_table_id AS ez_nft_contract_metrics_id, - receiver_id, - tokens, - transfers_24h, - transfers_3d, - all_transfers, - owners, - transactions, - mints, - COALESCE(inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp - FROM - {{ ref('silver__atlas_nft_table') }} -) -SELECT - * -FROM - nft_data diff --git a/models/gold/atlas/atlas__ez_nft_contract_metrics.yml b/models/gold/atlas/atlas__ez_nft_contract_metrics.yml deleted file mode 100644 index 33ebde6..0000000 --- a/models/gold/atlas/atlas__ez_nft_contract_metrics.yml +++ /dev/null @@ -1,99 +0,0 @@ -version: 2 - -models: - - name: atlas__ez_nft_contract_metrics - description: |- - Deprecating Soon - these tables previously supported NEAR Atlas which is no longer live. These tables will be removed on Febreary 1, 2025. - Please cease using these tables. - tests: - - dbt_utils.recency: - datepart: days - field: inserted_timestamp - interval: 1 - - columns: - - name: EZ_NFT_CONTRACT_METRICS_ID - description: "{ { doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: RECEIVER_ID - description: "{ { doc('receiver_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TOKENS - description: "{{ doc('tokens_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: TRANSFERS_24H - description: "The count of 'nft_transfer' transactions that occurred in the last 24 hours." - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: TRANSFERS_3D - description: "The count of 'nft_transfer' transactions that occurred in the last 3 days." - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: ALL_TRANSFERS - description: "{{ doc('all_transfers')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: OWNERS - description: "{{ doc('owner_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: TRANSACTIONS - description: "{{ doc('tx_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: MINTS - description: "{{ doc('mint_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp')}}" diff --git a/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.sql b/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.sql deleted file mode 100644 index ca74d83..0000000 --- a/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.sql +++ /dev/null @@ -1,33 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'ATLAS' - } - } - }, - tags = ['atlas'] -) }} - -WITH nft_detailed AS ( - - SELECT - atlas_nft_detailed_id AS ez_nft_contract_metrics_daily_id, - DAY, - receiver_id, - tokens, - all_transfers, - owners, - transactions, - mints, - COALESCE(inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp - FROM - {{ ref('silver__atlas_nft_detailed') }} -) -SELECT - * -FROM - nft_detailed diff --git a/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.yml b/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.yml deleted file mode 100644 index 309344e..0000000 --- a/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.yml +++ /dev/null @@ -1,86 +0,0 @@ -version: 2 - -models: - - name: atlas__ez_nft_contract_metrics_daily - description: |- - Deprecating Soon - these tables previously supported NEAR Atlas which is no longer live. These tables will be removed on Febreary 1, 2025. - Please cease using these tables. - tests: - - dbt_utils.recency: - datepart: days - field: inserted_timestamp - interval: 1 - -columns: - - name: EZ_NFT_CONTRACT_METRICS_DAILY_ID - description: "{{ doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: DAY - description: "{{ doc('date')}}" - tests: - - not_null - - - name: RECEIVER_ID - description: "{{ doc('tx_receiver')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TOKENS - description: "{{ doc('tokens_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: ALL_TRANSFERS - description: "{{ doc('all_transfers')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: OWNERS - description: "{{ doc('owner_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: TRANSACTIONS - description: "{{ doc('tx_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: MINTS - description: "{{ doc('mint_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp')}}" diff --git a/models/gold/atlas/atlas__fact_accounts_created.sql b/models/gold/atlas/atlas__fact_accounts_created.sql deleted file mode 100644 index d759523..0000000 --- a/models/gold/atlas/atlas__fact_accounts_created.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'ATLAS' - } - } - }, - tags = ['atlas'] -) }} - -SELECT - atlas_account_created_id AS fact_accounts_created_id, - DAY, - wallets_created, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__atlas_accounts_created') }} diff --git a/models/gold/atlas/atlas__fact_accounts_created.yml b/models/gold/atlas/atlas__fact_accounts_created.yml deleted file mode 100644 index 44abe6f..0000000 --- a/models/gold/atlas/atlas__fact_accounts_created.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: 2 - -models: - - name: atlas__fact_accounts_created - description: |- - Deprecating Soon - these tables previously supported NEAR Atlas which is no longer live. These tables will be removed on Febreary 1, 2025. - Please cease using these tables. - tests: - - dbt_utils.recency: - datepart: days - field: inserted_timestamp - interval: 1 - - columns: - - name: FACT_ACCOUNTS_CREATED_ID - description: "{ { doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: DAY - description: "{{ doc('date')}}" - tests: - - not_null - - - name: WALLETS_CREATED - description: "{{ doc('wallets_created')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/gold/atlas/atlas__fact_maas.sql b/models/gold/atlas/atlas__fact_maas.sql deleted file mode 100644 index 717dc4a..0000000 --- a/models/gold/atlas/atlas__fact_maas.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ config( - materialized = 'view', - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'ATLAS' - } - } - }, - tags = ['atlas'] -) }} - -SELECT - atlas_maa_id AS fact_maas_id, - day, - maa, - new_maas, - returning_maas, - COALESCE(inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp -FROM - {{ ref('silver__atlas_maa') }} diff --git a/models/gold/atlas/atlas__fact_maas.yml b/models/gold/atlas/atlas__fact_maas.yml deleted file mode 100644 index fe2f089..0000000 --- a/models/gold/atlas/atlas__fact_maas.yml +++ /dev/null @@ -1,48 +0,0 @@ -version: 2 - -models: - - name: atlas__fact_maas - description: |- - Deprecating Soon - these tables previously supported NEAR Atlas which is no longer live. These tables will be removed on Febreary 1, 2025. - Please cease using these tables. - tests: - - dbt_utils.recency: - datepart: days - field: inserted_timestamp - interval: 1 - - columns: - - name: fact_maas_id - description: "{{ doc('id') }}" - tests: - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: day - description: "{{ doc('active_day') }}" - tests: - - not_null - - unique - - - name: maa - description: "{{ doc('maa')}}" - tests: - - not_null - - - name: new_maas - description: "{{ doc('new_maas') }}" - tests: - - not_null - - - name: returning_maas - description: "{{ doc('returning_maas') }}" - tests: - - not_null - - - name: inserted_timestamp - description: "{{ doc('inserted_timestamp') }}" - - - name: modified_timestamp - description: "{{ doc('modified_timestamp') }}" diff --git a/models/gold/atlas/atlas__fact_nft_monthly_txs.sql b/models/gold/atlas/atlas__fact_nft_monthly_txs.sql deleted file mode 100644 index 00413e9..0000000 --- a/models/gold/atlas/atlas__fact_nft_monthly_txs.sql +++ /dev/null @@ -1,28 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'ATLAS' - } - } - }, - tags = ['atlas'] -) }} - -WITH TRAILING AS ( - - SELECT - atlas_nft_30_trailing_id AS fact_nft_monthly_txs_id, - DAY, - txns, - COALESCE(inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp - FROM - {{ ref('silver__atlas_nft_30_trailing') }} -) -SELECT - * -FROM - TRAILING diff --git a/models/gold/atlas/atlas__fact_nft_monthly_txs.yml b/models/gold/atlas/atlas__fact_nft_monthly_txs.yml deleted file mode 100644 index 9b1e83a..0000000 --- a/models/gold/atlas/atlas__fact_nft_monthly_txs.yml +++ /dev/null @@ -1,41 +0,0 @@ -version: 2 - -models: - - name: atlas__fact_nft_monthly_txs - description: |- - Deprecating Soon - these tables previously supported NEAR Atlas which is no longer live. These tables will be removed on Febreary 1, 2025. - Please cease using these tables. - tests: - - dbt_utils.recency: - datepart: days - field: inserted_timestamp - interval: 1 - - columns: - - name: FACT_NFT_MONTHLY_TXS_ID - description: "{{ doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: DAY - description: "{{ doc('date')}}" - tests: - - not_null - - - name: TXNS - description: "{{ doc('tx_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp')}}" diff --git a/models/gold/core/core__ez_actions.sql b/models/gold/core/core__ez_actions.sql index 6f9d96d..5ef321c 100644 --- a/models/gold/core/core__ez_actions.sql +++ b/models/gold/core/core__ez_actions.sql @@ -8,8 +8,8 @@ post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receipt_receiver_id,receipt_signer_id,receipt_predecessor_id);", tags = ['actions', 'curated', 'scheduled_core', 'grail'] ) }} --- depends_on: {{ ref('silver__streamline_transactions_final') }} --- depends_on: {{ ref('silver__streamline_receipts_final') }} +-- depends_on: {{ ref('silver__transactions_final') }} +-- depends_on: {{ ref('silver__receipts_final') }} {% if execute %} @@ -41,14 +41,14 @@ SELECT MIN(block_timestamp) block_timestamp FROM - {{ ref('silver__streamline_transactions_final') }} A + {{ ref('silver__transactions_final') }} A WHERE modified_timestamp >= '{{max_mod}}' UNION ALL SELECT MIN(block_timestamp) block_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} A + {{ ref('silver__receipts_final') }} A WHERE modified_timestamp >= '{{max_mod}}' ) @@ -76,7 +76,7 @@ WITH transactions AS ( transaction_fee AS tx_fee, modified_timestamp FROM - {{ ref('silver__streamline_transactions_final') }} + {{ ref('silver__transactions_final') }} {% if var("MANUAL_FIX") %} WHERE @@ -92,19 +92,18 @@ receipts AS ( tx_hash, block_id, block_timestamp, - receipt_object_id AS receipt_id, + receipt_id, receiver_id AS receipt_receiver_id, - signer_id AS receipt_signer_id, - receipt_actions :predecessor_id :: STRING AS receipt_predecessor_id, + receipt_json :receipt :Action :signer_id :: STRING AS receipt_signer_id, + predecessor_id AS receipt_predecessor_id, receipt_succeeded, - gas_burnt AS receipt_gas_burnt, - status_value, - receipt_actions, + outcome_json :outcome :gas_burnt :: NUMBER AS receipt_gas_burnt, + outcome_json :outcome :status :: VARIANT AS status_value, + receipt_json, _partition_by_block_number, - _inserted_timestamp, modified_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} {% if var("MANUAL_FIX") %} WHERE @@ -132,9 +131,8 @@ join_data AS ( r.receipt_succeeded, r.receipt_gas_burnt, r.status_value, - r.receipt_actions, - r._partition_by_block_number, - r._inserted_timestamp + r.receipt_json, + r._partition_by_block_number FROM receipts r LEFT JOIN transactions t @@ -184,7 +182,7 @@ flatten_actions AS ( ) as receipt_status_value, False AS is_delegated, INDEX AS action_index, - receipt_actions :receipt :Action :gas_price :: NUMBER AS action_gas_price, + receipt_json :receipt :Action :gas_price :: NUMBER AS action_gas_price, IFF( VALUE = 'CreateAccount', VALUE, @@ -247,12 +245,11 @@ flatten_actions AS ( ), action_data ) AS action_data_parsed, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM join_data, LATERAL FLATTEN( - receipt_actions :receipt :Action :actions :: ARRAY + receipt_json :receipt :Action :actions :: ARRAY ) ), flatten_delegated_actions AS ( @@ -333,7 +330,6 @@ SELECT action_data_parsed AS action_data, action_gas_price, _partition_by_block_number, - _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['receipt_id', 'action_index'] ) }} AS actions_id, diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index d0f1264..77f0498 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -39,9 +39,9 @@ {% if not min_block_timestamp_day or min_block_timestamp_day == 'None' %} {% set min_block_timestamp_day = '2020-07-01' %} {% endif %} +{{ log("min_block_timestamp_day: " ~ min_block_timestamp_day, info=True) }} {% endif %} -{{ log("min_block_timestamp_day: " ~ min_block_timestamp_day, info=True) }} WITH hourly_prices AS ( SELECT diff --git a/models/gold/core/core__fact_actions_events.sql b/models/gold/core/core__fact_actions_events.sql deleted file mode 100644 index 5dc2df0..0000000 --- a/models/gold/core/core__fact_actions_events.sql +++ /dev/null @@ -1,38 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - tags = ['core'] -) }} - -WITH actions AS ( - - SELECT - * - FROM - {{ ref('silver__actions_events_s3') }} -) -SELECT - action_id, - tx_hash, - receipt_object_id AS receipt_id, - predecessor_id, - receiver_id, - signer_id, - block_id, - block_timestamp, - action_index, - action_name, - action_data, - logs, - receipt_succeeded, - COALESCE( - actions_events_id, - {{ dbt_utils.generate_surrogate_key( - ['receipt_object_id', 'action_index'] - ) }} - ) AS fact_actions_events_id, - receipt_object_id, - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp -FROM - actions diff --git a/models/gold/core/core__fact_actions_events.yml b/models/gold/core/core__fact_actions_events.yml deleted file mode 100644 index 6135eee..0000000 --- a/models/gold/core/core__fact_actions_events.yml +++ /dev/null @@ -1,130 +0,0 @@ -version: 2 - -models: - - name: core__fact_actions_events - description: |- - Deprecating soon! Please migrate to using the new `core__ez_actions` table. - This view will be removed in Q1 2025. - tests: - - dbt_utils.recency: - datepart: hours - field: block_timestamp - interval: 2 - - columns: - - name: ACTION_ID - description: "{{ doc('action_id')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: RECEIPT_ID - description: "{{ doc('receipt_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: PREDECESSOR_ID - description: "{{ doc('predecessor_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: ACTION_INDEX - description: "{{ doc('action_index')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: ACTION_NAME - description: "{{ doc('action_name')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: ACTION_DATA - description: "{{ doc('action_data')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - OBJECT - - VARIANT - - - name: LOGS - description: "{{ doc('logs')}}" - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - - - name: FACT_ACTIONS_EVENTS_ID - description: "{{ doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' AND FACT_ACTIONS_EVENTS_ID != 'cf646ad92e6df243ffabf07c47c0f2c1' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp')}}" diff --git a/models/gold/core/core__fact_actions_events_function_call.sql b/models/gold/core/core__fact_actions_events_function_call.sql deleted file mode 100644 index 637250a..0000000 --- a/models/gold/core/core__fact_actions_events_function_call.sql +++ /dev/null @@ -1,38 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - tags = ['core'] -) }} - -WITH actions_events_function_call AS ( - - SELECT - * - FROM - {{ ref('silver__actions_events_function_call_s3') }} -) -SELECT - action_id, - tx_hash, - receiver_id, - predecessor_id, - signer_id, - block_id, - block_timestamp, - action_name, - method_name, - args, - deposit, - attached_gas, - logs, - receipt_succeeded, - COALESCE( - actions_events_function_call_id, - {{ dbt_utils.generate_surrogate_key( - ['action_id'] - ) }} - ) AS fact_actions_events_function_call_id, - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp -FROM - actions_events_function_call diff --git a/models/gold/core/core__fact_actions_events_function_call.yml b/models/gold/core/core__fact_actions_events_function_call.yml deleted file mode 100644 index c496fa9..0000000 --- a/models/gold/core/core__fact_actions_events_function_call.yml +++ /dev/null @@ -1,146 +0,0 @@ -version: 2 - -models: - - name: core__fact_actions_events_function_call - description: |- - Deprecating soon! Please migrate to using the new `core__ez_actions` table. All FunctionCalls are decoded in the args element where action_name = 'FunctionCall'. - This view will be removed in Q1 2025. - tests: - - dbt_utils.recency: - datepart: hours - field: block_timestamp - interval: 2 - - columns: - - name: ACTION_ID - description: "{{ doc('action_id')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: PREDECESSOR_ID - description: "{{ doc('predecessor_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: ACTION_NAME - description: "{{ doc('action_name')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: METHOD_NAME - description: "{{ doc('method_name')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: ARGS - description: "{{ doc('args')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - - OBJECT - - - name: DEPOSIT - description: "{{ doc('deposit')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: ATTACHED_GAS - description: "{{ doc('attached_gas')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: LOGS - description: "{{ doc('logs')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - VARIANT - - OBJECT - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - tests: - - not_null - - - name: FACT_ACTIONS_EVENTS_FUNCTION_CALL_ID - description: "{{doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index fb700e3..b2cde7d 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -9,47 +9,42 @@ WITH blocks AS ( SELECT * FROM - {{ ref('silver__streamline_blocks') }} + {{ ref('silver__blocks_final') }} ) SELECT block_id, block_timestamp, block_hash, block_author, - header, - header :challenges_result :: ARRAY AS block_challenges_result, - header :challenges_root :: STRING AS block_challenges_root, - header :chunk_headers_root :: STRING AS chunk_headers_root, - header :chunk_tx_root :: STRING AS chunk_tx_root, - header :chunk_mask :: ARRAY AS chunk_mask, - header :chunk_receipts_root :: STRING AS chunk_receipts_root, - chunks, - header :chunks_included :: NUMBER AS chunks_included, - epoch_id, - header :epoch_sync_data_hash :: STRING AS epoch_sync_data_hash, - gas_price, - header :last_ds_final_block :: STRING AS last_ds_final_block, - header :last_final_block :: STRING AS last_final_block, - latest_protocol_version, - header: next_bp_hash :: STRING AS next_bp_hash, - next_epoch_id, - header :outcome_root :: STRING AS outcome_root, + header_json AS header, + header_json :challenges_result :: ARRAY AS block_challenges_result, + header_json :challenges_root :: STRING AS block_challenges_root, + header_json :chunk_headers_root :: STRING AS chunk_headers_root, + header_json :chunk_tx_root :: STRING AS chunk_tx_root, + header_json :chunk_mask :: ARRAY AS chunk_mask, + header_json :chunk_receipts_root :: STRING AS chunk_receipts_root, + chunks_json AS chunks, + header_json :chunks_included :: NUMBER AS chunks_included, + header_json :epoch_id :: STRING AS epoch_id, + header_json :epoch_sync_data_hash :: STRING AS epoch_sync_data_hash, + header_json :gas_price :: FLOAT AS gas_price, + header_json :last_ds_final_block :: STRING AS last_ds_final_block, + header_json :last_final_block :: STRING AS last_final_block, + header_json :latest_protocol_version :: INT AS latest_protocol_version, + header_json :next_bp_hash :: STRING AS next_bp_hash, + header_json :next_epoch_id :: STRING AS next_epoch_id, + header_json :outcome_root :: STRING AS outcome_root, prev_hash, - header :prev_height :: NUMBER AS prev_height, - header :prev_state_root :: STRING AS prev_state_root, - header :random_value :: STRING AS random_value, - header :rent_paid :: FLOAT AS rent_paid, - header :signature :: STRING AS signature, - total_supply, - validator_proposals, - validator_reward, - COALESCE( - streamline_blocks_id, - {{ dbt_utils.generate_surrogate_key( - ['block_id'] - ) }} - ) AS fact_blocks_id, - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp + header_json :prev_height :: NUMBER AS prev_height, + header_json :prev_state_root :: STRING AS prev_state_root, + header_json :random_value :: STRING AS random_value, + header_json :rent_paid :: FLOAT AS rent_paid, + header_json :signature :: STRING AS signature, + header_json :total_supply :: FLOAT AS total_supply, + header_json :validator_proposals :: ARRAY AS validator_proposals, + header_json :validator_reward :: FLOAT AS validator_reward, + blocks_final_id AS fact_blocks_id, + inserted_timestamp, + modified_timestamp FROM blocks diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index cde8c8e..5b2b750 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -33,7 +33,7 @@ models: tests: - not_null - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -154,7 +154,8 @@ models: - name: GAS_PRICE description: "{{ doc('gas_price')}}" tests: - - not_null + - not_null: + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER @@ -181,11 +182,12 @@ models: - name: LATEST_PROTOCOL_VERSION description: "{{ doc('latest_protocol_version')}}" tests: - - not_null + - not_null: + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - - FLOAT + - INT - name: NEXT_BP_HASH description: "{{ doc('next_bp_hash')}}" @@ -297,9 +299,9 @@ models: description: "{{doc('id')}}" tests: - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: INSERTED_TIMESTAMP description: "{{doc('inserted_timestamp')}}" diff --git a/models/gold/core/core__fact_logs.sql b/models/gold/core/core__fact_logs.sql index a1fb883..87bc254 100644 --- a/models/gold/core/core__fact_logs.sql +++ b/models/gold/core/core__fact_logs.sql @@ -15,7 +15,7 @@ SELECT block_id, block_timestamp, tx_hash, - receipt_object_id AS receipt_id, + COALESCE(receipt_id, receipt_object_id) AS receipt_id, predecessor_id, receiver_id, signer_id, @@ -30,7 +30,6 @@ SELECT ['log_id'] ) }} ) AS fact_logs_id, - receipt_object_id, -- will drop col eventually COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp FROM diff --git a/models/gold/core/core__fact_receipts.sql b/models/gold/core/core__fact_receipts.sql index b403770..51b2efa 100644 --- a/models/gold/core/core__fact_receipts.sql +++ b/models/gold/core/core__fact_receipts.sql @@ -4,37 +4,24 @@ tags = ['core'] ) }} -WITH receipts AS ( - - SELECT - * - FROM - {{ ref('silver__streamline_receipts_final') }} -) SELECT block_timestamp, block_id, tx_hash, - receipt_object_id AS receipt_id, - receipt_outcome_id, + receipt_id, + outcome_json :outcome :receipt_ids :: ARRAY AS receipt_outcome_id, -- TODO DEPRECATE THIS, it's in outcome_json receiver_id, - receipt_actions :predecessor_id :: STRING AS predecessor_id, - receipt_actions AS actions, - execution_outcome AS outcome, - gas_burnt, - status_value, - logs, - proof, - metadata, + predecessor_id, + receipt_json AS actions, -- TODO this should be renamed. It's not just actions, it's the full receipt input + outcome_json AS outcome, + outcome_json :outcome :gas_burnt :: NUMBER AS gas_burnt, + outcome_json :outcome :status :: VARIANT AS status_value, + outcome_json :outcome :logs :: ARRAY AS logs, + outcome_json :proof :: ARRAY AS proof, -- TODO DEPRECATE THIS, it's in outcome_json + outcome_json :outcome :metadata :: VARIANT AS metadata, -- TODO DEPRECATE THIS, it's in outcome_json receipt_succeeded, - COALESCE( - streamline_receipts_final_id, - {{ dbt_utils.generate_surrogate_key( - ['receipt_object_id'] - ) }} - ) AS fact_receipts_id, - receipt_object_id, -- to be deprecated - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp + receipts_final_id AS fact_receipts_id, + inserted_timestamp, + modified_timestamp FROM - receipts + {{ ref('silver__receipts_final') }} diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 9cf1193..237a200 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -4,34 +4,21 @@ tags = ['core'] ) }} -WITH transactions AS ( - - SELECT - * - FROM - {{ ref('silver__streamline_transactions_final') }} -) SELECT tx_hash, block_id, - block_hash, block_timestamp, - nonce, - signature, + transaction_json :nonce :: INT AS nonce, + transaction_json :signature :: STRING AS signature, tx_receiver, tx_signer, - tx, + transaction_json AS tx, gas_used, transaction_fee, attached_gas, tx_succeeded, - COALESCE( - streamline_transactions_final_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash'] - ) }} - ) AS fact_transactions_id, - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp + transactions_final_id AS fact_transactions_id, + inserted_timestamp, + modified_timestamp FROM - transactions + {{ ref('silver__transactions_final') }} diff --git a/models/gold/defi/defi__fact_intents.yml b/models/gold/defi/defi__fact_intents.yml index c9ed012..d6b2150 100644 --- a/models/gold/defi/defi__fact_intents.yml +++ b/models/gold/defi/defi__fact_intents.yml @@ -65,7 +65,8 @@ models: - name: DIP4_VERSION description: "{{ doc('dip4_version') }}" tests: - - not_null + - not_null: + where: memo != 'refund' - name: GAS_BURNT description: "{{ doc('gas_burnt') }}" diff --git a/models/gold/horizon/horizon__fact_decoded_actions.sql b/models/gold/horizon/horizon__fact_decoded_actions.sql deleted file mode 100644 index 3867d49..0000000 --- a/models/gold/horizon/horizon__fact_decoded_actions.sql +++ /dev/null @@ -1,37 +0,0 @@ -{{ config( - materialized = 'view', - tags = ['core', 'horizon'] -) }} - -WITH horizon AS ( - - SELECT - action_id_horizon, - receipt_object_id, - tx_hash, - block_id, - block_timestamp, - method_name, - args, - deposit, - attached_gas, - receiver_id, - signer_id, - receipt_succeeded, - COALESCE( - horizon_decoded_actions_id, - {{ dbt_utils.generate_surrogate_key( - ['action_id_horizon'] - ) }} - ) AS fact_decoded_actions_id, - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp - FROM - {{ ref('silver_horizon__decoded_actions') }} - WHERE - method_name != 'set' -) -SELECT - * -FROM - horizon diff --git a/models/gold/horizon/horizon__fact_decoded_actions.yml b/models/gold/horizon/horizon__fact_decoded_actions.yml deleted file mode 100644 index b538dee..0000000 --- a/models/gold/horizon/horizon__fact_decoded_actions.yml +++ /dev/null @@ -1,64 +0,0 @@ - -version: 2 - -models: - - name: horizon__fact_decoded_actions - description: |- - Deprecating Soon - please cease using this table by February 1, 2025. - - columns: - - name: ACTION_ID_HORIZON - description: "{{ doc('action_id')}}" - - - name: RECEIPT_OBJECT_ID - description: "{{ doc('receipt_object_id')}}" - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: METHOD_NAME - description: "{{ doc('method_name')}}" - - - name: ARGS - description: "{{ doc('args')}}" - - - name: DEPOSIT - description: "{{ doc('deposit')}}" - - - name: ATTACHED_GAS - description: "{{ doc('attached_gas')}}" - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - - - name: FACT_DECODED_ACTIONS_ID - description: "{{doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index 203fa5d..db693c5 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('silver__streamline_blocks') }} +-- depends_on: {{ ref('silver__blocks_final') }} {{ config( materialized = 'incremental', unique_key = 'test_timestamp', @@ -16,8 +16,8 @@ WITH blocks_joined AS ( b.block_timestamp AS next_block_timestamp, b.prev_hash AS next_prev_hash FROM - {{ ref('silver__streamline_blocks') }} A -- Streamline Migration TODO - change this to fact blocks once table - LEFT JOIN {{ ref('silver__streamline_blocks') }} + {{ ref('silver__blocks_final') }} A + LEFT JOIN {{ ref('silver__blocks_final') }} b ON A.block_hash = b.prev_hash WHERE @@ -34,7 +34,7 @@ AND ( SELECT MIN(block_id) AS block_id FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) AND DATEADD('hour', -95, SYSDATE()) diff --git a/models/silver/_observability/silver_observability__chunks_completeness.sql b/models/silver/_observability/silver_observability__chunks_completeness.sql index 5d630c6..b4c8067 100644 --- a/models/silver/_observability/silver_observability__chunks_completeness.sql +++ b/models/silver/_observability/silver_observability__chunks_completeness.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('silver__streamline_blocks') }} +-- depends_on: {{ ref('silver__blocks_final') }} {{ config( materialized = 'incremental', unique_key = 'test_timestamp', @@ -12,11 +12,11 @@ WITH block_chunks_included AS ( SELECT block_id, block_timestamp, - header :chunks_included :: INT AS chunks_included, + header_json :chunks_included :: INT AS chunks_included, _partition_by_block_number, - _inserted_timestamp + inserted_timestamp AS _inserted_timestamp FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE block_timestamp <= DATEADD('hour', -12, SYSDATE()) {% if is_incremental() %} @@ -29,7 +29,7 @@ AND ( SELECT MIN(block_id) AS block_id FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) AND DATEADD('hour', -95, SYSDATE()) @@ -70,13 +70,13 @@ summary_stats AS ( ), chunks_per_block AS ( SELECT - block_id, + origin_block_id AS block_id, MAX(_inserted_timestamp) AS _inserted_timestamp, COUNT( - DISTINCT chunk :header :chunk_hash :: STRING + chunk_hash ) AS chunk_ct FROM - {{ ref('silver__streamline_shards') }} -- Streamline Migration TODO - change this to fact shards once table + {{ ref('silver__transactions_v2') }} WHERE block_id >= (SELECT min_block FROM summary_stats) AND diff --git a/models/silver/_observability/silver_observability__logs_completeness.sql b/models/silver/_observability/silver_observability__logs_completeness.sql index b37a29f..de4cb5b 100644 --- a/models/silver/_observability/silver_observability__logs_completeness.sql +++ b/models/silver/_observability/silver_observability__logs_completeness.sql @@ -4,7 +4,7 @@ full_refresh = False, tags = ['observability'] ) }} - +-- TODO this can be deprecated. Not a good metric of completeness. WITH summary_stats AS ( SELECT @@ -14,7 +14,7 @@ WITH summary_stats AS ( MAX(block_timestamp) AS max_block_timestamp, COUNT(1) AS blocks_tested FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE block_timestamp <= DATEADD('hour', -12, SYSDATE()) @@ -28,7 +28,7 @@ AND ( SELECT MIN(block_id) AS block_id FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) AND DATEADD('hour', -95, SYSDATE()) @@ -82,7 +82,7 @@ broken_blocks AS ( SELECT DISTINCT block_id as block_id FROM - {{ ref('silver__streamline_receipts_final') }} -- Streamline Migration TODO - change this to fact receipts once table + {{ ref('silver__receipts_final') }} r LEFT JOIN {{ ref('silver__logs_s3') }} l USING ( @@ -93,7 +93,7 @@ broken_blocks AS ( WHERE l.tx_hash IS NULL AND ARRAY_SIZE( - r.logs + r.outcome_json :outcome :logs :: ARRAY ) > 0 ), impacted_blocks AS ( diff --git a/models/silver/actions/README.md b/models/silver/actions/README.md new file mode 100644 index 0000000..4913a3a --- /dev/null +++ b/models/silver/actions/README.md @@ -0,0 +1,4 @@ +# Deprecating Actions Models + +All 3 of these models will be deprecated and dropped. +There are a number of silver models that need to be migrated to core.ez_actions. Migrate those by EOM (March 2025). diff --git a/models/silver/actions/silver__actions_events_addkey_s3.yml b/models/silver/actions/silver__actions_events_addkey_s3.yml deleted file mode 100644 index aadc86c..0000000 --- a/models/silver/actions/silver__actions_events_addkey_s3.yml +++ /dev/null @@ -1,55 +0,0 @@ -version: 2 - -models: - - name: silver__actions_events_addkey_s3 - description: |- - Deprecting soon - no longer updating. - - columns: - - name: ACTION_ID - description: "{{ doc('action_id')}}" - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - - - name: NONCE - description: "{{ doc('nonce')}}" - - - name: PUBLIC_KEY - description: "{{ doc('public_key')}}" - - - name: PERMISSION - description: "{{ doc('permission')}}" - - - name: ALLOWANCE - description: "{{ doc('allowance')}}" - - - name: METHOD_NAME - description: "{{ doc('method_name')}}" - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: actions_events_addkey_id - description: "{{doc('id')}}" - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - - - name: _invocation_id - description: "{{doc('invocation_id')}}" diff --git a/models/silver/actions/silver__actions_events_function_call_s3.yml b/models/silver/actions/silver__actions_events_function_call_s3.yml deleted file mode 100644 index 382c489..0000000 --- a/models/silver/actions/silver__actions_events_function_call_s3.yml +++ /dev/null @@ -1,73 +0,0 @@ -version: 2 - -models: - - name: silver__actions_events_function_call_s3 - description: |- - This table extracts all FunctionCall events from actions and decodes the arguments for easy use. - - columns: - - name: ACTION_ID - description: "{{ doc('action_id')}}" - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - - - name: PREDECESSOR_ID - description: "{{ doc('predecessor_id')}}" - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: ACTION_NAME - description: "{{ doc('action_name')}}" - - - name: METHOD_NAME - description: "{{ doc('method_name')}}" - - - name: ARGS - description: "{{ doc('args')}}" - - - name: DEPOSIT - description: "{{ doc('deposit')}}" - - - name: ATTACHED_GAS - description: "{{ doc('attached_gas')}}" - - - name: LOGS - description: "{{ doc('logs')}}" - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: ACTIONS_EVENTS_FUNCTION_CALL_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/actions/silver__actions_events_s3.sql b/models/silver/actions/silver__actions_events_s3.sql index f939946..cf3a903 100644 --- a/models/silver/actions/silver__actions_events_s3.sql +++ b/models/silver/actions/silver__actions_events_s3.sql @@ -15,21 +15,20 @@ WITH receipts AS ( SELECT tx_hash, - receipt_object_id, + receipt_id AS receipt_object_id, receiver_id, - signer_id, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, block_id, block_timestamp, chunk_hash, - logs, - receipt_actions, - execution_outcome, + outcome_json :outcome :logs :: ARRAY AS logs, + receipt_json AS receipt_actions, + outcome_json AS execution_outcome, receipt_succeeded, - gas_burnt, - _partition_by_block_number, - _inserted_timestamp + outcome_json :outcome :gas_burnt :: NUMBER AS gas_burnt, + _partition_by_block_number FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} {% if var("MANUAL_FIX") %} WHERE @@ -64,12 +63,11 @@ flatten_actions AS ( VALUE AS action_object, INDEX AS action_index, receipt_succeeded, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM receipts, LATERAL FLATTEN( - input => receipt_actions :receipt :Action :actions + input => receipt_actions :receipt :Action :actions :: ARRAY ) ), FINAL AS ( @@ -95,8 +93,7 @@ FINAL AS ( gas_price, gas_burnt, tokens_burnt, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM flatten_actions, LATERAL FLATTEN( diff --git a/models/silver/actions/silver__actions_events_s3.yml b/models/silver/actions/silver__actions_events_s3.yml deleted file mode 100644 index f03d4a4..0000000 --- a/models/silver/actions/silver__actions_events_s3.yml +++ /dev/null @@ -1,82 +0,0 @@ -version: 2 - -models: - - name: silver__actions_events_s3 - description: |- - This table extracts all action events from a receipt and stores the argument data under action_data. - - columns: - - name: ACTION_ID - description: "{{ doc('action_id')}}" - tests: - - unique: - where: tx_hash != 'J4CZZQrZK6kYPVLkrdbTEpcqhUNZiRxktbMzHviqeGgf' - - not_null - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: RECEIPT_OBJECT_ID - description: "{{ doc('receipt_object_id')}}" - - - name: CHUNK_HASH - description: "{{ doc('chunk_hash')}}" - tests: - - not_null: - where: "block_id not in (34691244, 34691277)" - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - - - name: PREDECESSOR_ID - description: "{{ doc('predecessor_id')}}" - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - - - name: ACTION_INDEX - description: "{{ doc('action_index')}}" - - - name: ACTION_NAME - description: "{{ doc('action_name')}}" - - - name: ACTION_DATA - description: "{{ doc('action_data')}}" - - - name: LOGS - description: "{{ doc('logs')}}" - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - - - name: ACTIONS_EVENTS_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/atlas/silver__atlas_accounts_created.sql b/models/silver/atlas/silver__atlas_accounts_created.sql deleted file mode 100644 index e9a2969..0000000 --- a/models/silver/atlas/silver__atlas_accounts_created.sql +++ /dev/null @@ -1,63 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'incremental', - unique_key = 'atlas_account_created_id', - incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - tags = ['atlas'] -) }} - -WITH accts AS ( - - SELECT - receiver_id, - block_timestamp, - modified_timestamp AS _modified_timestamp - FROM - {{ ref('silver__streamline_receipts_final') }} - WHERE - receipt_succeeded - {% if var("MANUAL_FIX") %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - AND _modified_timestamp >= ( - SELECT - MAX(_modified_timestamp) - INTERVAL '2 days' - FROM - {{ this }} - ) - {% endif %} - {% endif %} - - qualify ROW_NUMBER() over ( - PARTITION BY receiver_id - ORDER BY - block_timestamp - ) = 1 -), -FINAL AS ( - SELECT - block_timestamp :: DATE AS "DAY", - COUNT(*) AS wallets_created, - MAX(_modified_timestamp) AS _modified_timestamp - FROM - accts - GROUP BY - 1 -) -SELECT - day, - wallets_created, - _modified_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['DAY'] - ) }} AS atlas_account_created_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL -WHERE - DAY IS NOT NULL diff --git a/models/silver/atlas/silver__atlas_accounts_created.yml b/models/silver/atlas/silver__atlas_accounts_created.yml deleted file mode 100644 index 710c2cf..0000000 --- a/models/silver/atlas/silver__atlas_accounts_created.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_accounts_created - description: |- - Wallet creation on NEAR - columns: - - name: atlas_account_created_id - description: "{{ doc('id')}}" - tests: - - not_null - - unique - - - name: day - description: "{{ doc('date')}}" - tests: - - not_null - - - name: wallets_created - description: "{{ doc('wallets_created')}}" - tests: - - not_null - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - tests: - - not_null - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - tests: - - not_null - - - name: _invocation_id - description: "{{doc('invocation_id')}}" diff --git a/models/silver/atlas/silver__atlas_address_first_action.sql b/models/silver/atlas/silver__atlas_address_first_action.sql deleted file mode 100644 index 2d5653b..0000000 --- a/models/silver/atlas/silver__atlas_address_first_action.sql +++ /dev/null @@ -1,66 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'incremental', - incremental_stratege = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'address', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['atlas'] -) }} - -WITH txs AS ( - - SELECT - tx_signer AS address, - block_id, - block_timestamp, - tx_hash, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp - FROM - {{ ref('silver__streamline_transactions_final') }} - WHERE - {% if var("MANUAL_FIX") %} - {{ partition_load_manual('no_buffer') }} - {% else %} - {% if var('IS_MIGRATION') %} - {{ incremental_load_filter('_inserted_timestamp') }} - {% else %} - {{ incremental_load_filter('_modified_timestamp') }} - {% endif %} - {% endif %} -), -FINAL AS ( - SELECT - address, - MIN(block_timestamp) AS first_tx_timestamp, - MIN(block_id) AS first_tx_block_id, - MIN(_partition_by_block_number) AS _partition_by_block_number, - MIN(_inserted_timestamp) AS _inserted_timestamp, - MIN(_modified_timestamp) AS _modified_timestamp - FROM - txs - GROUP BY - 1 -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['address'] - ) }} AS atlas_address_first_action_id, - address, - first_tx_timestamp, - first_tx_block_id, - _partition_by_block_number, - _inserted_timestamp, - _modified_timestamp, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL qualify ROW_NUMBER() over ( - PARTITION BY address - ORDER BY - first_tx_timestamp - ) = 1 diff --git a/models/silver/atlas/silver__atlas_address_first_action.yml b/models/silver/atlas/silver__atlas_address_first_action.yml deleted file mode 100644 index 80c2f36..0000000 --- a/models/silver/atlas/silver__atlas_address_first_action.yml +++ /dev/null @@ -1,54 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_address_first_action - description: |- - Parses transactions table for the block number and timestamp of a wallets first signed transaction. - - columns: - - name: atlas_address_first_action_id - description: "{{ doc('id') }}" - tests: - - not_null - - unique - - - name: address - description: "{{ doc('address') }}" - tests: - - not_null - - unique - - - name: first_tx_timestamp - description: "{{ doc('block_timestamp')}}" - tests: - - not_null - - - name: first_tx_block_id - description: "{{ doc('block_id') }}" - tests: - - not_null - - - name: _partition_by_block_number - description: "{{ doc('_partition_by_block_number') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - name: not_null_silver__atlas_address_first_action_INSERTED_TIMESTAMP_ - test_name: not_null - - - name: _modified_timestamp - description: "{{ doc('_modified_timestamp') }}" - - - name: inserted_timestamp - description: "{{ doc('inserted_timestamp') }}" - tests: - - not_null - - - name: modified_timestamp - description: "{{ doc('modified_timestamp') }}" - tests: - - not_null - - - name: _invocation_id - description: "{{ doc('invocation_id') }}" diff --git a/models/silver/atlas/silver__atlas_maa.sql b/models/silver/atlas/silver__atlas_maa.sql deleted file mode 100644 index a34b5fc..0000000 --- a/models/silver/atlas/silver__atlas_maa.sql +++ /dev/null @@ -1,109 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'incremental', - incremental_stratege = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'day', - tags = ['atlas'] -) }} - -WITH dates AS ( - - SELECT - date_day AS day - FROM - {{ source( - 'crosschain', - 'dim_dates' - ) }} - -{% if is_incremental() %} -WHERE - date_day > ( - SELECT - MAX(day) - FROM - {{ this }} - ) - AND date_day < SYSDATE() :: DATE -{% else %} -WHERE - date_day BETWEEN '2020-07-22' - AND SYSDATE() :: DATE -{% endif %} -), -signer_first_date AS ( - SELECT - address, - first_tx_timestamp - FROM - {{ ref('silver__atlas_address_first_action') }} -), -txns AS ( - SELECT - block_timestamp :: DATE AS day, - tx_signer, - first_tx_timestamp, - _inserted_timestamp - FROM - {{ ref('silver__streamline_transactions_final') }} - t - LEFT JOIN signer_first_date s - ON t.tx_signer = s.address - - {% if var("MANUAL_FIX") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% else %} - -{% if is_incremental() %} -WHERE - block_timestamp :: DATE >= ( - SELECT - MAX(day) - FROM - {{ this }} - ) - INTERVAL '30 days' -{% endif %} -{% endif %} -), -FINAL AS ( - SELECT - d.day, - COUNT( - DISTINCT tx_signer - ) AS maa, - COUNT( - DISTINCT IFF( - first_tx_timestamp >= d.day - INTERVAL '30 Days' - AND first_tx_timestamp < d.day, - tx_signer, - NULL - ) - ) AS new_maas, - MAX(_inserted_timestamp) AS _inserted_timestamp - FROM - dates d - LEFT JOIN txns t - ON t.day < d.day - AND t.day >= d.day - INTERVAL '30 days' - WHERE - d.day != SYSDATE() :: DATE - GROUP BY - 1 -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['day'] - ) }} AS atlas_maa_id, - day, - maa, - new_maas, - maa - new_maas AS returning_maas, - _inserted_timestamp, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL diff --git a/models/silver/atlas/silver__atlas_maa.yml b/models/silver/atlas/silver__atlas_maa.yml deleted file mode 100644 index e8d98ce..0000000 --- a/models/silver/atlas/silver__atlas_maa.yml +++ /dev/null @@ -1,53 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_maa - description: |- - Monthly Active Accounts (wallets) on NEAR, including new and returning wallets, calculated over a rolling 30 day window. An active account, here, is defined as the signing of at least one transaction. - - columns: - - name: atlas_maa_id - description: "{{ doc('id') }}" - tests: - - not_null - - unique - - - name: day - description: "{{ doc('active_day') }}" - tests: - - not_null - - unique - - - name: maa - description: "{{ doc('maa')}}" - tests: - - not_null - - - name: new_maas - description: "{{ doc('new_maas') }}" - tests: - - not_null - - - name: returning_maas - description: "{{ doc('returning_maas') }}" - tests: - - not_null - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - name: not_null_silver__atlas_near_maa_INSERTED_TIMESTAMP_ - test_name: not_null - - - name: inserted_timestamp - description: "{{ doc('inserted_timestamp') }}" - tests: - - not_null - - - name: modified_timestamp - description: "{{ doc('modified_timestamp') }}" - tests: - - not_null - - - name: _invocation_id - description: "{{ doc('invocation_id') }}" diff --git a/models/silver/atlas/silver__atlas_nft_30_trailing.sql b/models/silver/atlas/silver__atlas_nft_30_trailing.sql deleted file mode 100644 index 8de466e..0000000 --- a/models/silver/atlas/silver__atlas_nft_30_trailing.sql +++ /dev/null @@ -1,51 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'incremental', - unique_key = 'atlas_nft_30_trailing_id', - incremental_strategy = "merge", - merge_exclude_columns = ["inserted_timestamp"], - tags = ['atlas'] -) }} - -WITH date_range AS ( - - SELECT - date_day AS DAY - FROM - {{ ref('silver__dates') }} - WHERE - -{% if is_incremental() %} -date_day >= SYSDATE() - INTERVAL '3 DAY' -{% else %} - date_day >= '2021-01-01' -- first day of data -{% endif %} -AND date_day <= SYSDATE() :: DATE -), -FINAL AS ( - SELECT - d.day AS DAY, - COUNT( - t.tx_hash - ) AS txns - FROM - date_range d - LEFT JOIN {{ ref('silver__atlas_nft_transactions') }} - t - ON t.day BETWEEN d.day - INTERVAL '29 day' - AND d.day - GROUP BY - d.day -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['day'] - ) }} AS atlas_nft_30_trailing_id, - DAY, - txns, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL diff --git a/models/silver/atlas/silver__atlas_nft_30_trailing.yml b/models/silver/atlas/silver__atlas_nft_30_trailing.yml deleted file mode 100644 index df44dee..0000000 --- a/models/silver/atlas/silver__atlas_nft_30_trailing.yml +++ /dev/null @@ -1,36 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_nft_30_trailing - description: |- - This incremental dbt model generates a summary of NFT transactions from the 'silver__atlas_nft_transactions' table. It provides a daily count of transactions, accounting for a 30-day lookback period for each day within the specified date range. - - columns: - - name: atlas_nft_30_trailing_id - description: "{{ doc('id')}}" - tests: - - not_null - - unique - - - name: day - description: "{{ doc('date')}}" - tests: - - not_null - - unique - - name: txns - description: "{{ doc('tx_count')}}" - tests: - - not_null - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - tests: - - not_null - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - tests: - - not_null - - - name: _invocation_id - description: "{{doc('invocation_id')}}" diff --git a/models/silver/atlas/silver__atlas_nft_detailed.sql b/models/silver/atlas/silver__atlas_nft_detailed.sql deleted file mode 100644 index 933b029..0000000 --- a/models/silver/atlas/silver__atlas_nft_detailed.sql +++ /dev/null @@ -1,50 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'table', - unique_key = 'atlas_nft_detailed_id', - tags = ['atlas'] -) }} - -WITH nft_data AS ( - - SELECT - * - FROM - {{ ref('silver__atlas_nft_transactions') }} -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['DAY', 'receiver_id'] - ) }} AS atlas_nft_detailed_id, - DAY, - receiver_id, - COUNT( - DISTINCT token_id - ) AS tokens, - COUNT( - CASE - WHEN method_name = 'nft_transfer' THEN tx_hash - END - ) AS all_transfers, - COUNT( - DISTINCT owner - ) AS owners, - COUNT(*) AS transactions, - COUNT( - CASE - WHEN method_name != 'nft_transfer' THEN tx_hash - END - ) AS mints, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id, - MAX(_inserted_timestamp) AS _inserted_timestamp -FROM - nft_data -GROUP BY - 1, - 2, - 3 -ORDER BY - 4 DESC diff --git a/models/silver/atlas/silver__atlas_nft_detailed.yml b/models/silver/atlas/silver__atlas_nft_detailed.yml deleted file mode 100644 index 7687c0e..0000000 --- a/models/silver/atlas/silver__atlas_nft_detailed.yml +++ /dev/null @@ -1,61 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_nft_detailed - description: |- - This is an incremental dbt model that gives an overview of NFT transactions in NEAR. - -columns: - - name: atlas_nft_detailed_id - description: "{{ doc('id')}}" - tests: - - not_null - - unique - - - name: day - description: "{{ doc('date')}}" - tests: - - not_null - - - name: receiver_id - description: "The identifier of the receiver in the NFT transaction." - tests: - - not_null - - - name: tokens - description: "The count of unique tokens transferred to the receiver on the given day." - tests: - - not_null - - - name: all_transfers - description: "The total number of 'nft_transfer' method transactions that occurred." - tests: - - not_null - - - name: owners - description: "The count of distinct owners who have interacted with the NFT." - tests: - - not_null - - - name: transactions - description: "{{ doc('tx_count')}}" - tests: - - not_null - - - name: mints - description: "The count of transactions where the 'method_name' is not 'nft_transfer', indicating minting actions." - tests: - - not_null - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - tests: - - not_null - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - tests: - - not_null - - - name: _invocation_id - description: "{{doc('invocation_id')}}" diff --git a/models/silver/atlas/silver__atlas_nft_table.sql b/models/silver/atlas/silver__atlas_nft_table.sql deleted file mode 100644 index 2950dd0..0000000 --- a/models/silver/atlas/silver__atlas_nft_table.sql +++ /dev/null @@ -1,57 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'table', - unique_key = 'atlas_nft_table_id', - tags = ['atlas'] -) }} - -WITH nft_data AS ( - - SELECT - * - FROM - {{ ref('silver__atlas_nft_transactions') }} -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['receiver_id'] - ) }} AS atlas_nft_table_id, - receiver_id, - COUNT( - DISTINCT token_id - ) AS tokens, - COUNT( - CASE - WHEN method_name = 'nft_transfer' - AND DAY >= (SYSDATE() :: DATE - INTERVAL '1 day') THEN tx_hash END - ) AS transfers_24h, - COUNT( - CASE - WHEN method_name = 'nft_transfer' - AND DAY >= (SYSDATE() :: DATE - INTERVAL '3 day') THEN tx_hash END - ) AS transfers_3d, - COUNT( - CASE - WHEN method_name = 'nft_transfer' THEN tx_hash - END - ) AS all_transfers, - COUNT( - DISTINCT owner - ) AS owners, - COUNT(*) AS transactions, - COUNT( - CASE - WHEN method_name != 'nft_transfer' THEN tx_hash - END - ) AS mints, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id - FROM - nft_data - GROUP BY - 1, - 2 - ORDER BY - 3 DESC diff --git a/models/silver/atlas/silver__atlas_nft_table.yml b/models/silver/atlas/silver__atlas_nft_table.yml deleted file mode 100644 index 68c28f4..0000000 --- a/models/silver/atlas/silver__atlas_nft_table.yml +++ /dev/null @@ -1,66 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_nft_table - description: |- - This view model provides a breakdown of NFT transaction activities by receiver_id. It includes counts of unique tokens, transfers within the last 24 hours and 3 days, all transfers, unique owners, total transactions, and minting events. - - columns: - - name: atlas_nft_table_id - description: "{ { doc('id')}}" - tests: - - unique - - not_null - - - name: receiver_id - description: "{ { doc('receiver_id')}}" - tests: - - not_null - - - name: tokens - description: "The count of unique tokens that have been received by the receiver_id." - tests: - - not_null - - - name: transfers_24h - description: "The count of 'nft_transfer' transactions that occurred in the last 24 hours." - tests: - - not_null - - - name: transfers_3d - description: "The count of 'nft_transfer' transactions that occurred in the last 3 days." - tests: - - not_null - - - name: all_transfers - description: "The total count of 'nft_transfer' transactions." - tests: - - not_null - - - name: owners - description: "The count of distinct owners that have interacted with the receiver's tokens." - tests: - - not_null - - - name: transactions - description: "{{ doc('tx_count')}}" - tests: - - not_null - - - name: mints - description: "The count of transactions where the method_name indicates a minting event rather than a transfer." - tests: - - not_null - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - tests: - - not_null - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - tests: - - not_null - - - name: _invocation_id - description: "{{doc('invocation_id')}}" diff --git a/models/silver/atlas/silver__atlas_nft_transactions.sql b/models/silver/atlas/silver__atlas_nft_transactions.sql deleted file mode 100644 index e69ada6..0000000 --- a/models/silver/atlas/silver__atlas_nft_transactions.sql +++ /dev/null @@ -1,107 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = "incremental", - cluster_by = ["day"], - unique_key = "atlas_nft_transactions_id", - merge_exclude_columns = ["inserted_timestamp"], - incremental_strategy = "merge", - tags = ['atlas'] -) }} - -WITH nft_mints AS ( - - SELECT - block_timestamp :: DATE AS DAY, - receipt_object_id, - tx_hash, - method_name, - receiver_id, - signer_id, - owner_id AS owner, - token_id, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp - FROM - {{ ref('silver__standard_nft_mint_s3') }} - WHERE - {% if var("MANUAL_FIX") %} - {{ partition_load_manual('no_buffer') }} - {% else %} - {% if var('IS_MIGRATION') %} - {{ incremental_load_filter('_inserted_timestamp') }} - {% else %} - {{ incremental_load_filter('_modified_timestamp') }} - {% endif %} - {% endif %} -), -nft_transfers AS ( - SELECT - block_timestamp :: DATE AS DAY, - SPLIT( - action_id, - '-' - ) [0] :: STRING AS receipt_object_id, - tx_hash, - method_name, - receiver_id, - signer_id, - args ['receiver_id'] AS owner, - args ['token_id'] AS token_id, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp - FROM - {{ ref('silver__actions_events_function_call_s3') }} - WHERE - method_name = 'nft_transfer' - AND {% if var("MANUAL_FIX") %} - {{ partition_load_manual('no_buffer') }} - {% else %} - {% if var('IS_MIGRATION') %} - {{ incremental_load_filter('_inserted_timestamp') }} - {% else %} - {{ incremental_load_filter('_modified_timestamp') }} - {% endif %} - {% endif %} -), -unioned_nft_data AS ( - SELECT - * - FROM - nft_mints - UNION ALL - SELECT - * - FROM - nft_transfers -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['receipt_object_id', 'method_name', 'token_id', 'owner'] - ) }} AS atlas_nft_transactions_id, - DAY, - tx_hash, - method_name, - receiver_id, - signer_id, - owner, - token_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id, - _partition_by_block_number, - _inserted_timestamp, - _modified_timestamp -FROM - unioned_nft_data -WHERE - -- failed receipts may have unparsable base64 FunctionCall args - token_id IS NOT NULL - AND owner IS NOT NULL -qualify ROW_NUMBER() over ( - PARTITION BY atlas_nft_transactions_id - ORDER BY - _inserted_timestamp DESC -) = 1 diff --git a/models/silver/atlas/silver__atlas_nft_transactions.yml b/models/silver/atlas/silver__atlas_nft_transactions.yml deleted file mode 100644 index 620c8a6..0000000 --- a/models/silver/atlas/silver__atlas_nft_transactions.yml +++ /dev/null @@ -1,77 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_nft_transactions - description: |- - This incremental dbt model unifies NFT minting and transfer data into a single view, providing a comprehensive look at NFT activities. It captures daily activities by transaction hash, method name, receiver ID, signer ID, owner, and token ID. - tests: - - dbt_utils.recency: - datepart: day - field: _inserted_timestamp - interval: 1 - - columns: - - name: atlas_nft_transactions_id - description: "{{doc('id')}}" - tests: - - unique - - not_null - - - name: day - description: "{{doc('date')}}" - tests: - - not_null - - - name: tx_hash - description: "{{doc('tx_hash')}}" - tests: - - not_null - - - name: method_name - description: "{{doc('method_name')}}" - tests: - - not_null - - - name: receiver_id - description: "{{doc('receiver_id')}}" - tests: - - not_null - - - name: signer_id - description: "{{doc('signer_id')}}" - tests: - - not_null - - - name: owner - description: "{{doc('owner')}}" - tests: - - not_null - - - name: token_id - description: "{{doc('token_id')}}" - tests: - - not_null - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - tests: - - not_null - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - tests: - - not_null - - - name: _invocation_id - description: "{{doc('invocation_id')}}" - - - name: _partition_by_block_number - description: "{{doc('_partition_by_block_number')}}" - - - name: _inserted_timestamp - description: "{{doc('_inserted_timestamp')}}" - tests: - - not_null - - - name: _modified_timestamp - description: "{{doc('_modified_timestamp')}}" \ No newline at end of file diff --git a/models/silver/atlas/supply/silver__atlas_supply_epochs.sql b/models/silver/atlas/supply/silver__atlas_supply_epochs.sql index 11ad3dd..f6f15f1 100644 --- a/models/silver/atlas/supply/silver__atlas_supply_epochs.sql +++ b/models/silver/atlas/supply/silver__atlas_supply_epochs.sql @@ -8,14 +8,23 @@ WITH blocks AS ( SELECT - * + block_id, + block_timestamp, + block_author, + header_json :total_supply :: NUMBER AS total_supply, + header_json :epoch_id :: STRING AS epoch_id, + _partition_by_block_number FROM - {{ ref('silver__streamline_blocks') }} - WHERE + {{ ref('silver__blocks_final') }} {% if var("MANUAL_FIX") %} - {{ partition_load_manual('no_buffer') }} + WHERE {{ partition_load_manual('no_buffer') }} {% else %} - {{ incremental_load_filter('_inserted_timestamp') }} + WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) {% endif %} ), epochs AS ( diff --git a/models/silver/atlas/supply/silver__atlas_supply_lockup_receipts.sql b/models/silver/atlas/supply/silver__atlas_supply_lockup_receipts.sql index 5f065ae..18b72da 100644 --- a/models/silver/atlas/supply/silver__atlas_supply_lockup_receipts.sql +++ b/models/silver/atlas/supply/silver__atlas_supply_lockup_receipts.sql @@ -10,27 +10,27 @@ WITH receipts AS ( SELECT - receipt_object_id, + receipt_id AS receipt_object_id, tx_hash, block_timestamp, - receipt_actions, + receipt_json AS receipt_actions, receiver_id, - status_value, - logs, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp + predecessor_id, + outcome_json :outcome :status :: VARIANT AS status_value, + outcome_json :outcome :logs :: ARRAY AS logs, + _partition_by_block_number FROM - {{ ref('silver__streamline_receipts_final') }} - WHERE + {{ ref('silver__receipts_final') }} {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} {% else %} - {% if var('IS_MIGRATION') %} - {{ incremental_load_filter('_inserted_timestamp') }} - {% else %} - {{ incremental_load_filter('_modified_timestamp') }} - {% endif %} + WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) {% endif %} ), FINAL AS ( @@ -38,16 +38,14 @@ FINAL AS ( receipt_object_id, tx_hash, block_timestamp, - receipt_actions :predecessor_id :: STRING AS predecessor_id, + predecessor_id, receiver_id, receipt_actions AS actions, object_keys( status_value ) [0] :: STRING AS status, logs, - _partition_by_block_number, - _inserted_timestamp, - _modified_timestamp + _partition_by_block_number FROM receipts WHERE diff --git a/models/silver/core/lake/helpers/_retry_range.sql b/models/silver/core/lake/helpers/_retry_range.sql index ccfce15..3a84e43 100644 --- a/models/silver/core/lake/helpers/_retry_range.sql +++ b/models/silver/core/lake/helpers/_retry_range.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'ephemeral', - tags = ['helper', 'receipt_map','scheduled_core'] + tags = ['helper', 'receipt_map'] ) }} SELECT diff --git a/models/silver/core/lake/helpers/silver__flatten_receipts.sql b/models/silver/core/lake/helpers/silver__flatten_receipts.sql index 64997c8..a24f26d 100644 --- a/models/silver/core/lake/helpers/silver__flatten_receipts.sql +++ b/models/silver/core/lake/helpers/silver__flatten_receipts.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - tags = ['helper', 'receipt_map','scheduled_core'] + tags = ['helper', 'receipt_map'] ) }} WITH receipts AS ( diff --git a/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql b/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql index 568609f..8801b6c 100644 --- a/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql +++ b/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql @@ -1,7 +1,7 @@ {{ config( materalized = 'view', unique_key = 'receipt_id', - tags = ['helper', 'receipt_map','scheduled_core'] + tags = ['helper', 'receipt_map'] ) }} WITH diff --git a/models/silver/core/lake/silver__streamline_blocks.sql b/models/silver/core/lake/silver__streamline_blocks.sql index e76813b..27dd868 100644 --- a/models/silver/core/lake/silver__streamline_blocks.sql +++ b/models/silver/core/lake/silver__streamline_blocks.sql @@ -6,7 +6,7 @@ merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE', '_partition_by_block_number'], unique_key = 'block_id', - tags = ['load', 'load_blocks','scheduled_core'], + tags = ['load', 'load_blocks', 'deprecated_lake_archive'], full_refresh = False ) }} diff --git a/models/silver/core/lake/silver__streamline_receipts.sql b/models/silver/core/lake/silver__streamline_receipts.sql index 1da7fc2..283952c 100644 --- a/models/silver/core/lake/silver__streamline_receipts.sql +++ b/models/silver/core/lake/silver__streamline_receipts.sql @@ -5,7 +5,7 @@ merge_exclude_columns = ['inserted_timestamp'], unique_key = 'receipt_id', cluster_by = ['modified_timestamp::date', '_partition_by_block_number'], - tags = ['load', 'load_shards','scheduled_core'] + tags = ['load', 'load_shards', 'deprecated_lake_archive'] ) }} WITH shards AS ( diff --git a/models/silver/core/lake/silver__streamline_receipts_final.sql b/models/silver/core/lake/silver__streamline_receipts_final.sql index 260dbf5..b48a2a5 100644 --- a/models/silver/core/lake/silver__streamline_receipts_final.sql +++ b/models/silver/core/lake/silver__streamline_receipts_final.sql @@ -6,7 +6,7 @@ unique_key = 'receipt_object_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number', ], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", - tags = ['receipt_map','scheduled_core'], + tags = ['receipt_map', 'deprecated_lake_archive'], full_refresh = False ) }} diff --git a/models/silver/core/lake/silver__streamline_shards.sql b/models/silver/core/lake/silver__streamline_shards.sql index e7682bb..be6e9f6 100644 --- a/models/silver/core/lake/silver__streamline_shards.sql +++ b/models/silver/core/lake/silver__streamline_shards.sql @@ -5,7 +5,7 @@ merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['_inserted_timestamp::DATE', '_partition_by_block_number'], unique_key = 'shard_id', - tags = ['load', 'load_shards','scheduled_core'], + tags = ['load', 'load_shards', 'deprecated_lake_archive'], full_refresh = False ) }} diff --git a/models/silver/core/lake/silver__streamline_transactions.sql b/models/silver/core/lake/silver__streamline_transactions.sql index a4502d7..53a51fe 100644 --- a/models/silver/core/lake/silver__streamline_transactions.sql +++ b/models/silver/core/lake/silver__streamline_transactions.sql @@ -5,7 +5,7 @@ merge_exclude_columns = ['inserted_timestamp'], unique_key = 'tx_hash', cluster_by = ['modified_timestamp::date', '_partition_by_block_number'], - tags = ['load', 'load_shards','scheduled_core'] + tags = ['load', 'load_shards', 'deprecated_lake_archive'] ) }} WITH chunks AS ( diff --git a/models/silver/core/lake/silver__streamline_transactions_final.sql b/models/silver/core/lake/silver__streamline_transactions_final.sql index b2d772d..1ff853c 100644 --- a/models/silver/core/lake/silver__streamline_transactions_final.sql +++ b/models/silver/core/lake/silver__streamline_transactions_final.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'tx_hash', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number'], - tags = ['receipt_map','scheduled_core'] + tags = ['receipt_map', 'deprecated_lake_archive'] ) }} WITH int_txs AS ( diff --git a/models/silver/core/migration/_migrate_blocks.sql b/models/silver/core/migration/_migrate_blocks.sql new file mode 100644 index 0000000..6226e76 --- /dev/null +++ b/models/silver/core/migration/_migrate_blocks.sql @@ -0,0 +1,34 @@ +{{ config( + materialized = 'ephemeral' +) }} + +SELECT + block_id, + block_timestamp, + block_hash, + prev_hash, + block_author, + chunks AS chunks_json, + header AS header_json, + _partition_by_block_number, + {{ dbt_utils.generate_surrogate_key( + ['block_id'] + ) }} AS blocks_final_id, + COALESCE( + inserted_timestamp, + _inserted_timestamp, + _load_timestamp + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + _inserted_timestamp, + _load_timestamp + ) AS modified_timestamp, + _invocation_id +FROM + {{ ref('silver__streamline_blocks') }} + + {% if var("NEAR_MIGRATE_ARCHIVE") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% endif %} diff --git a/models/silver/core/migration/_migrate_receipts.sql b/models/silver/core/migration/_migrate_receipts.sql new file mode 100644 index 0000000..9d0ed8e --- /dev/null +++ b/models/silver/core/migration/_migrate_receipts.sql @@ -0,0 +1,79 @@ +{{ config( + materialized = 'ephemeral' +) }} + +WITH lake_receipts_final AS ( + + SELECT + chunk_hash, + block_id, + block_timestamp, + tx_hash, + COALESCE( + receipt_id, + receipt_object_id + ) AS receipt_id, + COALESCE( + predecessor_id, + receipt_actions :predecessor_id :: STRING + ) AS predecessor_id, + receiver_id, + receipt_actions AS receipt_json, + execution_outcome AS outcome_json, + receipt_succeeded, + _partition_by_block_number, + {{ dbt_utils.generate_surrogate_key( + ['COALESCE(receipt_id, receipt_object_id)'] + ) }} AS receipts_final_id, + COALESCE( + inserted_timestamp, + _inserted_timestamp, + _load_timestamp + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + _inserted_timestamp, + _load_timestamp + ) AS modified_timestamp, + _invocation_id + FROM + {{ ref('silver__streamline_receipts_final') }} + + {% if var("NEAR_MIGRATE_ARCHIVE") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% endif %} +), +lake_transactions_final AS ( + SELECT + tx_hash, + tx_succeeded + FROM + {{ ref('silver__streamline_transactions_final') }} + + {% if var("NEAR_MIGRATE_ARCHIVE") %} + WHERE + {{ partition_load_manual('front') }} + {% endif %} +) +SELECT + chunk_hash, + block_id, + block_timestamp, + r.tx_hash, + receipt_id, + predecessor_id, + receiver_id, + receipt_json, + outcome_json, + tx_succeeded, + receipt_succeeded, + _partition_by_block_number, + receipts_final_id, + inserted_timestamp, + modified_timestamp, + _invocation_id +FROM + lake_receipts_final r + LEFT JOIN lake_transactions_final tx + ON r.tx_hash = tx.tx_hash diff --git a/models/silver/core/migration/_migrate_txs.sql b/models/silver/core/migration/_migrate_txs.sql new file mode 100644 index 0000000..c112321 --- /dev/null +++ b/models/silver/core/migration/_migrate_txs.sql @@ -0,0 +1,86 @@ +{{ config( + materialized = 'ephemeral' +) }} + +WITH lake_transactions_final AS ( + + SELECT + block_id, + block_timestamp, + tx_hash, + tx_signer, + tx_receiver, + tx_succeeded, + gas_used, + transaction_fee, + attached_gas, + _partition_by_block_number, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS transactions_final_id, + COALESCE( + inserted_timestamp, + _inserted_timestamp, + _load_timestamp + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + _inserted_timestamp, + _load_timestamp + ) AS modified_timestamp, + _invocation_id + FROM + {{ ref('silver__streamline_transactions_final') }} + + {% if var("NEAR_MIGRATE_ARCHIVE") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% endif %} +), +lake_transactions_int AS ( + SELECT + tx_hash, + block_id, + shard_number, + chunk_hash, + tx :transaction :: variant AS transaction_json, + tx :outcome :execution_outcome :: variant AS outcome_json, + _partition_by_block_number + FROM + {{ ref('silver__streamline_transactions') }} + + {% if var("NEAR_MIGRATE_ARCHIVE") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% endif %} +), +transaction_archive AS ( + SELECT + i.chunk_hash, + i.shard_number AS shard_id, + f.block_id, + f.block_timestamp, + f.tx_hash, + f.tx_signer, + f.tx_receiver, + i.transaction_json, + i.outcome_json, + f.tx_succeeded, + f.gas_used, + f.transaction_fee, + f.attached_gas, + f._partition_by_block_number, + f.transactions_final_id, + f.inserted_timestamp, + f.modified_timestamp, + f._invocation_id + FROM + lake_transactions_final f + LEFT JOIN lake_transactions_int i + ON f.tx_hash = i.tx_hash + AND f._partition_by_block_number = i._partition_by_block_number +) +SELECT + * +FROM + transaction_archive diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql new file mode 100644 index 0000000..6ae3975 --- /dev/null +++ b/models/silver/core/silver__blocks_final.sql @@ -0,0 +1,70 @@ +{{ config( + materialized = 'incremental', + incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + unique_key = 'block_id', + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_id, block_hash);", + tags = ['scheduled_core', 'core_v2'], + full_refresh = false +) }} + +{% if var('NEAR_MIGRATE_ARCHIVE', False) %} + {% if execute %} + {% do log('Migrating blocks ' ~ var('RANGE_START') ~ ' to ' ~ var('RANGE_END'), info=True) %} + {% do log('Invocation ID: ' ~ invocation_id, info=True) %} + {% endif %} + SELECT + block_id, + block_timestamp, + block_hash, + prev_hash, + block_author, + chunks_json, + header_json, + _partition_by_block_number, + blocks_final_id, + inserted_timestamp, + modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + {{ ref('_migrate_blocks') }} + +{% else %} + +WITH blocks AS ( + SELECT + block_id, + block_timestamp, + block_hash, + block_json :header :prev_hash :: STRING AS prev_hash, + block_json :author :: STRING AS block_author, + block_json :chunks :: ARRAY AS chunks_json, + block_json :header :: OBJECT AS header_json, + partition_key AS _partition_by_block_number + FROM + {{ ref('silver__blocks_v2') }} + + {% if is_incremental() %} + WHERE + modified_timestamp >= ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01') + FROM + {{ this }} + ) + {% endif %} +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['block_id'] + ) }} AS blocks_final_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + blocks + +{% endif %} diff --git a/models/silver/core/silver__blocks_final.yml b/models/silver/core/silver__blocks_final.yml new file mode 100644 index 0000000..75eeae9 --- /dev/null +++ b/models/silver/core/silver__blocks_final.yml @@ -0,0 +1,75 @@ +version: 2 + +models: + - name: silver__blocks_final + description: |- + Table containing blocks for NEAR. + tests: + - dbt_utils.recency: + datepart: hour + field: inserted_timestamp + interval: 1 + + columns: + - name: block_id + description: "{{ doc('block_id') }}" + tests: + - not_null + - unique + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + + - name: block_hash + description: "{{ doc('block_hash') }}" + tests: + - not_null + - unique + + - name: prev_hash + description: "{{ doc('prev_hash') }}" + tests: + - not_null + - unique + + - name: block_author + description: "{{ doc('block_author') }}" + tests: + - not_null + + - name: chunks_json + description: "{{ doc('chunks_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - ARRAY + - OBJECT + + - name: header_json + description: "{{ doc('header_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: _partition_by_block_number + description: "{{ doc('_partition_by_block_number') }}" + + - name: blocks_final_id + description: "{{ doc('id') }}" + tests: + - not_null + - unique + + - name: inserted_timestamp + description: "{{ doc('inserted_timestamp') }}" + + - name: modified_timestamp + description: "{{ doc('modified_timestamp') }}" + + - name: _invocation_id + diff --git a/models/silver/core/silver__blocks_v2.sql b/models/silver/core/silver__blocks_v2.sql index 159cd50..7da6dc4 100644 --- a/models/silver/core/silver__blocks_v2.sql +++ b/models/silver/core/silver__blocks_v2.sql @@ -5,8 +5,7 @@ incremental_strategy = 'merge', incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], unique_key = "block_hash", - cluster_by = ['modified_timestamp::DATE','partition_key'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_hash)", + cluster_by = ['modified_timestamp::DATE','block_timestamp::date'], tags = ['scheduled_core', 'core_v2'] ) }} diff --git a/models/silver/core/silver__chunks_v2.sql b/models/silver/core/silver__chunks_v2.sql index fbcf1cf..229e318 100644 --- a/models/silver/core/silver__chunks_v2.sql +++ b/models/silver/core/silver__chunks_v2.sql @@ -5,8 +5,7 @@ incremental_strategy = 'merge', incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], unique_key = "chunk_hash", - cluster_by = ['modified_timestamp::DATE','partition_key'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(chunk_hash)", + cluster_by = ['modified_timestamp::DATE','block_timestamp::date'], tags = ['scheduled_core', 'core_v2'] ) }} @@ -15,8 +14,7 @@ WITH bronze_chunks AS ( SELECT VALUE :BLOCK_ID :: INT AS block_id, VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS block_timestamp_epoch, - DATA :header :shard_id :: INT AS shard_id, - DATA :header :chunk_hash :: STRING AS chunk_hash, + VALUE :CHUNK_HASH :: STRING AS chunk_hash, partition_key, DATA :: variant AS chunk_json, _inserted_timestamp @@ -38,11 +36,10 @@ WHERE {% endif %} ) SELECT + chunk_hash, block_id, block_timestamp_epoch, TO_TIMESTAMP_NTZ(block_timestamp_epoch, 9) AS block_timestamp, - shard_id, - chunk_hash, partition_key, chunk_json, _inserted_timestamp, diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql new file mode 100644 index 0000000..cdc26d7 --- /dev/null +++ b/models/silver/core/silver__receipts_final.sql @@ -0,0 +1,248 @@ +{{ config( + materialized = 'incremental', + incremental_predicates = ["dynamic_range_predicate_custom","block_timestamp::date"], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + unique_key = 'receipt_id', + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", + tags = ['scheduled_core', 'core_v2'], + full_refresh = false +) }} + +{% if var('NEAR_MIGRATE_ARCHIVE', False) %} + {% if execute %} + {% do log('Migrating receipts ' ~ var('RANGE_START') ~ ' to ' ~ var('RANGE_END'), info=True) %} + {% do log('Invocation ID: ' ~ invocation_id, info=True) %} + {% endif %} + + SELECT + chunk_hash, + block_id, + block_timestamp, + tx_hash, + receipt_id, + predecessor_id, + receiver_id, + receipt_json, + outcome_json, + tx_succeeded, + receipt_succeeded, + _partition_by_block_number, + receipts_final_id, + inserted_timestamp, + modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + {{ ref('_migrate_receipts') }} + +{% else %} + {% if execute and not var("MANUAL_FIX") %} + {% if is_incremental() %} + {% set max_mod_query %} + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01') modified_timestamp + FROM + {{ this }} + {% endset %} + + {% set max_mod = run_query(max_mod_query) [0] [0] %} + {% do log('max_mod: ' ~ max_mod, info=True) %} + {% set min_block_date_query %} + SELECT + MIN(origin_block_timestamp :: DATE) block_timestamp + FROM + {{ ref('silver__transactions_v2') }} + WHERE + modified_timestamp >= '{{max_mod}}' + {% endset %} + + {% set min_bd = run_query(min_block_date_query) [0] [0] %} + {% do log('min_bd: ' ~ min_bd, info=True) %} + {% if not min_bd or min_bd == 'None' %} + {% set min_bd = '2099-01-01' %} + {% do log('min_bd: ' ~ min_bd, info=True) %} + {% endif %} + + {% do log('min_block_date: ' ~ min_bd, info=True) %} + + {% endif %} + {% endif %} + +WITH txs_with_receipts AS ( + SELECT + chunk_hash, + origin_block_id, + origin_block_timestamp, + tx_hash, + response_json, + response_json :transaction_outcome :outcome :receipt_ids [0] :: STRING AS initial_receipt_id, + response_json :status :Failure IS NULL AS tx_succeeded, + partition_key AS _partition_by_block_number, + modified_timestamp + FROM + {{ ref('silver__transactions_v2') }} + {% if var("MANUAL_FIX") %} + WHERE + {{ partition_load_manual('no_buffer', 'partition_key') }} + {% else %} + {% if is_incremental() %} + WHERE origin_block_timestamp :: DATE >= '{{min_bd}}' + {% endif %} + {% endif %} +), +blocks AS ( + SELECT + block_id, + block_hash, + block_timestamp, + modified_timestamp + FROM + {{ ref('silver__blocks_v2') }} + {% if var("MANUAL_FIX") %} + WHERE + {{ partition_load_manual('no_buffer', 'partition_key') }} + {% else %} + {% if is_incremental() %} + WHERE block_timestamp :: DATE >= '{{min_bd}}' :: DATE + {% endif %} + {% endif %} +), +flatten_receipts AS ( + SELECT + origin_block_timestamp, + chunk_hash, + tx_hash, + tx_succeeded, + VALUE :receipt_id :: STRING AS receipt_id, + VALUE :: variant AS receipt_json, + _partition_by_block_number, + modified_timestamp + FROM + txs_with_receipts, + LATERAL FLATTEN( + input => response_json :receipts :: ARRAY + ) +), +flatten_receipt_outcomes AS ( + SELECT + VALUE :block_hash :: STRING AS block_hash, + tx_hash, + VALUE :id :: STRING AS receipt_id, + VALUE :: variant AS outcome_json + FROM + txs_with_receipts, + LATERAL FLATTEN( + input => response_json :receipts_outcome :: ARRAY + ) +), +receipts_full AS ( + SELECT + chunk_hash, + ro.block_hash, + block_id, + block_timestamp, + r.tx_hash, + r.receipt_id, + receipt_json, + outcome_json, + tx_succeeded, + _partition_by_block_number + FROM + flatten_receipts r + LEFT JOIN flatten_receipt_outcomes ro + ON r.receipt_id = ro.receipt_id + LEFT JOIN blocks b + ON ro.block_hash = b.block_hash + {% if is_incremental() and not var("MANUAL_FIX") %} + WHERE + GREATEST( + COALESCE(r.modified_timestamp, '1970-01-01'), + COALESCE(b.modified_timestamp, '1970-01-01') + ) >= '{{max_mod}}' + {% endif %} +), +initial_receipt_full AS ( + SELECT + chunk_hash, + origin_block_id AS block_id, + origin_block_timestamp AS block_timestamp, + txr.tx_hash, + initial_receipt_id AS receipt_id, + OBJECT_CONSTRUCT( + 'predecessor_id', response_json :transaction :signer_id :: STRING, + 'priority', response_json :transaction :priority_fee :: INTEGER, + 'receipt', OBJECT_CONSTRUCT( + 'Action', OBJECT_CONSTRUCT( + 'actions', response_json :transaction :actions :: ARRAY, + 'gas_price', Null, + 'input_data_ids', Null, + 'is_promise_yield', Null, + 'output_data_receivers', Null, + 'signer_id', response_json :transaction :signer_id :: STRING, + 'signer_public_key', response_json :transaction :public_key :: STRING + ) + ), + 'receipt_id', initial_receipt_id :: STRING, + 'receiver_id', response_json :transaction :receiver_id :: STRING + ) AS receipt_json, + outcome_json, + tx_succeeded, + _partition_by_block_number + FROM + txs_with_receipts txr + LEFT JOIN flatten_receipt_outcomes ro + ON txr.initial_receipt_id = ro.receipt_id + AND txr.tx_hash = ro.tx_hash + {% if is_incremental() and not var("MANUAL_FIX") %} + WHERE + modified_timestamp >= '{{max_mod}}' + {% endif %} +), +FINAL AS ( + SELECT + chunk_hash, + block_id, + block_timestamp, + tx_hash, + receipt_id, + receipt_json :predecessor_id :: STRING AS predecessor_id, + receipt_json :receiver_id :: STRING AS receiver_id, + receipt_json, + outcome_json, + tx_succeeded, + outcome_json :outcome :status :Failure IS NULL AS receipt_succeeded, + _partition_by_block_number + FROM + receipts_full + UNION ALL + SELECT + chunk_hash, + block_id, + block_timestamp, + tx_hash, + receipt_id, + receipt_json :predecessor_id :: STRING AS predecessor_id, + receipt_json :receiver_id :: STRING AS receiver_id, + receipt_json, + outcome_json, + tx_succeeded, + tx_succeeded AS receipt_succeeded, + _partition_by_block_number + FROM + initial_receipt_full +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['receipt_id'] + ) }} AS receipts_final_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL + +qualify(row_number() over (partition by receipt_id order by block_id is not null desc, modified_timestamp desc) = 1) + +{% endif %} diff --git a/models/silver/core/silver__receipts_final.yml b/models/silver/core/silver__receipts_final.yml new file mode 100644 index 0000000..0d8f2f7 --- /dev/null +++ b/models/silver/core/silver__receipts_final.yml @@ -0,0 +1,97 @@ +version: 2 + +models: + - name: silver__receipts_final + description: |- + Table containing transaction receipts for NEAR blockchain. + tests: + - dbt_utils.recency: + datepart: hour + field: inserted_timestamp + interval: 1 + + columns: + - name: chunk_hash + description: "{{ doc('chunk_hash') }}" + tests: + - not_null + + - name: block_id + description: "{{ doc('block_id') }}" + tests: + - not_null + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + + - name: tx_hash + description: "{{ doc('tx_hash') }}" + tests: + - not_null + + - name: receipt_id + description: "{{ doc('receipt_id') }}" + tests: + - not_null + - unique + + - name: predecessor_id + description: "{{ doc('predecessor_id') }}" + tests: + - not_null + + - name: receiver_id + description: "{{ doc('receiver_id') }}" + tests: + - not_null + + - name: receipt_json + description: "{{ doc('receipt_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: outcome_json + description: "{{ doc('outcome_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: tx_succeeded + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + + - name: receipt_succeeded + description: "{{ doc('receipt_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + + - name: _partition_by_block_number + description: "{{ doc('_partition_by_block_number') }}" + + - name: receipts_final_id + description: "{{ doc('id') }}" + tests: + - not_null + + - name: inserted_timestamp + description: "{{ doc('inserted_timestamp') }}" + + - name: modified_timestamp + description: "{{ doc('modified_timestamp') }}" + + - name: _invocation_id + description: "{{ doc('invocation_id') }}" diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql new file mode 100644 index 0000000..5cdf435 --- /dev/null +++ b/models/silver/core/silver__transactions_final.sql @@ -0,0 +1,146 @@ +{{ config( + materialized = 'incremental', + incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + unique_key = 'tx_hash', + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,tx_signer,tx_receiver);", + tags = ['scheduled_core', 'core_v2'], + full_refresh = false +) }} + +{% if var('NEAR_MIGRATE_ARCHIVE', False) %} + {% if execute %} + {% do log('Migrating transactions ' ~ var('RANGE_START') ~ ' to ' ~ var('RANGE_END'), info=True) %} + {% do log('Invocation ID: ' ~ invocation_id, info=True) %} + {% endif %} + + SELECT + chunk_hash, + block_id, + block_timestamp, + tx_hash, + tx_receiver, + tx_signer, + transaction_json, + outcome_json, + OBJECT_CONSTRUCT() AS status_json, + tx_succeeded, + gas_used, + transaction_fee, + attached_gas, + _partition_by_block_number, + transactions_final_id, + inserted_timestamp, + modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + {{ ref('_migrate_txs') }} + + {% else %} + +WITH txs_with_receipts AS ( + SELECT + chunk_hash, + origin_block_id AS block_id, + origin_block_timestamp AS block_timestamp, + tx_hash, + response_json :transaction :: variant AS transaction_json, + response_json :transaction_outcome :outcome :: variant AS outcome_json, + response_json :status :: variant AS status_json, + response_json :receipts_outcome :: ARRAY AS receipts_outcome_json, + response_json :status :Failure IS NULL AS tx_succeeded, + partition_key AS _partition_by_block_number + FROM + {{ ref('silver__transactions_v2') }} + + {% if is_incremental() %} + WHERE + modified_timestamp >= ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01') + FROM + {{ this }} + ) + {% endif %} + +), +determine_receipt_gas_burnt AS ( + SELECT + tx_hash, + SUM( + ZEROIFNULL(VALUE :outcome :gas_burnt :: INT) + ) AS total_gas_burnt_receipts, + SUM( + ZEROIFNULL(VALUE :outcome :tokens_burnt :: INT) + ) AS total_tokens_burnt_receipts + FROM + txs_with_receipts, + LATERAL FLATTEN ( + input => receipts_outcome_json + ) + GROUP BY + 1 +), +determine_attached_gas AS ( + SELECT + tx_hash, + SUM( + VALUE :FunctionCall :gas :: INT + ) AS total_attached_gas + FROM + txs_with_receipts, + LATERAL FLATTEN ( + input => transaction_json :actions :: ARRAY + ) + GROUP BY + 1 +), +transactions_final AS ( + SELECT + chunk_hash, + block_id, + block_timestamp, + t.tx_hash, + transaction_json, + outcome_json, + status_json, + total_gas_burnt_receipts, + total_tokens_burnt_receipts, + total_attached_gas, + tx_succeeded, + _partition_by_block_number + FROM + txs_with_receipts t + LEFT JOIN determine_receipt_gas_burnt d USING (tx_hash) + LEFT JOIN determine_attached_gas A USING (tx_hash) +) +SELECT + chunk_hash, + block_id, + block_timestamp, + tx_hash, + transaction_json :receiver_id :: STRING AS tx_receiver, + transaction_json :signer_id :: STRING AS tx_signer, + transaction_json, + outcome_json, + status_json, + tx_succeeded, + ZEROIFNULL(outcome_json :gas_burnt :: INT) + total_gas_burnt_receipts AS gas_used, + ZEROIFNULL(outcome_json :tokens_burnt :: INT) + total_tokens_burnt_receipts AS transaction_fee, + COALESCE( + total_attached_gas, + gas_used + ) AS attached_gas, + _partition_by_block_number, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS transactions_final_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + transactions_final + +{% endif %} diff --git a/models/silver/core/silver__transactions_final.yml b/models/silver/core/silver__transactions_final.yml new file mode 100644 index 0000000..c728306 --- /dev/null +++ b/models/silver/core/silver__transactions_final.yml @@ -0,0 +1,119 @@ +version: 2 + +models: + - name: silver__transactions_final + description: |- + Table containing finalized transactions for NEAR blockchain. + tests: + - dbt_utils.recency: + datepart: hour + field: inserted_timestamp + interval: 1 + + columns: + - name: chunk_hash + description: "{{ doc('chunk_hash') }}" + tests: + - not_null + + - name: block_id + description: "{{ doc('block_id') }}" + tests: + - not_null + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + + - name: tx_hash + description: "{{ doc('tx_hash') }}" + tests: + - not_null + - unique + + - name: tx_receiver + description: "{{ doc('tx_receiver') }}" + tests: + - not_null + + - name: tx_signer + description: "{{ doc('tx_signer') }}" + tests: + - not_null + + - name: transaction_json + description: "{{ doc('transaction_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: outcome_json + description: "{{ doc('outcome_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: status_json + description: "{{ doc('status_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: tx_succeeded + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + + - name: gas_used + description: "{{ doc('gas_used') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: transaction_fee + description: "{{ doc('transaction_fee') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: attached_gas + description: "{{ doc('attached_gas') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: _partition_by_block_number + description: "{{ doc('_partition_by_block_number') }}" + + - name: transactions_final_id + description: "{{ doc('id') }}" + tests: + - not_null + + - name: inserted_timestamp + description: "{{ doc('inserted_timestamp') }}" + + - name: modified_timestamp + description: "{{ doc('modified_timestamp') }}" + + - name: _invocation_id + description: "{{ doc('invocation_id') }}" diff --git a/models/silver/core/silver__transactions_v2.sql b/models/silver/core/silver__transactions_v2.sql index a606990..b17d475 100644 --- a/models/silver/core/silver__transactions_v2.sql +++ b/models/silver/core/silver__transactions_v2.sql @@ -3,18 +3,21 @@ {{ config ( materialized = "incremental", incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], + incremental_predicates = ["dynamic_range_predicate","origin_block_timestamp::date"], unique_key = "tx_hash", - cluster_by = ['modified_timestamp::DATE','partition_key'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_hash)", + cluster_by = ['modified_timestamp::DATE','origin_block_timestamp::date'], tags = ['scheduled_core', 'core_v2'] ) }} WITH bronze_transactions AS ( SELECT - VALUE :BLOCK_ID :: INT AS block_id, - VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS block_timestamp_epoch, + VALUE :BLOCK_ID :: INT AS origin_block_id, + VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS origin_block_timestamp_epoch, + VALUE :SHARD_ID :: INT AS shard_id, + VALUE :CHUNK_HASH :: STRING AS chunk_hash, + VALUE :HEIGHT_CREATED :: INT AS chunk_height_created, + VALUE :HEIGHT_INCLUDED :: INT AS chunk_height_included, DATA :transaction :hash :: STRING AS tx_hash, DATA :transaction :signer_id :: STRING AS signer_id, partition_key, @@ -38,9 +41,13 @@ WHERE {% endif %} ) SELECT - block_id, - block_timestamp_epoch, - TO_TIMESTAMP_NTZ(block_timestamp_epoch, 9) AS block_timestamp, + origin_block_id, + origin_block_timestamp_epoch, + TO_TIMESTAMP_NTZ(origin_block_timestamp_epoch, 9) AS origin_block_timestamp, + shard_id, + chunk_hash, + chunk_height_created, + chunk_height_included, tx_hash, signer_id, partition_key, diff --git a/models/silver/curated/defi/silver__dex_swaps_s3.sql b/models/silver/curated/defi/silver__dex_swaps_s3.sql deleted file mode 100644 index 640d4ac..0000000 --- a/models/silver/curated/defi/silver__dex_swaps_s3.sql +++ /dev/null @@ -1,258 +0,0 @@ -{{ config( - materialized = "incremental", - unique_key = "swap_id", - incremental_strategy = "merge", - merge_exclude_columns = ["inserted_timestamp"], - cluster_by = ["block_timestamp::DATE"], - tags = ['curated','scheduled_non_core'], - enabled = False -) }} -{# DEPRECATED JANUARY 2024 #} -WITH base_swap_calls AS ( - - SELECT - block_id, - block_timestamp, - tx_hash, - action_id, - args, - _inserted_timestamp, - method_name - FROM - {{ ref('silver__actions_events_function_call_s3') }} - WHERE - method_name IN ( - 'swap', - 'ft_transfer_call' - ) {% if var("MANUAL_FIX") %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} - AND {{ incremental_load_filter('_inserted_timestamp') }} - {% endif %} -), -base_swaps AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - action_id, - IFF( - method_name = 'ft_transfer_call', - TRY_PARSE_JSON(TRY_PARSE_JSON(args) :msg), - TRY_PARSE_JSON(args) - ) :actions AS actions, - _inserted_timestamp - FROM - base_swap_calls -), -agg_swaps AS ( - SELECT - tx_hash, - ANY_VALUE(block_id) AS block_id, - ANY_VALUE(block_timestamp) AS block_timestamp, - ARRAY_AGG( - action.value - ) within GROUP ( - ORDER BY - action_id, - action.index - ) AS action_list, - ANY_VALUE(_inserted_timestamp) AS _inserted_timestamp - FROM - base_swaps, - LATERAL FLATTEN( - input => actions - ) action - GROUP BY - 1 -), -actions AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - NULLIF( - action.value :pool_id, - NULL - ) AS pool_id, - NULLIF( - action.value :token_in, - NULL - ) :: text AS token_in, - NULLIF( - action.value :token_out, - NULL - ) :: text AS token_out, - action.index AS swap_index, - _inserted_timestamp - FROM - agg_swaps, - LATERAL FLATTEN( - input => action_list - ) action - WHERE - NOT RLIKE( - pool_id, - '.*[a-z].*', - 'i' - ) -), -receipts AS ( - SELECT - block_id, - tx_hash, - -- TODO use the receipt succeeded column here - CASE - WHEN PARSE_JSON( - r.status_value - ) :Failure IS NOT NULL THEN 'Fail' - ELSE 'Success' - END AS success_or_fail, - logs - FROM - {{ ref("silver__streamline_receipts_final") }} - r - WHERE - tx_hash IN ( - SELECT - tx_hash - FROM - actions - ) -), -flat_receipts AS ( - SELECT - tx_hash, - l.value, - l.index, - success_or_fail - FROM - receipts, - LATERAL FLATTEN( - input => logs - ) l -), -swap_logs AS ( - SELECT - tx_hash, - ROW_NUMBER() over ( - PARTITION BY tx_hash - ORDER BY - INDEX ASC - ) - 1 AS swap_index, - VALUE, - success_or_fail - FROM - flat_receipts - WHERE - VALUE LIKE 'Swapped%' -), -transactions AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - tx_signer, - tx_receiver - FROM - {{ ref("silver__streamline_transactions_final") }} - WHERE - tx_hash IN ( - SELECT - tx_hash - FROM - actions - ) -), -token_labels AS ( - SELECT - * - FROM - {{ ref("silver__token_labels") }} -), -final_table AS ( - SELECT - swap_logs.swap_index, - actions._inserted_timestamp, - actions.block_id, - actions.block_timestamp, - swap_logs.tx_hash, - CONCAT( - swap_logs.tx_hash, - '-', - swap_logs.swap_index - ) AS swap_id, - swap_logs.value AS log_data, - transactions.tx_signer AS trader, - transactions.tx_receiver AS platform, - LAST_VALUE( - swap_logs.success_or_fail - ) over ( - PARTITION BY swap_logs.tx_hash - ORDER BY - swap_logs.success_or_fail DESC - ) AS txn_status, - actions.pool_id :: INT AS pool_id, - actions.token_in, - actions.token_out - FROM - actions - INNER JOIN swap_logs - ON ( - swap_logs.tx_hash = actions.tx_hash - AND swap_logs.swap_index = actions.swap_index - ) - JOIN transactions - ON actions.tx_hash = transactions.tx_hash -), -FINAL AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - swap_id, - platform, - trader, - pool_id, - token_in, - token_labels_in.symbol AS token_in_symbol, - REGEXP_SUBSTR( - log_data, - 'Swapped (\\d+)', - 1, - 1, - 'e' - ) :: NUMBER AS amount_in_raw, - amount_in_raw / pow(10, IFNULL(token_labels_in.decimals, 0)) AS amount_in, - token_out, - token_labels_out.symbol AS token_out_symbol, - REGEXP_SUBSTR( - log_data, - 'Swapped \\d+ .+ for (\\d+)', - 1, - 1, - 'e' - ) :: NUMBER AS amount_out_raw, - amount_out_raw / pow(10, IFNULL(token_labels_out.decimals, 0)) AS amount_out, - swap_index, - _inserted_timestamp - FROM - final_table - LEFT JOIN token_labels AS token_labels_in - ON final_table.token_in = token_labels_in.token_contract - LEFT JOIN token_labels AS token_labels_out - ON final_table.token_out = token_labels_out.token_contract - WHERE - txn_status = 'Success' - AND log_data IS NOT NULL -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['swap_id'] - ) }} AS dex_swaps_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL diff --git a/models/silver/curated/defi/silver__dex_swaps_s3.yml b/models/silver/curated/defi/silver__dex_swaps_s3.yml deleted file mode 100644 index 950b17a..0000000 --- a/models/silver/curated/defi/silver__dex_swaps_s3.yml +++ /dev/null @@ -1,72 +0,0 @@ -version: 2 - -models: - - name: silver__dex_swaps_s3 - description: |- - This table records all the swap transactions occurring in NEAR. This model is being deprecated as of January 2024. It will remain live through February for users to migrate to the new model. - This logic is outdated / inaccurate. - - columns: - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null: - where: _inserted_timestamp <= CURRENT_TIMESTAMP - interval '1 hour' - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null: - where: _inserted_timestamp <= CURRENT_TIMESTAMP - interval '1 hour' - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: SWAP_ID - description: "{{ doc('swap_id')}}" - - - name: PLATFORM - description: "{{ doc('platform')}}" - - - name: TRADER - description: "{{ doc('trader')}}" - - - name: POOL_ID - description: "{{ doc('pool_id')}}" - - - name: TOKEN_IN - description: "{{ doc('token_in')}}" - - - name: AMOUNT_IN - description: "{{ doc('amount_in')}}" - - - name: TOKEN_OUT - description: "{{ doc('token_out')}}" - - - name: AMOUNT_OUT - description: "{{ doc('amount_out')}}" - - - name: SWAP_INDEX - description: "{{ doc('swap_index')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: DEX_SWAPS_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/curated/defi/silver__dex_swaps_v2.sql b/models/silver/curated/defi/silver__dex_swaps_v2.sql index 2fa7a1c..a46db8e 100644 --- a/models/silver/curated/defi/silver__dex_swaps_v2.sql +++ b/models/silver/curated/defi/silver__dex_swaps_v2.sql @@ -10,7 +10,7 @@ ) }} {# Note - multisource model #} -- depends on {{ ref('silver__logs_s3') }} --- depends on {{ ref('silver__streamline_receipts_final') }} +-- depends on {{ ref('silver__receipts_final') }} {% if execute %} @@ -49,7 +49,7 @@ SELECT MIN(block_timestamp) block_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} A + {{ ref('silver__receipts_final') }} A WHERE modified_timestamp >= '{{max_mod}}' ) @@ -78,7 +78,6 @@ WITH swap_logs AS ( log_index, clean_log, _partition_by_block_number, - _inserted_timestamp, modified_timestamp FROM {{ ref('silver__logs_s3') }} @@ -97,15 +96,14 @@ WITH swap_logs AS ( ), receipts AS ( SELECT - receipt_object_id, - receipt_actions, + receipt_id AS receipt_object_id, + receipt_json AS receipt_actions, receiver_id, - signer_id, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, _partition_by_block_number, - _inserted_timestamp, modified_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} WHERE receipt_object_id IN ( SELECT @@ -157,7 +155,6 @@ swap_outcome AS ( '\\1' ) :: STRING AS token_out, _partition_by_block_number, - _inserted_timestamp, modified_timestamp FROM swap_logs @@ -212,8 +209,7 @@ parse_actions AS ( ) AS swap_input_data, r.receiver_id AS receipt_receiver_id, r.signer_id AS receipt_signer_id, - o._partition_by_block_number, - o._inserted_timestamp + o._partition_by_block_number FROM swap_outcome o LEFT JOIN receipts r USING (receipt_object_id) @@ -241,8 +237,7 @@ FINAL AS ( token_in, swap_input_data, LOG, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM parse_actions ) diff --git a/models/silver/curated/nft/sales/silver__nft_paras_sales.sql b/models/silver/curated/nft/sales/silver__nft_paras_sales.sql index 6d7a4d2..c4f7d43 100644 --- a/models/silver/curated/nft/sales/silver__nft_paras_sales.sql +++ b/models/silver/curated/nft/sales/silver__nft_paras_sales.sql @@ -53,15 +53,15 @@ WITH actions_events AS ( status_value AS ( SELECT tx_hash, - status_value, - TRY_PARSE_JSON(REPLACE(LOGS[0] :: STRING, 'EVENT_JSON:', '')) AS event, - PARSE_JSON(BASE64_DECODE_STRING(status_value:SuccessValue)) as SuccessValue, + outcome_json :outcome :status AS status_value, + TRY_PARSE_JSON(REPLACE(outcome_json :outcome :logs[0] :: STRING, 'EVENT_JSON:', '')) AS event, + PARSE_JSON(BASE64_DECODE_STRING(outcome_json :outcome :status :SuccessValue)) as SuccessValue, _partition_by_block_number, - _inserted_timestamp + inserted_timestamp AS _inserted_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} WHERE - receipt_actions:predecessor_id = 'marketplace.paras.near' + predecessor_id = 'marketplace.paras.near' AND event:event = 'nft_transfer' diff --git a/models/silver/curated/nft/silver__standard_nft_mint_s3.sql b/models/silver/curated/nft/silver__standard_nft_mint_s3.sql index ca8a8a8..36ba53c 100644 --- a/models/silver/curated/nft/silver__standard_nft_mint_s3.sql +++ b/models/silver/curated/nft/silver__standard_nft_mint_s3.sql @@ -47,10 +47,9 @@ tx AS ( tx_signer, tx_receiver, tx_succeeded, - tx_status, -- TODO deprecate col transaction_fee FROM - {{ ref('silver__streamline_transactions_final') }} + {{ ref('silver__transactions_final') }} {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} @@ -214,7 +213,6 @@ mint_tx AS ( tx_signer, tx_receiver, tx_succeeded, - tx_status, transaction_fee FROM tx @@ -239,7 +237,6 @@ FINAL AS ( mint_tx.tx_signer AS tx_signer, mint_tx.tx_receiver AS tx_receiver, mint_tx.tx_succeeded AS tx_succeeded, - mint_tx.tx_status AS tx_status, mint_events.receipt_object_id, mint_events.receiver_id, mint_events.signer_id, diff --git a/models/silver/curated/silver__lockup_actions.sql b/models/silver/curated/silver__lockup_actions.sql index 6fdcd03..ed08599 100644 --- a/models/silver/curated/silver__lockup_actions.sql +++ b/models/silver/curated/silver__lockup_actions.sql @@ -16,9 +16,9 @@ WITH txs AS ( tx_hash, tx_succeeded, _partition_by_block_number, - _inserted_timestamp + inserted_timestamp AS _inserted_timestamp FROM - {{ ref('silver__streamline_transactions_final') }} + {{ ref('silver__transactions_final') }} {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} diff --git a/models/silver/curated/silver__logs_s3.sql b/models/silver/curated/silver__logs_s3.sql index 3fae89a..69be8f1 100644 --- a/models/silver/curated/silver__logs_s3.sql +++ b/models/silver/curated/silver__logs_s3.sql @@ -15,17 +15,16 @@ WITH receipts AS ( block_id, block_timestamp, tx_hash, - receipt_object_id, - logs, + receipt_id, + outcome_json :outcome :logs AS logs, receiver_id, - receipt_actions :predecessor_id :: STRING AS predecessor_id, -- TODO once exists in receipts final can select directly - signer_id, - gas_burnt, + predecessor_id, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, + outcome_json :outcome :gas_burnt AS gas_burnt, receipt_succeeded, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} @@ -45,10 +44,10 @@ FINAL AS ( block_id, block_timestamp, tx_hash, - receipt_object_id, + receipt_id, concat_ws( '-', - receipt_object_id, + receipt_id, INDEX ) AS log_id, INDEX AS log_index, @@ -63,8 +62,7 @@ FINAL AS ( VALUE ILIKE 'event_json:%' AS is_standard, gas_burnt, receipt_succeeded, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM receipts, LATERAL FLATTEN( @@ -73,6 +71,7 @@ FINAL AS ( ) SELECT *, + receipt_id AS receipt_object_id, -- maintain for a run but then need to copy values over and then drop {{ dbt_utils.generate_surrogate_key( ['log_id'] ) }} AS logs_id, diff --git a/models/silver/curated/staking/silver__pool_events.sql b/models/silver/curated/staking/silver__pool_events.sql index 822c78d..6dddd95 100644 --- a/models/silver/curated/staking/silver__pool_events.sql +++ b/models/silver/curated/staking/silver__pool_events.sql @@ -12,16 +12,16 @@ WITH receipts AS ( tx_hash, block_id, block_timestamp, - receipt_object_id, + receipt_id AS receipt_object_id, receiver_id, - signer_id, - receipt_actions :predecessor_id :: STRING AS predecessor_id, - status_value, - logs, - _inserted_timestamp, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, + predecessor_id, + receipt_json AS receipt_actions, + outcome_json :outcome :status :: VARIANT AS status_value, + outcome_json :outcome :logs :: ARRAY AS logs, _partition_by_block_number FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} WHERE receipt_succeeded {% if var("MANUAL_FIX") %} @@ -48,8 +48,7 @@ FINAL AS ( status_value, logs, VALUE AS LOG, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM receipts, LATERAL FLATTEN(logs) diff --git a/models/silver/curated/staking/silver__staking_epochs.sql b/models/silver/curated/staking/silver__staking_epochs.sql index d1b48e2..5203e4d 100644 --- a/models/silver/curated/staking/silver__staking_epochs.sql +++ b/models/silver/curated/staking/silver__staking_epochs.sql @@ -13,11 +13,18 @@ WITH pool_events AS ( * FROM {{ ref('silver__pool_events') }} - WHERE + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} {% else %} - {{ incremental_load_filter('_inserted_timestamp') }} + WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) {% endif %} AND LOG LIKE 'Epoch%' ), diff --git a/models/silver/curated/staking/silver__staking_pools_s3.sql b/models/silver/curated/staking/silver__staking_pools_s3.sql index d228312..9a9baf4 100644 --- a/models/silver/curated/staking/silver__staking_pools_s3.sql +++ b/models/silver/curated/staking/silver__staking_pools_s3.sql @@ -17,12 +17,10 @@ WITH txs AS ( block_id, tx_signer, tx_receiver, - tx, - tx_status, - _partition_by_block_number, - _inserted_timestamp + transaction_json AS tx, + _partition_by_block_number FROM - {{ ref('silver__streamline_transactions_final') }} + {{ ref('silver__transactions_final') }} {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} @@ -48,8 +46,7 @@ function_calls AS ( signer_id, method_name, args, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM {{ ref('silver__actions_events_function_call_s3') }} WHERE @@ -84,9 +81,7 @@ add_addresses_from_tx AS ( signer_id, method_name, args, - tx_status, - txs._partition_by_block_number, - txs._inserted_timestamp + txs._partition_by_block_number FROM function_calls fc LEFT JOIN txs USING (tx_hash) @@ -102,8 +97,7 @@ new_pools AS ( args :reward_fee_fraction ) AS reward_fee_fraction, 'Create' AS tx_type, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM add_addresses_from_tx WHERE @@ -128,8 +122,7 @@ updated_pools AS ( args :reward_fee_fraction ) AS reward_fee_fraction, 'Update' AS tx_type, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM add_addresses_from_tx WHERE diff --git a/models/silver/horizon/silver_horizon__decoded_actions.sql b/models/silver/horizon/silver_horizon__decoded_actions.sql deleted file mode 100644 index 13f1d5d..0000000 --- a/models/silver/horizon/silver_horizon__decoded_actions.sql +++ /dev/null @@ -1,124 +0,0 @@ -{{ config( - materialized = 'incremental', - merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'action_id_horizon', - cluster_by = ['_inserted_timestamp::date', '_partition_by_block_number'], - tags = ['curated', 'horizon','scheduled_non_core'] -) }} -{# Note - multisource model #} --- TODO ez_actions refactor - -WITH all_horizon_receipts AS ( - - SELECT - tx_hash, - receipt_object_id, - receiver_id, - signer_id, - receipt_succeeded, - logs, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver_horizon__receipts') }} - - {% if var("MANUAL_FIX") %} - WHERE {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} -), -decoded_function_calls AS ( - SELECT - SPLIT( - action_id, - '-' - ) [0] :: STRING AS receipt_object_id, - action_id, - tx_hash, - block_id, - block_timestamp, - method_name, - args, - deposit, - attached_gas, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__actions_events_function_call_s3') }} - WHERE - _partition_by_block_number >= 85000000 - AND SPLIT( - action_id, - '-' - ) [0] :: STRING IN ( - SELECT - DISTINCT receipt_object_id - FROM - all_horizon_receipts - ) - - {% if var("MANUAL_FIX") %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - AND modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} -), -FINAL AS ( - SELECT - fc.action_id, - fc.tx_hash, - r.receipt_object_id, - fc.block_id, - fc.block_timestamp, - fc.method_name, - fc.args, - fc.deposit, - fc.attached_gas, - r.receiver_id, - r.signer_id, - r.receipt_succeeded, - r.logs, - fc._partition_by_block_number, - fc._inserted_timestamp - FROM - decoded_function_calls fc - LEFT JOIN all_horizon_receipts r USING (receipt_object_id) -) -SELECT - action_id AS action_id_horizon, - tx_hash, - receipt_object_id, - block_id, - block_timestamp, - method_name, - args, - deposit, - attached_gas, - receiver_id, - signer_id, - receipt_succeeded, - _partition_by_block_number, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['action_id_horizon'] - ) }} AS horizon_decoded_actions_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL diff --git a/models/silver/horizon/silver_horizon__decoded_actions.yml b/models/silver/horizon/silver_horizon__decoded_actions.yml deleted file mode 100644 index 3a56304..0000000 --- a/models/silver/horizon/silver_horizon__decoded_actions.yml +++ /dev/null @@ -1,80 +0,0 @@ - -version: 2 - -models: - - name: silver_horizon__decoded_actions - description: |- - Decoded FunctionCall events for receipts where the contract nearhorizon.near was called. - - columns: - - name: action_id_horizon - description: "{{ doc('action_id')}}" - tests: - - unique - - - name: tx_hash - description: "{{ doc('tx_hash')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: receipt_object_id - description: "{{ doc('receipt_object_id')}}" - - - name: block_id - description: "{{ doc('block_id')}}" - - - name: block_timestamp - description: "{{ doc('block_timestamp')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: method_name - description: "{{ doc('method_name')}}" - tests: - - not_null - - - name: args - description: "{{ doc('args')}}" - tests: - - not_null - - - name: deposit - description: "{{ doc('deposit')}}" - - - name: attached_gas - description: "{{ doc('attached_gas')}}" - - - name: receiver_id - description: "{{ doc('receiver_id')}}" - tests: - - not_null - - - name: signer_id - description: "{{ doc('signer_id')}}" - tests: - - not_null - - - name: receipt_succeeded - description: "{{ doc('receipt_succeeded')}}" - - - name: _partition_by_block_number - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - - - name: HORIZON_DECODED_ACTIONS_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/horizon/silver_horizon__receipts.sql b/models/silver/horizon/silver_horizon__receipts.sql deleted file mode 100644 index f7af0b5..0000000 --- a/models/silver/horizon/silver_horizon__receipts.sql +++ /dev/null @@ -1,64 +0,0 @@ -{{ config( - materialized = 'incremental', - merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'receipt_object_id', - cluster_by = ['_inserted_timestamp::date', 'block_timestamp::DATE'], - tags = ['curated', 'horizon','scheduled_non_core'] -) }} - -WITH all_horizon_receipts AS ( - - SELECT - tx_hash, - receipt_object_id, - block_id, - block_timestamp, - receipt_index, - chunk_hash, - receipt_actions, - execution_outcome, - receipt_outcome_id, - receiver_id, - signer_id, - receipt_type, - gas_burnt, - status_value, - receipt_succeeded, - logs, - proof, - metadata, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__streamline_receipts_final') }} - - WHERE ( - LOWER(signer_id) = 'nearhorizon.near' - OR LOWER(receiver_id) = 'nearhorizon.near' - ) - AND _partition_by_block_number >= 86000000 - - {% if var("MANUAL_FIX") %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - AND modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} -) - - SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['receipt_object_id'] - ) }} AS horizon_receipts_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id - FROM - all_horizon_receipts diff --git a/models/silver/horizon/silver_horizon__receipts.yml b/models/silver/horizon/silver_horizon__receipts.yml deleted file mode 100644 index be74f69..0000000 --- a/models/silver/horizon/silver_horizon__receipts.yml +++ /dev/null @@ -1,81 +0,0 @@ - -version: 2 - -models: - - name: silver_horizon__receipts - description: |- - Filtered receipts where the signer or receiver is the contract nearsocial.near. - - columns: - - name: tx_hash - description: "{{ doc('tx_hash')}}" - - - name: block_id - description: "{{ doc('block_id')}}" - - - name: receipt_index - description: "{{ doc('receipt_index')}}" - - - name: chunk_hash - description: "{{ doc('chunk_hash')}}" - - - name: receipt_actions - description: "{{ doc('receipt')}}" - - - name: execution_outcome - description: "{{ doc('execution_outcome')}}" - - - name: receipt_object_id - description: "{{ doc('receipt_object_id')}}" - tests: - - unique - - - name: receipt_outcome_id - description: "{{ doc('receipt_outcome_id')}}" - - - name: receiver_id - description: "{{ doc('receiver_id')}}" - - - name: signer_id - description: "{{ doc('signer_id')}}" - - - name: receipt_type - description: "{{ doc('receipt_type')}}" - - - name: gas_burnt - description: "{{ doc('gas_burnt')}}" - - - name: status_value - description: "{{ doc('status_value')}}" - - - name: receipt_succeeded - description: "{{ doc('receipt_succeeded')}}" - - - name: logs - description: "{{ doc('logs')}}" - - - name: proof - description: "{{ doc('proof')}}" - - - name: metadata - description: "{{ doc('metadata')}}" - - - name: _partition_by_block_number - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - - - name: HORIZON_RECEIPTS_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/social/silver_social__addkey.sql b/models/silver/social/silver_social__addkey.sql index cb7ff80..2852fe8 100644 --- a/models/silver/social/silver_social__addkey.sql +++ b/models/silver/social/silver_social__addkey.sql @@ -12,12 +12,12 @@ WITH receipts AS ( SELECT - receipt_object_id, - signer_id, + receipt_id AS receipt_object_id, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, _partition_by_block_number, - _inserted_timestamp + inserted_timestamp AS _inserted_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} WHERE _partition_by_block_number >= 59670000 diff --git a/models/silver/social/silver_social__receipts.sql b/models/silver/social/silver_social__receipts.sql index f532c13..5008b87 100644 --- a/models/silver/social/silver_social__receipts.sql +++ b/models/silver/social/silver_social__receipts.sql @@ -2,7 +2,7 @@ materialized = 'incremental', merge_exclude_columns = ["inserted_timestamp"], unique_key = 'receipt_object_id', - cluster_by = ['_inserted_timestamp::date', '_partition_by_block_number'], + cluster_by = ['modified_timestamp::date', '_partition_by_block_number'], tags = ['curated', 'social','scheduled_non_core'] ) }} @@ -10,27 +10,26 @@ WITH all_social_receipts AS ( SELECT tx_hash, - receipt_object_id, + receipt_id AS receipt_object_id, block_id, block_timestamp, - receipt_index, + NULL AS receipt_index, chunk_hash, - receipt_actions, - execution_outcome, - receipt_outcome_id, + receipt_json AS receipt_actions, + outcome_json AS execution_outcome, + outcome_json :outcome :receipt_ids :: ARRAY AS receipt_outcome_id, receiver_id, - receipt_actions :predecessor_id :: STRING AS predecessor_id, - signer_id, - receipt_type, - gas_burnt, - status_value, - logs, - proof, - metadata, - _partition_by_block_number, - _inserted_timestamp + predecessor_id, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, + NULL AS receipt_type, + outcome_json :outcome :gas_burnt :: NUMBER AS gas_burnt, + outcome_json :outcome :status :: VARIANT AS status_value, + outcome_json :outcome :logs :: ARRAY AS logs, + outcome_json :proof :: ARRAY AS proof, + outcome_json :outcome :metadata :: VARIANT AS metadata, + _partition_by_block_number FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} WHERE ( LOWER(signer_id) = 'social.near' diff --git a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql index 68cc9e2..7832e92 100644 --- a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql +++ b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = "block_timestamp_hour", cluster_by = ['block_timestamp_hour::DATE'], - tags = ['curated','scheduled_non_core'] + tags = ['stats','scheduled_non_core'] ) }} /* run incremental timestamp value first then use it as a static value */ {% if execute %} @@ -14,11 +14,11 @@ SELECT MIN(DATE_TRUNC('hour', block_timestamp)) block_timestamp_hour FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) {% endset %} @@ -35,7 +35,7 @@ SELECT COUNT( 1 ) AS block_count, - MAX(_inserted_timestamp) AS _inserted_timestamp, + MAX(inserted_timestamp) AS _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['block_timestamp_hour'] ) }} AS core_metrics_block_hourly_id, @@ -43,7 +43,7 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__streamline_blocks') }} + {{ ref('silver__blocks_final') }} WHERE block_timestamp_hour < DATE_TRUNC( 'hour', diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql index 66e1329..e5372be 100644 --- a/models/silver/stats/silver_stats__core_metrics_hourly.sql +++ b/models/silver/stats/silver_stats__core_metrics_hourly.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = "block_timestamp_hour", cluster_by = ['block_timestamp_hour::DATE'], - tags = ['curated','scheduled_non_core'] + tags = ['stats','scheduled_non_core'] ) }} /* run incremental timestamp value first then use it as a static value */ {% if execute %} @@ -14,11 +14,11 @@ SELECT MIN(DATE_TRUNC('hour', block_timestamp)) block_timestamp_hour FROM - {{ ref('silver__streamline_transactions_final') }} -- Streamline Migration TODO - change this to fact transactions once table + {{ ref('silver__transactions_final') }} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) {% endset %} @@ -50,7 +50,7 @@ SELECT DISTINCT tx_receiver ) AS unique_to_count, SUM(transaction_fee / pow(10, 24)) AS total_fees, - MAX(_inserted_timestamp) AS _inserted_timestamp, + MAX(inserted_timestamp) AS _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['block_timestamp_hour'] ) }} AS core_metrics_hourly_id, @@ -58,7 +58,7 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__streamline_transactions_final') }} -- Streamline Migration TODO - change this to fact transactions once table + {{ ref('silver__transactions_final') }} WHERE block_timestamp_hour < DATE_TRUNC( 'hour', diff --git a/models/streamline/core/complete/streamline__chunks_complete.sql b/models/streamline/core/complete/streamline__chunks_complete.sql index 6d66ec1..bfc84a3 100644 --- a/models/streamline/core/complete/streamline__chunks_complete.sql +++ b/models/streamline/core/complete/streamline__chunks_complete.sql @@ -13,6 +13,8 @@ SELECT VALUE :BLOCK_ID :: INT AS block_id, VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS block_timestamp_epoch, VALUE :CHUNK_HASH :: STRING AS chunk_hash, + DATA :header :height_created :: INT AS height_created, + DATA :header :height_included :: INT AS height_included, DATA :header: shard_id :: INT AS shard_id, ARRAY_SIZE( DATA :receipts :: ARRAY diff --git a/models/streamline/core/complete/streamline__transactions_complete.sql b/models/streamline/core/complete/streamline__transactions_complete.sql index 6c79df2..262615f 100644 --- a/models/streamline/core/complete/streamline__transactions_complete.sql +++ b/models/streamline/core/complete/streamline__transactions_complete.sql @@ -10,10 +10,9 @@ ) }} SELECT - VALUE :BLOCK_ID :: INT AS block_id, - VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS block_timestamp_epoch, VALUE :TX_HASH :: STRING AS tx_hash, - DATA :transaction :signer_id :: STRING AS signer_id, + VALUE :CHUNK_HASH :: STRING AS chunk_hash, + VALUE :BLOCK_ID :: INT AS block_id, partition_key, _inserted_timestamp, DATA :transaction :hash :: STRING AS complete_transactions_id, diff --git a/models/streamline/core/realtime/streamline__blocks_realtime.sql b/models/streamline/core/realtime/streamline__blocks_realtime.sql index be321d8..043ce51 100644 --- a/models/streamline/core/realtime/streamline__blocks_realtime.sql +++ b/models/streamline/core/realtime/streamline__blocks_realtime.sql @@ -26,13 +26,20 @@ tbl AS ( FROM near.tests_full.final_gaps_tx ) {% else %} -last_3_days AS ( + {% if var('STREAMLINE_BACKFILL', false) %} + last_3_days AS ( + SELECT + 9820210 AS block_id + ), + {% else %} + last_3_days AS ( - SELECT - ZEROIFNULL(block_id) AS block_id - FROM - {{ ref("_block_lookback") }} -), + SELECT + ZEROIFNULL(block_id) AS block_id + FROM + {{ ref("_block_lookback") }} + ), + {% endif %} tbl AS ( SELECT block_id diff --git a/models/streamline/core/realtime/streamline__chunks_realtime.sql b/models/streamline/core/realtime/streamline__chunks_realtime.sql index 56ec8ca..35579a0 100644 --- a/models/streamline/core/realtime/streamline__chunks_realtime.sql +++ b/models/streamline/core/realtime/streamline__chunks_realtime.sql @@ -37,41 +37,22 @@ last_3_days AS ( {% endif %} tbl AS ( SELECT - block_id, - block_timestamp_epoch, - chunk_hash + A.block_id, + A.block_timestamp_epoch, + A.chunk_hash FROM - {{ ref('streamline__chunks') }} + {{ ref('streamline__chunks') }} A + LEFT JOIN {{ ref('streamline__chunks_complete') }} B ON A.chunk_hash = B.chunk_hash WHERE ( - block_id >= ( + A.block_id >= ( SELECT block_id FROM last_3_days ) ) - AND chunk_hash IS NOT NULL - EXCEPT - SELECT - block_id, - block_timestamp_epoch, - chunk_hash - FROM - {{ ref('streamline__chunks_complete') }} - WHERE - block_id >= ( - SELECT - block_id - FROM - last_3_days - ) - AND _inserted_timestamp >= DATEADD( - 'day', - -4, - SYSDATE() - ) - AND chunk_hash IS NOT NULL + AND B.chunk_hash IS NULL ) SELECT block_id, diff --git a/models/streamline/core/realtime/streamline__transactions_realtime.sql b/models/streamline/core/realtime/streamline__transactions_realtime.sql index 9e5c694..3144e1f 100644 --- a/models/streamline/core/realtime/streamline__transactions_realtime.sql +++ b/models/streamline/core/realtime/streamline__transactions_realtime.sql @@ -37,51 +37,38 @@ last_3_days AS ( {% endif %} tbl AS ( SELECT - block_id, - block_timestamp_epoch, - tx_hash, - signer_id + A.block_id, + A.block_timestamp_epoch, + A.tx_hash, + A.signer_id, + A.shard_id, + A.chunk_hash, + A.height_created, + A.height_included FROM - {{ ref('streamline__transactions') }} + {{ ref('streamline__transactions') }} A + LEFT JOIN {{ ref('streamline__transactions_complete') }} B ON A.tx_hash = B.tx_hash WHERE ( - block_id >= ( + A.block_id >= ( SELECT block_id FROM last_3_days ) ) - AND tx_hash IS NOT NULL - AND signer_id IS NOT NULL - EXCEPT - SELECT - block_id, - block_timestamp_epoch, - tx_hash, - signer_id - FROM - {{ ref('streamline__transactions_complete') }} - WHERE - block_id >= ( - SELECT - block_id - FROM - last_3_days - ) - AND _inserted_timestamp >= DATEADD( - 'day', - -4, - SYSDATE() - ) - AND tx_hash IS NOT NULL - AND signer_id IS NOT NULL + AND A.signer_id IS NOT NULL + AND B.tx_hash IS NULL ) SELECT + shard_id, + chunk_hash, block_id, block_timestamp_epoch, - FLOOR(block_id, -3) AS partition_key, + height_created, + height_included, tx_hash, + FLOOR(block_id, -3) AS partition_key, DATE_PART('EPOCH', SYSDATE()) :: INTEGER AS request_timestamp, {{ target.database }}.live.udf_api( 'POST', diff --git a/models/streamline/core/streamline__chunks.sql b/models/streamline/core/streamline__chunks.sql index c45262f..3e2752a 100644 --- a/models/streamline/core/streamline__chunks.sql +++ b/models/streamline/core/streamline__chunks.sql @@ -23,7 +23,7 @@ WITH blocks_complete AS ( WHERE modified_timestamp >= ( SELECT - MAX(modified_timestamp) + COALESCE(MAX(modified_timestamp), '2025-01-01' :: TIMESTAMP_NTZ) FROM {{ this }} ) diff --git a/models/streamline/core/streamline__transactions.sql b/models/streamline/core/streamline__transactions.sql index 9cf82de..30d24a3 100644 --- a/models/streamline/core/streamline__transactions.sql +++ b/models/streamline/core/streamline__transactions.sql @@ -13,6 +13,8 @@ WITH chunks_complete AS ( SELECT block_id, block_timestamp_epoch, + height_created, + height_included, chunk_hash, shard_id, transaction_ids, @@ -25,7 +27,7 @@ WITH chunks_complete AS ( WHERE modified_timestamp >= ( SELECT - MAX(modified_timestamp) + COALESCE(MAX(modified_timestamp), '2025-01-01' :: TIMESTAMP_NTZ) FROM {{ this }} ) @@ -35,6 +37,8 @@ flatten_tx_ids AS ( SELECT block_id, block_timestamp_epoch, + height_created, + height_included, chunk_hash, shard_id, VALUE :: STRING AS tx_hash, @@ -50,6 +54,8 @@ flatten_tx_ids AS ( SELECT block_id, block_timestamp_epoch, + height_created, + height_included, chunk_hash, shard_id, tx_hash, diff --git a/tests/gaps/core/tests__block_gaps.sql b/tests/gaps/core/tests__block_gaps.sql index 01d8459..0308d35 100644 --- a/tests/gaps/core/tests__block_gaps.sql +++ b/tests/gaps/core/tests__block_gaps.sql @@ -1,11 +1,33 @@ {{ config( severity = 'error', - tags = ['gap_test'] + tags = ['gap_test', 'gap_test_core'] ) }} +-- depends_on: {{ ref('silver__blocks_v2') }} + +{% if execute %} + + {% if not var('DBT_FULL_TEST') %} + {% set min_block_sql %} + SELECT + MIN(block_id) + FROM + {{ ref('silver__blocks_v2') }} + WHERE + _inserted_timestamp >= SYSDATE() - INTERVAL '7 days' + {% endset %} + {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} + {% else %} + {% set min_block_id = 140868759 %} + {% endif %} + {% do log('Min block id: ' ~ min_block_id, info=True) %} +{% endif %} + WITH silver_blocks AS ( SELECT + block_timestamp, + block_hash, block_id, LAG(block_id) over ( ORDER BY @@ -13,34 +35,23 @@ WITH silver_blocks AS ( block_id ASC ) AS prior_block_id, block_id - prior_block_id AS gap_size, - block_timestamp, - block_hash, prev_hash, LAG(block_hash) over ( ORDER BY block_timestamp ASC, block_id ASC - ) AS prior_hash, - _partition_by_block_number, - _inserted_timestamp, - SYSDATE() AS _test_timestamp + ) AS prev_hash_actual FROM - {{ ref('silver__streamline_blocks') }} + {{ ref('silver__blocks_final') }} - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} + WHERE + block_id >= {{ min_block_id }} + ) SELECT * FROM silver_blocks WHERE - prior_hash <> prev_hash - {# Filter out false positive from blocks at start of window (whose parent hash was cut off) #} - AND (_inserted_timestamp > SYSDATE() - INTERVAL '7 days' + INTERVAL '1 hour') + prev_hash != COALESCE(prev_hash_actual, '') + AND block_id != {{ min_block_id }} diff --git a/tests/gaps/core/tests__chunk_gaps.sql b/tests/gaps/core/tests__chunk_gaps.sql index ee0161d..a1af7f4 100644 --- a/tests/gaps/core/tests__chunk_gaps.sql +++ b/tests/gaps/core/tests__chunk_gaps.sql @@ -1,81 +1,60 @@ {{ config( severity = 'error', - tags = ['gap_test'] + tags = ['gap_test', 'gap_test_core'] ) }} +-- depends_on: {{ ref('silver__blocks_v2') }} -WITH blocks AS ( +{% if execute %} + + {% if not var('DBT_FULL_TEST') %} + {% set min_block_sql %} + SELECT + MIN(block_id) + FROM + {{ ref('silver__blocks_v2') }} + WHERE + _inserted_timestamp >= SYSDATE() - INTERVAL '7 days' + {% endset %} + {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} + {% else %} + {% set min_block_id = 140868759 %} + {% endif %} + {% do log('Min block id: ' ~ min_block_id, info=True) %} +{% endif %} + +WITH expected_chunks AS ( SELECT block_id, - header :chunks_included :: INT AS chunk_ct_expected, - _partition_by_block_number, - _inserted_timestamp + _inserted_timestamp, + VALUE :chunk_hash :: STRING AS chunk_hash, + VALUE :height_created :: INT AS height_created, + VALUE :height_included :: INT AS height_included FROM - {{ ref('silver__streamline_blocks') }} + {{ ref('silver__blocks_v2') }}, lateral flatten(input => block_json :chunks :: ARRAY) + WHERE + block_id >= {{ min_block_id }} - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} + qualify(ROW_NUMBER() over (PARTITION BY chunk_hash ORDER BY block_id ASC)) = 1 ), -shards AS ( +actual_chunks AS ( SELECT - block_id, - MAX(_inserted_timestamp) AS _inserted_timestamp, - COUNT( - DISTINCT chunk :header :chunk_hash :: STRING - ) AS chunk_ct_actual + DISTINCT chunk_hash FROM - {{ ref('silver__streamline_shards') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1 -), -comp AS ( - SELECT - b.block_id AS b_block_id, - s.block_id AS s_block_id, - b.chunk_ct_expected, - s.chunk_ct_actual, - _partition_by_block_number, - b._inserted_timestamp AS _inserted_timestamp_blocks, - s._inserted_timestamp AS _inserted_timestamp_shards - FROM - blocks b full - OUTER JOIN shards s USING (block_id) + {{ ref('silver__chunks_v2') }} + WHERE + block_id >= {{ min_block_id }} ) SELECT - COALESCE( - b_block_id, - s_block_id - ) AS block_id, - chunk_ct_expected, - chunk_ct_actual, - _partition_by_block_number, - ( - chunk_ct_actual != chunk_ct_expected - OR b_block_id IS NULL - OR s_block_id IS NULL - ) AS is_missing + block_id, + _inserted_timestamp, + chunk_hash, + height_created, + height_included FROM - comp + expected_chunks e + LEFT JOIN actual_chunks a USING (chunk_hash) WHERE - chunk_ct_expected > 0 - AND is_missing - {# Filter out false positive from blocks at start of window #} - AND _inserted_timestamp_blocks > SYSDATE() - INTERVAL '7 days' + INTERVAL '1 hour' - AND _inserted_timestamp_shards > SYSDATE() - INTERVAL '7 days' + INTERVAL '1 hour' -ORDER BY - 1 + a.chunk_hash IS NULL + AND _inserted_timestamp <= SYSDATE() - interval '1 hour' + AND height_included >= {{ min_block_id }} diff --git a/tests/gaps/core/tests__receipt_gaps.sql b/tests/gaps/core/tests__receipt_gaps.sql index 781b85b..390501e 100644 --- a/tests/gaps/core/tests__receipt_gaps.sql +++ b/tests/gaps/core/tests__receipt_gaps.sql @@ -1,69 +1,60 @@ {{ config( severity = 'error', - tags = ['gap_test'] + tags = ['gap_test', 'gap_test_core'] ) }} +-- depends_on: {{ ref('silver__blocks_v2') }} -WITH shards AS ( +{% if execute %} + + {% if not var('DBT_FULL_TEST') %} + {% set min_block_sql %} + SELECT + MIN(block_id) + FROM + {{ ref('silver__blocks_v2') }} + WHERE + _inserted_timestamp >= SYSDATE() - INTERVAL '7 days' + {% endset %} + {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} + {% else %} + {% set min_block_id = 140868759 %} + {% endif %} + {% do log('Min block id: ' ~ min_block_id, info=True) %} +{% endif %} + +WITH expected_receipts AS ( SELECT block_id, - _partition_by_block_number, - SUM(ARRAY_SIZE(receipt_execution_outcomes)) AS receipt_ct_expected + chunk_hash, + chunk_json :height_created :: INT as chunk_height_created, + chunk_json :height_included :: INT as chunk_height_included, + _inserted_timestamp, + VALUE :receipt_id :: STRING AS receipt_id FROM - {{ ref('silver__streamline_shards') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1, - 2 + {{ ref('silver__chunks_v2') }}, lateral flatten(input => chunk_json :receipts :: ARRAY) + WHERE + block_id >= {{ min_block_id }} ), -receipts AS ( +actual_receipts AS ( SELECT - block_id, - _partition_by_block_number, - COUNT(DISTINCT receipt_id) AS receipt_ct_actual_distinct, - COUNT(1) AS receipt_ct_actual_all + DISTINCT receipt_id FROM - {{ ref('silver__streamline_receipts') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1, - 2 -), -diffs AS ( - SELECT - s.block_id, - s.receipt_ct_expected, - r.receipt_ct_actual_distinct, - r.receipt_ct_actual_all, - s._partition_by_block_number - FROM - shards s - LEFT JOIN receipts r - ON s.block_id = r.block_id + {{ ref('silver__receipts_final') }} + WHERE + block_id >= {{ min_block_id }} ) SELECT - * + block_id, + chunk_hash, + chunk_height_created, + chunk_height_included, + _inserted_timestamp, + receipt_id FROM - diffs + expected_receipts e + LEFT JOIN actual_receipts a USING (receipt_id) WHERE - receipt_ct_expected != receipt_ct_actual_distinct - OR receipt_ct_expected != receipt_ct_actual_all - OR receipt_ct_actual_distinct != receipt_ct_actual_all -ORDER BY - block_id + a.receipt_id IS NULL + AND _inserted_timestamp <= SYSDATE() - interval '2 hours' + AND chunk_height_included >= {{ min_block_id }} diff --git a/tests/gaps/core/tests__tx_gaps.sql b/tests/gaps/core/tests__tx_gaps.sql index 43812e6..5d3047c 100644 --- a/tests/gaps/core/tests__tx_gaps.sql +++ b/tests/gaps/core/tests__tx_gaps.sql @@ -1,69 +1,60 @@ {{ config( severity = 'error', - tags = ['gap_test'] + tags = ['gap_test', 'gap_test_core'] ) }} +-- depends_on: {{ ref('silver__blocks_v2') }} -WITH shards AS ( +{% if execute %} + + {% if not var('DBT_FULL_TEST') %} + {% set min_block_sql %} + SELECT + MIN(block_id) + FROM + {{ ref('silver__blocks_v2') }} + WHERE + _inserted_timestamp >= SYSDATE() - INTERVAL '7 days' + {% endset %} + {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} + {% else %} + {% set min_block_id = 140868759 %} + {% endif %} + {% do log('Min block id: ' ~ min_block_id, info=True) %} +{% endif %} + +WITH expected_txs AS ( SELECT block_id, - _partition_by_block_number, - SUM(ARRAY_SIZE(chunk :transactions :: ARRAY)) AS tx_ct_expected + chunk_hash, + chunk_json :height_created :: INT as chunk_height_created, + chunk_json :height_included :: INT as chunk_height_included, + _inserted_timestamp, + VALUE :hash :: STRING AS tx_hash FROM - {{ ref('silver__streamline_shards') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1, - 2 + {{ ref('silver__chunks_v2') }}, lateral flatten(input => chunk_json :transactions :: ARRAY) + WHERE + block_id >= {{ min_block_id }} ), -txs AS ( +actual_txs AS ( SELECT - block_id, - _partition_by_block_number, - COUNT(DISTINCT tx_hash) AS tx_ct_actual_distinct, - COUNT(1) AS tx_ct_actual_all + DISTINCT tx_hash FROM - {{ ref('silver__streamline_transactions') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1, - 2 -), -diffs AS ( - SELECT - s.block_id, - s.tx_ct_expected, - t.tx_ct_actual_distinct, - t.tx_ct_actual_all, - s._partition_by_block_number - FROM - shards s - LEFT JOIN txs t - ON s.block_id = t.block_id + {{ ref('silver__transactions_v2') }} + WHERE + origin_block_id >= {{ min_block_id }} ) SELECT - * + block_id, + chunk_hash, + chunk_height_created, + chunk_height_included, + _inserted_timestamp, + tx_hash FROM - diffs + expected_txs e + LEFT JOIN actual_txs a USING (tx_hash) WHERE - tx_ct_expected != tx_ct_actual_distinct - OR tx_ct_expected != tx_ct_actual_all - OR tx_ct_actual_distinct != tx_ct_actual_all -ORDER BY - block_id + a.tx_hash IS NULL + AND _inserted_timestamp <= SYSDATE() - interval '2 hours' + AND chunk_height_included >= {{ min_block_id }} diff --git a/tests/gaps/downstream/tests__final_gaps_rec.sql b/tests/gaps/downstream/tests__final_gaps_rec.sql deleted file mode 100644 index 0c16c72..0000000 --- a/tests/gaps/downstream/tests__final_gaps_rec.sql +++ /dev/null @@ -1,49 +0,0 @@ -{{ config( - severity = "error", - tags = ['gap_test'] -) }} - - -WITH r_receipts AS ( - SELECT - DISTINCT receipt_id, - block_id - FROM - {{ ref('silver__streamline_receipts') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -), -f_receipts AS ( - SELECT - DISTINCT receipt_object_id AS receipt_id, - block_id - FROM - {{ ref('silver__streamline_receipts_final') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -) -SELECT - r_receipts.receipt_id AS receipt_id, - r_receipts.block_id -FROM - r_receipts -LEFT JOIN - f_receipts -ON - r_receipts.receipt_id = f_receipts.receipt_id -WHERE - f_receipts.receipt_id IS NULL diff --git a/tests/gaps/downstream/tests__final_gaps_tx.sql b/tests/gaps/downstream/tests__final_gaps_tx.sql deleted file mode 100644 index 2809058..0000000 --- a/tests/gaps/downstream/tests__final_gaps_tx.sql +++ /dev/null @@ -1,55 +0,0 @@ -{{ config( - severity = "error", - tags = ['gap_test'] -) }} - - -WITH -r_transactions AS ( - SELECT - DISTINCT tx_hash, - block_id, - _signer_id AS signer_id, - shard_id - FROM - {{ ref('silver__streamline_transactions') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -), -f_transactions AS ( - SELECT - DISTINCT tx_hash, - block_id, - tx_signer AS signer_id - FROM - {{ ref('silver__streamline_transactions_final') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -) -SELECT - r_transactions.tx_hash AS tx_hash, - r_transactions.block_id, - r_transactions.signer_id, - r_transactions.shard_id -FROM - r_transactions -LEFT JOIN - f_transactions -ON - r_transactions.tx_hash = f_transactions.tx_hash -WHERE - f_transactions.tx_hash IS NULL diff --git a/tests/gaps/downstream/tests__logs_gap.sql b/tests/gaps/downstream/tests__logs_gap.sql deleted file mode 100644 index 658d1f7..0000000 --- a/tests/gaps/downstream/tests__logs_gap.sql +++ /dev/null @@ -1,78 +0,0 @@ -{{ config( - severity = "error", - tags = ['gap_test'] -) }} - -WITH r_logs AS ( - - SELECT - receipt_id, - block_id, - ARRAY_SIZE(execution_outcome :outcome :logs :: ARRAY) AS log_ct - FROM - {{ ref('silver__streamline_receipts') }} - WHERE - ARRAY_SIZE(execution_outcome :outcome :logs :: ARRAY) > 0 - {% if var('DBT_FULL_TEST') %} - AND _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - AND _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -), -r_final_logs AS ( - SELECT - receipt_object_id AS receipt_id, - block_timestamp, - block_id, - ARRAY_SIZE(logs) AS log_ct - FROM - {{ ref('silver__streamline_receipts_final') }} - WHERE - ARRAY_SIZE(logs) > 0 - - {% if var('DBT_FULL_TEST') %} - AND _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - AND _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -), -logs AS ( - SELECT - receipt_object_id AS receipt_id, - block_id, - count(1) AS log_ct - FROM - {{ ref('silver__logs_s3') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1, 2 -) -SELECT - r.receipt_id, - r.block_id, - rf.block_timestamp, - r.log_ct AS log_ct_expected, - rf.log_ct AS log_ct_final_expected, - l.log_ct AS log_ct_actual, - FLOOR(r.block_id, -3) AS _partition_by_block_number -FROM - r_logs r - LEFT JOIN r_final_logs rf - ON r.receipt_id = rf.receipt_id - AND r.block_id = rf.block_id - LEFT JOIN logs l - ON r.receipt_id = l.receipt_id - AND r.block_id = l.block_id -WHERE - r.log_ct != l.log_ct - OR rf.log_ct IS NULL diff --git a/tests/gaps/downstream/tests__logs_gap.yml b/tests/gaps/downstream/tests__logs_gap.yml deleted file mode 100644 index 2c47219..0000000 --- a/tests/gaps/downstream/tests__logs_gap.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: 2 - -tests: - name: tests__logs_gap - description: |- - This test checks for gaps in logs by checking against the initial receipts model and receipts_final. - If there are gaps between receipts and receipts_final, that will likely lead to log gaps. - The user must look at the data to determine where the gap may be.