diff --git a/models/silver/labels/DEPRECATING/livequery__request_pagoda_nft_metadata.sql b/models/silver/DEPRECATING/livequery__request_pagoda_nft_metadata.sql similarity index 100% rename from models/silver/labels/DEPRECATING/livequery__request_pagoda_nft_metadata.sql rename to models/silver/DEPRECATING/livequery__request_pagoda_nft_metadata.sql diff --git a/models/silver/labels/DEPRECATING/silver__nft_series_metadata.sql b/models/silver/DEPRECATING/silver__nft_series_metadata.sql similarity index 100% rename from models/silver/labels/DEPRECATING/silver__nft_series_metadata.sql rename to models/silver/DEPRECATING/silver__nft_series_metadata.sql diff --git a/models/silver/labels/DEPRECATING/silver__nft_series_metadata.yml b/models/silver/DEPRECATING/silver__nft_series_metadata.yml similarity index 100% rename from models/silver/labels/DEPRECATING/silver__nft_series_metadata.yml rename to models/silver/DEPRECATING/silver__nft_series_metadata.yml diff --git a/models/silver/_observability/silver_observability__logs_completeness.sql b/models/silver/_observability/silver_observability__logs_completeness.sql index 53c368f..61cfe58 100644 --- a/models/silver/_observability/silver_observability__logs_completeness.sql +++ b/models/silver/_observability/silver_observability__logs_completeness.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = 'test_timestamp', full_refresh = False, - tags = ['observability', 'deprecated'] + tags = ['observability', 'deprecated'], + enabled = false ) }} -- TODO this can be deprecated. Not a good metric of completeness. WITH summary_stats AS ( diff --git a/models/silver/core/lake/README.md b/models/silver/core/lake/README.md deleted file mode 100644 index 8d83d50..0000000 --- a/models/silver/core/lake/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Near Lake Silver Models - -The models in this directory were used to ingest and process the blocks and shards from the Near Lake Indexer. - -They are being deprecated as of February 2025 in favor of Streamline 2. - -Todo - around 30-60 days after the migration, these models will be deleted and the associated tables likely dropped from Snowflake. diff --git a/models/silver/core/lake/helpers/_retry_range.sql b/models/silver/core/lake/helpers/_retry_range.sql deleted file mode 100644 index 1bccdac..0000000 --- a/models/silver/core/lake/helpers/_retry_range.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - materialized = 'ephemeral', - tags = ['helper', 'receipt_map', 'deprecated'] -) }} - -SELECT - receipt_object_id, - block_id, - _partition_by_block_number, - _inserted_timestamp -FROM - {{ target.database }}.silver.streamline_receipts_final -WHERE - _inserted_timestamp >= SYSDATE() - INTERVAL '3 days' - AND ( - tx_hash IS NULL - OR block_timestamp IS NULL - ) diff --git a/models/silver/core/lake/helpers/silver__flatten_receipts.sql b/models/silver/core/lake/helpers/silver__flatten_receipts.sql deleted file mode 100644 index 04e111c..0000000 --- a/models/silver/core/lake/helpers/silver__flatten_receipts.sql +++ /dev/null @@ -1,41 +0,0 @@ -{{ config( - materialized = 'view', - tags = ['helper', 'receipt_map', 'deprecated'] -) }} - -WITH receipts AS ( - - SELECT - A.receipt_id PARENT, - b.value :: STRING item, - block_id, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__streamline_receipts') }} A - JOIN LATERAL FLATTEN( - A.outcome_receipts, - outer => TRUE - ) b - - {% if var("MANUAL_FIX") %} - WHERE - {{ partition_load_manual('front') }} - {% else %} - WHERE - _partition_by_block_number >= ( - SELECT - MIN(_partition_by_block_number) - (3000 * {{ var('RECEIPT_MAP_LOOKBACK_HOURS') }}) - FROM - ( - SELECT MIN(_partition_by_block_number) AS _partition_by_block_number FROM {{ ref('_retry_range') }} - UNION ALL - SELECT MAX(_partition_by_block_number) AS _partition_by_block_number FROM {{ target.database }}.silver.streamline_receipts_final - ) - ) - {% endif %} -) -SELECT - * -FROM - receipts diff --git a/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql b/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql deleted file mode 100644 index b3ad672..0000000 --- a/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql +++ /dev/null @@ -1,76 +0,0 @@ -{{ config( - materalized = 'view', - unique_key = 'receipt_id', - tags = ['helper', 'receipt_map', 'deprecated'] -) }} - -WITH -recursive ancestrytree AS ( - - SELECT - item, - PARENT - FROM - {{ ref('silver__flatten_receipts') }} - WHERE - PARENT IS NOT NULL - UNION ALL - SELECT - items.item, - t.parent - FROM - ancestrytree t - JOIN {{ ref('silver__flatten_receipts') }} - items - ON t.item = items.parent -), -txs AS ( - SELECT - tx_hash, - outcome_receipts, - _partition_by_block_number - FROM - {{ ref('silver__streamline_transactions') }} - - {% if var("MANUAL_FIX") %} - WHERE - {{ partition_load_manual('front') }} - {% else %} - WHERE - _partition_by_block_number >= ( - SELECT - MIN(_partition_by_block_number) - (3000 * {{ var('RECEIPT_MAP_LOOKBACK_HOURS') }}) - FROM - ( - SELECT MIN(_partition_by_block_number) AS _partition_by_block_number FROM {{ ref('_retry_range') }} - UNION ALL - SELECT MAX(_partition_by_block_number) AS _partition_by_block_number FROM {{ target.database }}.silver.streamline_receipts_final - ) - ) - {% endif %} -), -FINAL AS ( - SELECT - tx_hash, - A.item, - FALSE is_primary_receipt - FROM - ancestrytree A - JOIN txs b - ON A.parent = b.outcome_receipts [0] :: STRING - WHERE - item IS NOT NULL - UNION ALL - SELECT - A.tx_hash, - outcome_receipts [0] :: STRING AS receipt_id, - TRUE is_primary_receipt - FROM - txs A -) -SELECT - tx_hash, - item AS receipt_id, - is_primary_receipt -FROM - FINAL diff --git a/models/silver/core/lake/silver__streamline_blocks.sql b/models/silver/core/lake/silver__streamline_blocks.sql deleted file mode 100644 index 3407071..0000000 --- a/models/silver/core/lake/silver__streamline_blocks.sql +++ /dev/null @@ -1,106 +0,0 @@ --- Deprecated Process -{{ config( - materialized = 'incremental', - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], - incremental_strategy = 'merge', - merge_exclude_columns = ['inserted_timestamp'], - cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE', '_partition_by_block_number'], - unique_key = 'block_id', - tags = ['load', 'load_blocks', 'deprecated'], - full_refresh = False -) }} - -WITH external_blocks AS ( - - SELECT - metadata$filename AS _filename, - VALUE, - _partition_by_block_number - FROM - {{ source( - "streamline", - "blocks" - ) }} - WHERE - _partition_by_block_number >= ( - SELECT - MAX(_partition_by_block_number) - (3000 * {{ var('STREAMLINE_LOAD_LOOKBACK_HOURS') }}) - FROM - {{ this }} - ) -), -meta AS ( - SELECT - job_created_time AS _inserted_timestamp, - file_name AS _filename - FROM - TABLE( - information_schema.external_table_file_registration_history( - start_time => DATEADD( - 'hour', - -{{ var('STREAMLINE_LOAD_LOOKBACK_HOURS') }}, - SYSDATE() - ), - table_name => '{{ source( 'streamline', 'blocks' ) }}' - ) - ) A -), -blocks AS ( - SELECT - e.value :header :height :: NUMBER AS block_id, - TO_TIMESTAMP_NTZ( - e.value :header :timestamp :: STRING - ) AS block_timestamp, - e.value :header :hash :: STRING AS block_hash, - e.value :header :prev_hash :: STRING AS prev_hash, - e.value :author :: STRING AS block_author, - e.value :header :gas_price :: NUMBER AS gas_price, - e.value :header :total_supply :: NUMBER AS total_supply, - e.value :header :validator_proposals :: ARRAY AS validator_proposals, - e.value :header :validator_reward :: NUMBER AS validator_reward, - e.value :header :latest_protocol_version :: NUMBER AS latest_protocol_version, - e.value :header :epoch_id :: STRING AS epoch_id, - e.value :header :next_epoch_id :: STRING AS next_epoch_id, - NULL AS tx_count, - -- tx_count is legacy field, deprecate from core view - [] AS events, - -- events does not exist, Figment created this - e.value :chunks :: ARRAY AS chunks, - e.value :header :: OBJECT AS header, - e._partition_by_block_number, - m._inserted_timestamp - FROM - external_blocks e - LEFT JOIN meta m USING ( - _filename - ) - - {% if not var('MANUAL_FIX') %} - - {% if is_incremental() %} - WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} - -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['block_id'] - ) }} AS streamline_blocks_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - blocks - qualify ROW_NUMBER() over ( - PARTITION BY block_id - ORDER BY - _inserted_timestamp DESC - ) = 1 diff --git a/models/silver/core/lake/silver__streamline_blocks.yml b/models/silver/core/lake/silver__streamline_blocks.yml deleted file mode 100644 index 12535c4..0000000 --- a/models/silver/core/lake/silver__streamline_blocks.yml +++ /dev/null @@ -1,73 +0,0 @@ -version: 2 - -models: - - name: silver__streamline_blocks - description: |- - Parses the blocks JSON files for NEAR. - - columns: - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - - - name: BLOCK_HASH - description: "{{ doc('block_hash')}}" - - - name: PREV_HASH - description: "{{ doc('prev_hash')}}" - - - name: BLOCK_AUTHOR - description: "{{ doc('block_author')}}" - - - name: GAS_PRICE - description: "{{ doc('gas_price')}}" - - - name: TOTAL_SUPPLY - description: "{{ doc('total_supply')}}" - - - name: VALIDATOR_PROPOSALS - description: "{{ doc('validator_proposals')}}" - - - name: VALIDATOR_REWARD - description: "{{ doc('validator_reward')}}" - - - name: LATEST_PROTOCOL_VERSION - description: "{{ doc('latest_protocol_version')}}" - - - name: EPOCH_ID - description: "{{ doc('epoch_id')}}" - - - name: NEXT_EPOCH_ID - description: "{{ doc('next_epoch_id')}}" - - - name: TX_COUNT - description: "{{ doc('tx_count')}}" - - - name: EVENTS - description: "{{ doc('events')}}" - - - name: CHUNKS - description: "{{ doc('chunks')}}" - - - name: HEADER - description: "{{ doc('header')}}" - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: STREAMLINE_BLOCKS_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/core/lake/silver__streamline_receipts.sql b/models/silver/core/lake/silver__streamline_receipts.sql deleted file mode 100644 index 2eecb06..0000000 --- a/models/silver/core/lake/silver__streamline_receipts.sql +++ /dev/null @@ -1,127 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_predicates = ['DBT_INTERNAL_DEST._partition_by_block_number >= (select min(_partition_by_block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], - incremental_strategy = 'merge', - merge_exclude_columns = ['inserted_timestamp'], - unique_key = 'receipt_id', - cluster_by = ['modified_timestamp::date', '_partition_by_block_number'], - tags = ['load', 'load_shards', 'deprecated'] -) }} - -WITH shards AS ( - - SELECT - block_id, - shard_id, - receipt_execution_outcomes, - chunk :header :chunk_hash :: STRING AS chunk_hash, - chunk :header :shard_id :: INT AS shard_number, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__streamline_shards') }} - WHERE - ARRAY_SIZE(receipt_execution_outcomes) > 0 - {% if var('MANUAL_FIX') %} - AND - {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - AND modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} -), -flatten_receipts AS ( - - SELECT - concat_ws( - '-', - shard_id, - INDEX - ) AS receipt_execution_outcome_id, - block_id, - shard_id, - shard_number, - chunk_hash, - INDEX AS receipt_outcome_execution_index, - VALUE :execution_outcome :: OBJECT AS execution_outcome, - VALUE :receipt :: OBJECT AS receipt, - VALUE :receipt :receipt_id :: STRING AS receipt_id, - VALUE :execution_outcome :id :: STRING AS receipt_outcome_id, - _partition_by_block_number, - _inserted_timestamp - FROM - shards, - LATERAL FLATTEN( - input => receipt_execution_outcomes - ) -), --- TODO review new keys like priority, input_data_ids, output_data_receivers -FINAL AS ( - SELECT - receipt :receipt_id :: STRING AS receipt_id, - block_id, - shard_id, - shard_number, - receipt_outcome_execution_index AS receipt_index, - chunk_hash, - receipt, - execution_outcome, - execution_outcome :outcome :status :Failure IS NULL AS receipt_succeeded, - TRY_PARSE_JSON( - execution_outcome :outcome :status :Failure - ) AS failure_message, - object_keys( - failure_message - ) [0] :: STRING AS error_type_0, - COALESCE( - object_keys( - TRY_PARSE_JSON( - failure_message [error_type_0] :kind - ) - ) [0] :: STRING, - failure_message [error_type_0] :kind :: STRING - ) AS error_type_1, - COALESCE( - object_keys( - TRY_PARSE_JSON( - failure_message [error_type_0] :kind [error_type_1] - ) - ) [0] :: STRING, - failure_message [error_type_0] :kind [error_type_1] :: STRING - ) AS error_type_2, - failure_message [error_type_0] :kind [error_type_1] [error_type_2] :: STRING AS error_message, - execution_outcome :outcome :receipt_ids :: ARRAY AS outcome_receipts, - receipt :predecessor_id :: STRING AS predecessor_id, -- TODO manual backfill of this col required - receipt :receiver_id :: STRING AS receiver_id, - receipt :receipt :Action :signer_id :: STRING AS signer_id, - LOWER( - object_keys( - receipt :receipt - ) [0] :: STRING - ) AS receipt_type, - _partition_by_block_number, - _inserted_timestamp - FROM - flatten_receipts -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['receipt_id'] - ) }} AS streamline_receipts_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL - -QUALIFY(row_number() over - (partition by receipt_id - order by shard_number = split(shard_id, '-')[1] :: INT desc, modified_timestamp desc - )) = 1 diff --git a/models/silver/core/lake/silver__streamline_receipts.yml b/models/silver/core/lake/silver__streamline_receipts.yml deleted file mode 100644 index 2c22a04..0000000 --- a/models/silver/core/lake/silver__streamline_receipts.yml +++ /dev/null @@ -1,118 +0,0 @@ -version: 2 - -models: - - name: silver__streamline_receipts - description: |- - Singular receipt objects with the shard id and chunk hash from which it was included. - These receipts are only action receipts from the execution outcome of the shard. - - columns: - - name: RECEIPT_ID - description: "{{ doc('receipt_id')}}" - tests: - - not_null - - unique: - where: receipt_id != 'FA9zcm7WkWxdjkub7WFiKkQdnnQrcEmBht94VFzXfkm1' - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - tests: - - not_null - - - name: SHARD_ID - description: "{{ doc('shard_id')}}" - tests: - - not_null - - - name: RECEIPT_INDEX - description: "{{ doc('receipt_index')}}" - tests: - - not_null - - - name: CHUNK_HASH - description: "{{ doc('chunk_hash')}}" - tests: - - not_null: - where: _partition_by_block_number != 34690000 - - - name: RECEIPT - description: "{{ doc('receipt')}}" - - - name: EXECUTION_OUTCOME - description: "{{ doc('execution_outcome')}}" - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: FAILURE_MESSAGE - - - name: ERROR_TYPE_0 - description: "{{ doc('error_type_0')}}" - tests: - - not_null: - where: NOT RECEIPT_SUCCEEDED - - - name: ERROR_TYPE_1 - description: "{{ doc('error_type_1')}}" - tests: - - not_null: - where: NOT RECEIPT_SUCCEEDED - - - name: ERROR_TYPE_2 - description: "{{ doc('error_type_2')}}" - tests: - - not_null: - where: NOT RECEIPT_SUCCEEDED AND ERROR_TYPE_1 NOT IN ('DelegateActionExpired', 'DelegateActionInvalidSignature') - - - name: ERROR_MESSAGE - description: "{{ doc('error_message')}}" - - - name: OUTCOME_RECEIPTS - description: "{{ doc('receipt_outcome_id')}}" - tests: - - not_null - - - name: PREDECESSOR_ID - description: "{{ doc('predecessor_id')}}" - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - tests: - - not_null - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - tests: - - not_null - - - name: RECEIPT_TYPE - description: "{{ doc('receipt_type')}}" - tests: - - not_null - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: STREAMLINE_RECEIPTS_ID - description: "{{doc('id')}}" - tests: - - not_null - - unique: - where: receipt_id != 'FA9zcm7WkWxdjkub7WFiKkQdnnQrcEmBht94VFzXfkm1' - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/core/lake/silver__streamline_receipts_final.sql b/models/silver/core/lake/silver__streamline_receipts_final.sql deleted file mode 100644 index 98f48d1..0000000 --- a/models/silver/core/lake/silver__streamline_receipts_final.sql +++ /dev/null @@ -1,176 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], - incremental_strategy = 'merge', - merge_exclude_columns = ['inserted_timestamp'], - unique_key = 'receipt_object_id', - cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number', ], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", - tags = ['receipt_map', 'deprecated'], - full_refresh = False -) }} - -WITH retry_range AS ( - - SELECT - * - FROM - {{ ref('_retry_range')}} - -), -base_receipts AS ( - SELECT - receipt_id, - block_id, - shard_id, - receipt_index, - chunk_hash, - receipt, - execution_outcome, - outcome_receipts, - predecessor_id, - receiver_id, - signer_id, - receipt_type, - receipt_succeeded, - error_type_0, - error_type_1, - error_type_2, - error_message, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__streamline_receipts') }} r - - {% if var('MANUAL_FIX') %} - - WHERE - {{ partition_load_manual('no_buffer') }} - - {% else %} - - WHERE - _partition_by_block_number >= ( - SELECT - MIN(_partition_by_block_number) - (3000 * {{ var('RECEIPT_MAP_LOOKBACK_HOURS') }}) - FROM - ( - SELECT MIN(_partition_by_block_number) AS _partition_by_block_number FROM retry_range - UNION ALL - SELECT MAX(_partition_by_block_number) AS _partition_by_block_number FROM {{ this }} - ) - ) - AND ( - {% if is_incremental() %} - modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - OR - {% endif %} - receipt_id IN ( - SELECT - DISTINCT receipt_object_id - FROM - retry_range - ) - ) - {% endif %} -), -blocks AS ( - SELECT - block_id, - block_timestamp, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__streamline_blocks') }} - - {% if var('MANUAL_FIX') %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% else %} - WHERE - _partition_by_block_number >= ( - SELECT - MIN(_partition_by_block_number) - (3000 * {{ var('RECEIPT_MAP_LOOKBACK_HOURS') }}) - FROM - ( - SELECT MIN(_partition_by_block_number) AS _partition_by_block_number FROM retry_range - UNION ALL - SELECT MAX(_partition_by_block_number) AS _partition_by_block_number FROM {{ this }} - ) - ) - {% endif %} -), -append_tx_hash AS ( - SELECT - m.tx_hash, - r.receipt_id, - r.block_id, - r.shard_id, - r.receipt_index, - r.chunk_hash, - r.receipt, - r.execution_outcome, - r.outcome_receipts, - r.predecessor_id, - r.receiver_id, - r.signer_id, - r.receipt_type, - r.receipt_succeeded, - r.error_type_0, - r.error_type_1, - r.error_type_2, - r.error_message, - r._partition_by_block_number, - r._inserted_timestamp - FROM - base_receipts r - INNER JOIN {{ ref('silver__receipt_tx_hash_mapping') }} - m USING (receipt_id) -), -FINAL AS ( - SELECT - tx_hash, - receipt_id AS receipt_object_id, -- TODO drop this col after manual backfill of just receipt_id - receipt_id, - r.block_id, - b.block_timestamp, - receipt_index, - chunk_hash, - receipt AS receipt_actions, - execution_outcome, - outcome_receipts AS receipt_outcome_id, - predecessor_id, -- TODO manual backfill of this col required - receiver_id, - signer_id, - receipt_type, - execution_outcome :outcome :gas_burnt :: NUMBER AS gas_burnt, - execution_outcome :outcome :status :: variant AS status_value, - execution_outcome :outcome :logs :: ARRAY AS logs, - execution_outcome :proof :: ARRAY AS proof, - execution_outcome :outcome :metadata :: variant AS metadata, - receipt_succeeded, - error_type_0, - error_type_1, - error_type_2, - error_message, - _partition_by_block_number, - _inserted_timestamp - FROM - append_tx_hash r - INNER JOIN blocks b USING (block_id) -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['receipt_object_id'] - ) }} AS streamline_receipts_final_id, -- todo also will need to update this hash later - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL diff --git a/models/silver/core/lake/silver__streamline_receipts_final.yml b/models/silver/core/lake/silver__streamline_receipts_final.yml deleted file mode 100644 index f98b6da..0000000 --- a/models/silver/core/lake/silver__streamline_receipts_final.yml +++ /dev/null @@ -1,120 +0,0 @@ -version: 2 - -models: - - name: silver__streamline_receipts_final - description: |- - Singular receipt objects with the shard id and chunk hash from which it was included. - - columns: - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null - - - name: RECEIPT_OBJECT_ID - description: "{{ doc('receipt_object_id')}}" - tests: - - not_null - - - name: RECEIPT_ID - description: "{{ doc('receipt_id')}}" - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - tests: - - not_null - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null - - - name: RECEIPT_INDEX - description: "{{ doc('receipt_index')}}" - - - name: CHUNK_HASH - description: "{{ doc('chunk_hash')}}" - tests: - - not_null: - where: "block_id not in (34691244, 34691277)" - - - name: RECEIPT_ACTIONS - description: "{{ doc('receipt')}}" - - - name: EXECUTION_OUTCOME - description: "{{ doc('execution_outcome')}}" - - - name: RECEIPT_OUTCOME_ID - description: "{{ doc('receipt_outcome_id')}}" - - - name: PREDECESSOR_ID - description: "{{ doc('predecessor_id')}}" - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - tests: - - not_null - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - tests: - - not_null - - - name: RECEIPT_TYPE - description: "{{ doc('receipt_type')}}" - - - name: GAS_BURNT - description: "{{ doc('gas_burnt')}}" - - - name: STATUS_VALUE - description: "{{ doc('status_value')}}" - - - name: LOGS - description: "{{ doc('logs')}}" - - - name: PROOF - description: "{{ doc('proof')}}" - - - name: METADATA - description: "{{ doc('metadata')}}" - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: ERROR_TYPE_0 - description: "{{ doc('error_type_0')}}" - - - name: ERROR_TYPE_1 - description: "{{ doc('error_type_1')}}" - - - name: ERROR_TYPE_2 - description: "{{ doc('error_type_2')}}" - - - name: ERROR_MESSAGE - description: "{{ doc('error_message')}}" - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: STREAMLINE_RECEIPTS_FINAL_ID - description: "{{doc('id')}}" - tests: - - not_null - - unique - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/core/lake/silver__streamline_shards.sql b/models/silver/core/lake/silver__streamline_shards.sql deleted file mode 100644 index 74e24e6..0000000 --- a/models/silver/core/lake/silver__streamline_shards.sql +++ /dev/null @@ -1,96 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'merge', - incremental_predicates = ['DBT_INTERNAL_DEST._partition_by_block_number >= (select min(_partition_by_block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], - merge_exclude_columns = ['inserted_timestamp'], - cluster_by = ['_inserted_timestamp::DATE', '_partition_by_block_number'], - unique_key = 'shard_id', - tags = ['load', 'load_shards', 'deprecated'], - full_refresh = False -) }} - -WITH external_shards AS ( - - SELECT - metadata$filename AS _filename, - VALUE, - _partition_by_block_number - FROM - {{ source( - "streamline", - "shards" - ) }} - WHERE - _partition_by_block_number >= ( - SELECT - MAX(_partition_by_block_number) - (3000 * {{ var('STREAMLINE_LOAD_LOOKBACK_HOURS') }}) - FROM - {{ this }} - ) -), -meta AS ( - SELECT - job_created_time AS _inserted_timestamp, - file_name AS _filename - FROM - TABLE( - information_schema.external_table_file_registration_history( - start_time => DATEADD( - 'hour', - -{{ var('STREAMLINE_LOAD_LOOKBACK_HOURS') }}, - SYSDATE() - ), - table_name => '{{ source( 'streamline', 'shards' ) }}' - ) - ) A -), -shards AS ( - SELECT - e._filename, - SPLIT( - e._filename, - '/' - ) [0] :: NUMBER AS block_id, - RIGHT(SPLIT(e._filename, '.') [0], 1) :: NUMBER AS _shard_number, - concat_ws( - '-', - block_id :: STRING, - _shard_number :: STRING - ) AS shard_id, - e.value :chunk :: VARIANT AS chunk, - e.value :receipt_execution_outcomes :: VARIANT AS receipt_execution_outcomes, - e.value :shard_id :: NUMBER AS shard_number, - e.value :state_changes :: VARIANT AS state_changes, - e._partition_by_block_number, - m._inserted_timestamp - FROM - external_shards e - LEFT JOIN meta m USING (_filename) - - {% if not var('MANUAL_FIX') %} - {% if is_incremental() %} - WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['shard_id'] - ) }} AS streamline_shards_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - shards - qualify ROW_NUMBER() over ( - PARTITION BY shard_id - ORDER BY - _inserted_timestamp DESC - ) = 1 \ No newline at end of file diff --git a/models/silver/core/lake/silver__streamline_shards.yml b/models/silver/core/lake/silver__streamline_shards.yml deleted file mode 100644 index de7793f..0000000 --- a/models/silver/core/lake/silver__streamline_shards.yml +++ /dev/null @@ -1,70 +0,0 @@ -version: 2 - -models: - - name: silver__streamline_shards - description: |- - Parses the shards JSON files for NEAR. - - columns: - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: SHARD_ID - description: "{{ doc('shard_id')}}" - tests: - - not_null - - unique - - - name: CHUNK - description: "{{ doc('chunks')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - - OBJECT - - - name: RECEIPT_EXECUTION_OUTCOMES - description: "{{ doc('receipt_execution_outcomes')}}" - - - name: SHARD_NUMBER - description: "{{ doc('shard_number')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: STATE_CHANGES - description: "{{ doc('state_changes')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - VARIANT - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: STREAMLINE_SHARDS_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/core/lake/silver__streamline_transactions.sql b/models/silver/core/lake/silver__streamline_transactions.sql deleted file mode 100644 index 877d54d..0000000 --- a/models/silver/core/lake/silver__streamline_transactions.sql +++ /dev/null @@ -1,118 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_predicates = ['DBT_INTERNAL_DEST._partition_by_block_number >= (select min(_partition_by_block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], - incremental_strategy = 'merge', - merge_exclude_columns = ['inserted_timestamp'], - unique_key = 'tx_hash', - cluster_by = ['modified_timestamp::date', '_partition_by_block_number'], - tags = ['load', 'load_shards', 'deprecated'] -) }} - -WITH chunks AS ( - - SELECT - block_id, - shard_id, - chunk, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__streamline_shards') }} - WHERE - array_size(chunk :transactions :: ARRAY) > 0 - - {% if var('MANUAL_FIX') %} - AND - {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - AND modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} -), -flatten_transactions AS ( - SELECT - VALUE :transaction :hash :: STRING AS tx_hash, - block_id, - shard_id, - chunk :header :shard_id :: INT AS shard_number, - INDEX AS transactions_index, - chunk :header :chunk_hash :: STRING AS chunk_hash, - VALUE :outcome :execution_outcome :outcome :receipt_ids :: ARRAY AS outcome_receipts, - VALUE AS tx, - _partition_by_block_number, - _inserted_timestamp - FROM - chunks, - LATERAL FLATTEN( - input => chunk :transactions :: ARRAY - ) -), -txs AS ( - SELECT - tx_hash, - block_id, - shard_id, - shard_number, - transactions_index, - chunk_hash, - outcome_receipts, - tx, - tx :transaction :actions :: VARIANT AS _actions, - tx :transaction :hash :: STRING AS _hash, - tx :transaction :nonce :: STRING AS _nonce, - tx :outcome :execution_outcome :: VARIANT AS _outcome, - tx :transaction :public_key :: STRING AS _public_key, - [] AS _receipt, - tx :transaction :receiver_id :: STRING AS _receiver_id, - tx :transaction :signature :: STRING AS _signature, - tx :transaction :signer_id :: STRING AS _signer_id, - _partition_by_block_number, - _inserted_timestamp - FROM - flatten_transactions -), -FINAL AS ( - SELECT - tx_hash, - block_id, - shard_id, - shard_number, - transactions_index, - chunk_hash, - outcome_receipts, - tx, - _actions, - _hash, - _nonce, - _outcome, - _public_key, - _receipt, - _receiver_id, - _signature, - _signer_id, - _partition_by_block_number, - _inserted_timestamp - FROM - txs -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash'] - ) }} AS streamline_transactions_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL - -QUALIFY(row_number() over - (partition by tx_hash - order by shard_number = split(shard_id, '-')[1] :: INT desc, modified_timestamp desc - )) = 1 diff --git a/models/silver/core/lake/silver__streamline_transactions.yml b/models/silver/core/lake/silver__streamline_transactions.yml deleted file mode 100644 index 19df4b0..0000000 --- a/models/silver/core/lake/silver__streamline_transactions.yml +++ /dev/null @@ -1,92 +0,0 @@ -version: 2 - -models: - - name: silver__streamline_transactions - description: |- - Singular transaction objects with the shard id and chunk hash from which it was included. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_hash - - block_id - - columns: - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - tests: - - not_null - - - name: SHARD_ID - description: "{{ doc('shard_id')}}" - tests: - - not_null - - - name: TRANSACTIONS_INDEX - description: "{{ doc('receipt_index')}}" - tests: - - not_null - - - name: CHUNK_HASH - description: "{{ doc('chunk_hash')}}" - tests: - - not_null - - - name: OUTCOME_RECEIPTS - description: "{{ doc('outcome')}}" - - - name: TX - description: "{{ doc('tx')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - OBJECT - - VARIANT - - - name: _ACTIONS - description: "{{ doc('actions')}}" - - - name: _HASH - description: "{{ doc('tx_hash')}}" - - - name: _NONCE - description: "{{ doc('nonce')}}" - - - name: _OUTCOME - description: "{{ doc('outcome')}}" - - - name: _PUBLIC_KEY - description: "{{ doc('public_key')}}" - - - name: _RECEIPT - description: "{{ doc('receipt')}}" - - - name: _RECEIVER_ID - description: "{{ doc('tx_receiver')}}" - - - name: _SIGNATURE - description: "{{ doc('signature')}}" - - - name: _SIGNER_ID - description: "{{ doc('tx_signer')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: STREAMLINE_TRANSACTIONS_ID - description: "{{doc('id')}}" - tests: - - unique - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/core/lake/silver__streamline_transactions_final.sql b/models/silver/core/lake/silver__streamline_transactions_final.sql deleted file mode 100644 index ef5a471..0000000 --- a/models/silver/core/lake/silver__streamline_transactions_final.sql +++ /dev/null @@ -1,261 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'tx_hash', - cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number'], - tags = ['receipt_map', 'deprecated'] -) }} - -WITH int_txs AS ( - - SELECT - block_id, - tx_hash, - shard_id, - transactions_index, - chunk_hash, - outcome_receipts, - _actions, - _hash, - _nonce, - _outcome, - _public_key, - _receiver_id, - _signature, - _signer_id, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__streamline_transactions') }} - - {% if var('MANUAL_FIX') %} - - WHERE - {{ partition_load_manual('no_buffer') }} - - {% else %} - WHERE - {{ partition_incremental_load( - 8000, - 6000, - 0 - ) }} - - {% endif %} -), -int_receipts AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - receipt_object_id, - execution_outcome, - receipt_succeeded, - gas_burnt, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__streamline_receipts_final') }} - - {% if var('MANUAL_FIX') %} - - WHERE - {{ partition_load_manual('end') }} - - {% else %} - - WHERE - {{ partition_incremental_load( - 8000, - 6000, - 0 - ) }} - - {% endif %} - -), -int_blocks AS ( - SELECT - block_id, - block_hash, - block_timestamp, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__streamline_blocks') }} - WHERE - _partition_by_block_number >= ( - SELECT MIN(_partition_by_block_number) FROM int_txs - ) -), -receipt_array AS ( - SELECT - tx_hash, - ARRAY_AGG(execution_outcome) within GROUP ( - ORDER BY - block_timestamp - ) AS receipt - FROM - int_receipts - GROUP BY - 1 -), -base_transactions AS ( - SELECT - t.tx_hash, - t.block_id, - b.block_hash, - b.block_timestamp, - t.shard_id, - transactions_index, - t.chunk_hash, - outcome_receipts, - OBJECT_CONSTRUCT( - 'actions', - _actions, -- does not need to exist as col (in receipts) but fields used in gas calc! - 'hash', - _hash, -- exists as col - 'nonce', - _nonce, -- exists as col - 'outcome', - _outcome, -- does not need to exist as col (in receipts) but fields used in gas calc! - 'public_key', - _public_key, -- does not exist as col - 'receipt', - r.receipt, -- does not need to exist as col - 'receiver_id', - _receiver_id, -- exists as col - 'signature', - _signature, -- exists as col - 'signer_id', - _signer_id -- exists as col - ) AS tx, -- TODO dropping this object - _partition_by_block_number, - t._inserted_timestamp - FROM - int_txs t - INNER JOIN receipt_array r USING (tx_hash) - INNER JOIN int_blocks b USING (block_id) -), --- TODO review the below section in a subsequent PR. ~3y old at this point. --- it is calculating the gaas and fees. Largely accurate, but may have found an inaccuracy: --- Dojw9TnLbTLTxeJAuGtiSTZGdruCAmdu1KsLgjLHwkkb incorrect tx fees / gas burnt? - -{# The following steps were copied directly from legacy tx model to replicate columns #} -actions AS ( - SELECT - tx_hash, - SUM( - VALUE :FunctionCall :gas :: NUMBER - ) AS attached_gas - FROM - base_transactions, - LATERAL FLATTEN( - input => tx :actions - ) - GROUP BY - 1 -), -transactions AS ( - SELECT - block_id, - tx :outcome :block_hash :: STRING AS block_hash, - tx_hash, - block_timestamp, - tx :nonce :: NUMBER AS nonce, - tx :signature :: STRING AS signature, - tx :receiver_id :: STRING AS tx_receiver, - tx :signer_id :: STRING AS tx_signer, - tx, - tx :outcome :outcome :gas_burnt :: NUMBER AS transaction_gas_burnt, - tx :outcome :outcome :tokens_burnt :: NUMBER AS transaction_tokens_burnt, - _partition_by_block_number, - _inserted_timestamp - FROM - base_transactions -), -gas_burnt AS ( - SELECT - tx_hash, - SUM(gas_burnt) AS receipt_gas_burnt, - SUM(execution_outcome :outcome :tokens_burnt :: NUMBER) AS receipt_tokens_burnt - FROM - int_receipts - WHERE - execution_outcome :outcome: tokens_burnt :: NUMBER != 0 - GROUP BY - 1 -), -determine_tx_status AS ( - SELECT - DISTINCT tx_hash, - LAST_VALUE( - receipt_succeeded - ) over ( - PARTITION BY tx_hash - ORDER BY - block_id ASC - ) AS tx_succeeded - FROM - int_receipts -), -FINAL AS ( - SELECT - t.block_id, - t.block_hash, - t.tx_hash, - t.block_timestamp, - t.nonce, - t.signature, - t.tx_receiver, - t.tx_signer, - t.tx, - t.transaction_gas_burnt + g.receipt_gas_burnt AS gas_used, - t.transaction_tokens_burnt + g.receipt_tokens_burnt AS transaction_fee, - COALESCE( - actions.attached_gas, - gas_used - ) AS attached_gas, - s.tx_succeeded, - IFF ( - tx_succeeded, - 'Success', - 'Fail' - ) AS tx_status, -- DEPRECATE TX_STATUS IN GOLD - t._partition_by_block_number, - t._inserted_timestamp - FROM - transactions AS t - INNER JOIN determine_tx_status s - ON t.tx_hash = s.tx_hash - INNER JOIN actions - ON t.tx_hash = actions.tx_hash - INNER JOIN gas_burnt g - ON t.tx_hash = g.tx_hash -) -SELECT - tx_hash, - block_id, - block_hash, - block_timestamp, - nonce, - signature, - tx_receiver, - tx_signer, - tx, - gas_used, - transaction_fee, - attached_gas, - tx_succeeded, - tx_status, - _partition_by_block_number, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash'] - ) }} AS streamline_transactions_final_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL - diff --git a/models/silver/core/lake/silver__streamline_transactions_final.yml b/models/silver/core/lake/silver__streamline_transactions_final.yml deleted file mode 100644 index e4a7eb6..0000000 --- a/models/silver/core/lake/silver__streamline_transactions_final.yml +++ /dev/null @@ -1,98 +0,0 @@ -version: 2 - -models: - - name: silver__streamline_transactions_final - description: |- - Singular transaction objects with the shard id and chunk hash from which it was included. - - columns: - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: BLOCK_HASH - description: "{{ doc('block_hash')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - name: NONCE - description: "{{ doc('nonce')}}" - - - name: SIGNATURE - description: "{{ doc('signature')}}" - - - name: TX_RECEIVER - description: "{{ doc('tx_receiver')}}" - - - name: TX_SIGNER - description: "{{ doc('tx_signer')}}" - - - name: TX - description: "{{ doc('tx')}}" - - - name: TX_SUCCEEDED - description: "{{ doc('tx_succeeded')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: TX_STATUS - description: "{{ doc('tx_status')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: GAS_USED - description: "{{ doc('gas_used')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: ATTACHED_GAS - description: "{{ doc('attached_gas')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: TRANSACTION_FEE - description: "{{ doc('transaction_fee')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: STREAMLINE_TRANSACTIONS_FINAL_ID - description: "{{doc('id')}}" - tests: - - unique - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/core/migration/_migrate_blocks.sql b/models/silver/core/migration/_migrate_blocks.sql deleted file mode 100644 index 6226e76..0000000 --- a/models/silver/core/migration/_migrate_blocks.sql +++ /dev/null @@ -1,34 +0,0 @@ -{{ config( - materialized = 'ephemeral' -) }} - -SELECT - block_id, - block_timestamp, - block_hash, - prev_hash, - block_author, - chunks AS chunks_json, - header AS header_json, - _partition_by_block_number, - {{ dbt_utils.generate_surrogate_key( - ['block_id'] - ) }} AS blocks_final_id, - COALESCE( - inserted_timestamp, - _inserted_timestamp, - _load_timestamp - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - _inserted_timestamp, - _load_timestamp - ) AS modified_timestamp, - _invocation_id -FROM - {{ ref('silver__streamline_blocks') }} - - {% if var("NEAR_MIGRATE_ARCHIVE") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% endif %} diff --git a/models/silver/core/migration/_migrate_receipts.sql b/models/silver/core/migration/_migrate_receipts.sql deleted file mode 100644 index 9d0ed8e..0000000 --- a/models/silver/core/migration/_migrate_receipts.sql +++ /dev/null @@ -1,79 +0,0 @@ -{{ config( - materialized = 'ephemeral' -) }} - -WITH lake_receipts_final AS ( - - SELECT - chunk_hash, - block_id, - block_timestamp, - tx_hash, - COALESCE( - receipt_id, - receipt_object_id - ) AS receipt_id, - COALESCE( - predecessor_id, - receipt_actions :predecessor_id :: STRING - ) AS predecessor_id, - receiver_id, - receipt_actions AS receipt_json, - execution_outcome AS outcome_json, - receipt_succeeded, - _partition_by_block_number, - {{ dbt_utils.generate_surrogate_key( - ['COALESCE(receipt_id, receipt_object_id)'] - ) }} AS receipts_final_id, - COALESCE( - inserted_timestamp, - _inserted_timestamp, - _load_timestamp - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - _inserted_timestamp, - _load_timestamp - ) AS modified_timestamp, - _invocation_id - FROM - {{ ref('silver__streamline_receipts_final') }} - - {% if var("NEAR_MIGRATE_ARCHIVE") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% endif %} -), -lake_transactions_final AS ( - SELECT - tx_hash, - tx_succeeded - FROM - {{ ref('silver__streamline_transactions_final') }} - - {% if var("NEAR_MIGRATE_ARCHIVE") %} - WHERE - {{ partition_load_manual('front') }} - {% endif %} -) -SELECT - chunk_hash, - block_id, - block_timestamp, - r.tx_hash, - receipt_id, - predecessor_id, - receiver_id, - receipt_json, - outcome_json, - tx_succeeded, - receipt_succeeded, - _partition_by_block_number, - receipts_final_id, - inserted_timestamp, - modified_timestamp, - _invocation_id -FROM - lake_receipts_final r - LEFT JOIN lake_transactions_final tx - ON r.tx_hash = tx.tx_hash diff --git a/models/silver/core/migration/_migrate_txs.sql b/models/silver/core/migration/_migrate_txs.sql deleted file mode 100644 index c112321..0000000 --- a/models/silver/core/migration/_migrate_txs.sql +++ /dev/null @@ -1,86 +0,0 @@ -{{ config( - materialized = 'ephemeral' -) }} - -WITH lake_transactions_final AS ( - - SELECT - block_id, - block_timestamp, - tx_hash, - tx_signer, - tx_receiver, - tx_succeeded, - gas_used, - transaction_fee, - attached_gas, - _partition_by_block_number, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash'] - ) }} AS transactions_final_id, - COALESCE( - inserted_timestamp, - _inserted_timestamp, - _load_timestamp - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - _inserted_timestamp, - _load_timestamp - ) AS modified_timestamp, - _invocation_id - FROM - {{ ref('silver__streamline_transactions_final') }} - - {% if var("NEAR_MIGRATE_ARCHIVE") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% endif %} -), -lake_transactions_int AS ( - SELECT - tx_hash, - block_id, - shard_number, - chunk_hash, - tx :transaction :: variant AS transaction_json, - tx :outcome :execution_outcome :: variant AS outcome_json, - _partition_by_block_number - FROM - {{ ref('silver__streamline_transactions') }} - - {% if var("NEAR_MIGRATE_ARCHIVE") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% endif %} -), -transaction_archive AS ( - SELECT - i.chunk_hash, - i.shard_number AS shard_id, - f.block_id, - f.block_timestamp, - f.tx_hash, - f.tx_signer, - f.tx_receiver, - i.transaction_json, - i.outcome_json, - f.tx_succeeded, - f.gas_used, - f.transaction_fee, - f.attached_gas, - f._partition_by_block_number, - f.transactions_final_id, - f.inserted_timestamp, - f.modified_timestamp, - f._invocation_id - FROM - lake_transactions_final f - LEFT JOIN lake_transactions_int i - ON f.tx_hash = i.tx_hash - AND f._partition_by_block_number = i._partition_by_block_number -) -SELECT - * -FROM - transaction_archive diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index 6ae3975..48400a7 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -10,29 +10,6 @@ full_refresh = false ) }} -{% if var('NEAR_MIGRATE_ARCHIVE', False) %} - {% if execute %} - {% do log('Migrating blocks ' ~ var('RANGE_START') ~ ' to ' ~ var('RANGE_END'), info=True) %} - {% do log('Invocation ID: ' ~ invocation_id, info=True) %} - {% endif %} - SELECT - block_id, - block_timestamp, - block_hash, - prev_hash, - block_author, - chunks_json, - header_json, - _partition_by_block_number, - blocks_final_id, - inserted_timestamp, - modified_timestamp, - '{{ invocation_id }}' AS _invocation_id - FROM - {{ ref('_migrate_blocks') }} - -{% else %} - WITH blocks AS ( SELECT block_id, @@ -66,5 +43,3 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM blocks - -{% endif %} diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 4f87625..f6a4f11 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -10,64 +10,37 @@ full_refresh = false ) }} -{% if var('NEAR_MIGRATE_ARCHIVE', False) %} - {% if execute %} - {% do log('Migrating receipts ' ~ var('RANGE_START') ~ ' to ' ~ var('RANGE_END'), info=True) %} - {% do log('Invocation ID: ' ~ invocation_id, info=True) %} - {% endif %} +{% if execute and not var("MANUAL_FIX") %} + {% if is_incremental() %} + {% set max_mod_query %} + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01') modified_timestamp + FROM + {{ this }} + {% endset %} + + {% set max_mod = run_query(max_mod_query) [0] [0] %} + {% do log('max_mod: ' ~ max_mod, info=True) %} + {% set min_block_date_query %} + SELECT + MIN(origin_block_timestamp :: DATE) block_timestamp + FROM + {{ ref('silver__transactions_v2') }} + WHERE + modified_timestamp >= '{{max_mod}}' + {% endset %} - SELECT - chunk_hash, - block_id, - block_timestamp, - tx_hash, - receipt_id, - predecessor_id, - receiver_id, - receipt_json, - outcome_json, - tx_succeeded, - receipt_succeeded, - _partition_by_block_number, - receipts_final_id, - inserted_timestamp, - modified_timestamp, - '{{ invocation_id }}' AS _invocation_id - FROM - {{ ref('_migrate_receipts') }} - -{% else %} - {% if execute and not var("MANUAL_FIX") %} - {% if is_incremental() %} - {% set max_mod_query %} - SELECT - COALESCE(MAX(modified_timestamp), '1970-01-01') modified_timestamp - FROM - {{ this }} - {% endset %} - - {% set max_mod = run_query(max_mod_query) [0] [0] %} - {% do log('max_mod: ' ~ max_mod, info=True) %} - {% set min_block_date_query %} - SELECT - MIN(origin_block_timestamp :: DATE) block_timestamp - FROM - {{ ref('silver__transactions_v2') }} - WHERE - modified_timestamp >= '{{max_mod}}' - {% endset %} - - {% set min_bd = run_query(min_block_date_query) [0] [0] %} + {% set min_bd = run_query(min_block_date_query) [0] [0] %} + {% do log('min_bd: ' ~ min_bd, info=True) %} + {% if not min_bd or min_bd == 'None' %} + {% set min_bd = '2099-01-01' %} {% do log('min_bd: ' ~ min_bd, info=True) %} - {% if not min_bd or min_bd == 'None' %} - {% set min_bd = '2099-01-01' %} - {% do log('min_bd: ' ~ min_bd, info=True) %} - {% endif %} - - {% do log('min_block_date: ' ~ min_bd, info=True) %} - {% endif %} + + {% do log('min_block_date: ' ~ min_bd, info=True) %} + {% endif %} +{% endif %} WITH txs_with_receipts AS ( SELECT @@ -246,5 +219,3 @@ FROM FINAL qualify(row_number() over (partition by receipt_id order by modified_timestamp desc) = 1) - -{% endif %} diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index 5cdf435..6b12385 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -10,36 +10,6 @@ full_refresh = false ) }} -{% if var('NEAR_MIGRATE_ARCHIVE', False) %} - {% if execute %} - {% do log('Migrating transactions ' ~ var('RANGE_START') ~ ' to ' ~ var('RANGE_END'), info=True) %} - {% do log('Invocation ID: ' ~ invocation_id, info=True) %} - {% endif %} - - SELECT - chunk_hash, - block_id, - block_timestamp, - tx_hash, - tx_receiver, - tx_signer, - transaction_json, - outcome_json, - OBJECT_CONSTRUCT() AS status_json, - tx_succeeded, - gas_used, - transaction_fee, - attached_gas, - _partition_by_block_number, - transactions_final_id, - inserted_timestamp, - modified_timestamp, - '{{ invocation_id }}' AS _invocation_id - FROM - {{ ref('_migrate_txs') }} - - {% else %} - WITH txs_with_receipts AS ( SELECT chunk_hash, @@ -142,5 +112,3 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM transactions_final - -{% endif %} diff --git a/models/silver/curated/defi/lending/burrow/silver__burrow_borrows.sql b/models/silver/defi/lending/burrow/silver__burrow_borrows.sql similarity index 100% rename from models/silver/curated/defi/lending/burrow/silver__burrow_borrows.sql rename to models/silver/defi/lending/burrow/silver__burrow_borrows.sql diff --git a/models/silver/curated/defi/lending/burrow/silver__burrow_borrows.yml b/models/silver/defi/lending/burrow/silver__burrow_borrows.yml similarity index 100% rename from models/silver/curated/defi/lending/burrow/silver__burrow_borrows.yml rename to models/silver/defi/lending/burrow/silver__burrow_borrows.yml diff --git a/models/silver/curated/defi/lending/burrow/silver__burrow_collaterals.sql b/models/silver/defi/lending/burrow/silver__burrow_collaterals.sql similarity index 100% rename from models/silver/curated/defi/lending/burrow/silver__burrow_collaterals.sql rename to models/silver/defi/lending/burrow/silver__burrow_collaterals.sql diff --git a/models/silver/curated/defi/lending/burrow/silver__burrow_collaterals.yml b/models/silver/defi/lending/burrow/silver__burrow_collaterals.yml similarity index 100% rename from models/silver/curated/defi/lending/burrow/silver__burrow_collaterals.yml rename to models/silver/defi/lending/burrow/silver__burrow_collaterals.yml diff --git a/models/silver/curated/defi/lending/burrow/silver__burrow_deposits.sql b/models/silver/defi/lending/burrow/silver__burrow_deposits.sql similarity index 100% rename from models/silver/curated/defi/lending/burrow/silver__burrow_deposits.sql rename to models/silver/defi/lending/burrow/silver__burrow_deposits.sql diff --git a/models/silver/curated/defi/lending/burrow/silver__burrow_deposits.yml b/models/silver/defi/lending/burrow/silver__burrow_deposits.yml similarity index 100% rename from models/silver/curated/defi/lending/burrow/silver__burrow_deposits.yml rename to models/silver/defi/lending/burrow/silver__burrow_deposits.yml diff --git a/models/silver/curated/defi/lending/burrow/silver__burrow_repays.sql b/models/silver/defi/lending/burrow/silver__burrow_repays.sql similarity index 100% rename from models/silver/curated/defi/lending/burrow/silver__burrow_repays.sql rename to models/silver/defi/lending/burrow/silver__burrow_repays.sql diff --git a/models/silver/curated/defi/lending/burrow/silver__burrow_repays.yml b/models/silver/defi/lending/burrow/silver__burrow_repays.yml similarity index 100% rename from models/silver/curated/defi/lending/burrow/silver__burrow_repays.yml rename to models/silver/defi/lending/burrow/silver__burrow_repays.yml diff --git a/models/silver/curated/defi/lending/burrow/silver__burrow_withdraws.sql b/models/silver/defi/lending/burrow/silver__burrow_withdraws.sql similarity index 100% rename from models/silver/curated/defi/lending/burrow/silver__burrow_withdraws.sql rename to models/silver/defi/lending/burrow/silver__burrow_withdraws.sql diff --git a/models/silver/curated/defi/lending/burrow/silver__burrow_withdraws.yml b/models/silver/defi/lending/burrow/silver__burrow_withdraws.yml similarity index 100% rename from models/silver/curated/defi/lending/burrow/silver__burrow_withdraws.yml rename to models/silver/defi/lending/burrow/silver__burrow_withdraws.yml diff --git a/models/silver/curated/defi/lending/silver__burrow_lending.sql b/models/silver/defi/lending/silver__burrow_lending.sql similarity index 100% rename from models/silver/curated/defi/lending/silver__burrow_lending.sql rename to models/silver/defi/lending/silver__burrow_lending.sql diff --git a/models/silver/curated/defi/silver__dex_swaps_v2.sql b/models/silver/defi/silver__dex_swaps_v2.sql similarity index 100% rename from models/silver/curated/defi/silver__dex_swaps_v2.sql rename to models/silver/defi/silver__dex_swaps_v2.sql diff --git a/models/silver/curated/defi/silver__dex_swaps_v2.yml b/models/silver/defi/silver__dex_swaps_v2.yml similarity index 100% rename from models/silver/curated/defi/silver__dex_swaps_v2.yml rename to models/silver/defi/silver__dex_swaps_v2.yml diff --git a/models/silver/curated/silver__lockup_actions.sql b/models/silver/lockup/silver__lockup_actions.sql similarity index 100% rename from models/silver/curated/silver__lockup_actions.sql rename to models/silver/lockup/silver__lockup_actions.sql diff --git a/models/silver/curated/silver__lockup_actions.yml b/models/silver/lockup/silver__lockup_actions.yml similarity index 100% rename from models/silver/curated/silver__lockup_actions.yml rename to models/silver/lockup/silver__lockup_actions.yml diff --git a/models/silver/curated/nft/sales/silver__nft_complete_nft_sales.sql b/models/silver/nft/sales/silver__nft_complete_nft_sales.sql similarity index 100% rename from models/silver/curated/nft/sales/silver__nft_complete_nft_sales.sql rename to models/silver/nft/sales/silver__nft_complete_nft_sales.sql diff --git a/models/silver/curated/nft/sales/silver__nft_complete_nft_sales.yml b/models/silver/nft/sales/silver__nft_complete_nft_sales.yml similarity index 100% rename from models/silver/curated/nft/sales/silver__nft_complete_nft_sales.yml rename to models/silver/nft/sales/silver__nft_complete_nft_sales.yml diff --git a/models/silver/curated/nft/sales/silver__nft_mintbase_sales.sql b/models/silver/nft/sales/silver__nft_mintbase_sales.sql similarity index 100% rename from models/silver/curated/nft/sales/silver__nft_mintbase_sales.sql rename to models/silver/nft/sales/silver__nft_mintbase_sales.sql diff --git a/models/silver/curated/nft/sales/silver__nft_other_sales.sql b/models/silver/nft/sales/silver__nft_other_sales.sql similarity index 100% rename from models/silver/curated/nft/sales/silver__nft_other_sales.sql rename to models/silver/nft/sales/silver__nft_other_sales.sql diff --git a/models/silver/curated/nft/sales/silver__nft_paras_sales.sql b/models/silver/nft/sales/silver__nft_paras_sales.sql similarity index 100% rename from models/silver/curated/nft/sales/silver__nft_paras_sales.sql rename to models/silver/nft/sales/silver__nft_paras_sales.sql diff --git a/models/silver/curated/nft/silver__nft_transfers.sql b/models/silver/nft/silver__nft_transfers.sql similarity index 100% rename from models/silver/curated/nft/silver__nft_transfers.sql rename to models/silver/nft/silver__nft_transfers.sql diff --git a/models/silver/curated/nft/silver__nft_transfers.yml b/models/silver/nft/silver__nft_transfers.yml similarity index 100% rename from models/silver/curated/nft/silver__nft_transfers.yml rename to models/silver/nft/silver__nft_transfers.yml diff --git a/models/silver/curated/nft/silver__standard_nft_mint_s3.sql b/models/silver/nft/silver__standard_nft_mint_s3.sql similarity index 100% rename from models/silver/curated/nft/silver__standard_nft_mint_s3.sql rename to models/silver/nft/silver__standard_nft_mint_s3.sql diff --git a/models/silver/curated/nft/silver__standard_nft_mint_s3.yml b/models/silver/nft/silver__standard_nft_mint_s3.yml similarity index 100% rename from models/silver/curated/nft/silver__standard_nft_mint_s3.yml rename to models/silver/nft/silver__standard_nft_mint_s3.yml diff --git a/models/silver/curated/staking/silver__pool_balance_daily.sql b/models/silver/staking/silver__pool_balance_daily.sql similarity index 100% rename from models/silver/curated/staking/silver__pool_balance_daily.sql rename to models/silver/staking/silver__pool_balance_daily.sql diff --git a/models/silver/curated/staking/silver__pool_balance_daily.yml b/models/silver/staking/silver__pool_balance_daily.yml similarity index 100% rename from models/silver/curated/staking/silver__pool_balance_daily.yml rename to models/silver/staking/silver__pool_balance_daily.yml diff --git a/models/silver/curated/staking/silver__pool_balances.sql b/models/silver/staking/silver__pool_balances.sql similarity index 100% rename from models/silver/curated/staking/silver__pool_balances.sql rename to models/silver/staking/silver__pool_balances.sql diff --git a/models/silver/curated/staking/silver__pool_balances.yml b/models/silver/staking/silver__pool_balances.yml similarity index 100% rename from models/silver/curated/staking/silver__pool_balances.yml rename to models/silver/staking/silver__pool_balances.yml diff --git a/models/silver/curated/staking/silver__pool_events.sql b/models/silver/staking/silver__pool_events.sql similarity index 100% rename from models/silver/curated/staking/silver__pool_events.sql rename to models/silver/staking/silver__pool_events.sql diff --git a/models/silver/curated/staking/silver__pool_events.yml b/models/silver/staking/silver__pool_events.yml similarity index 100% rename from models/silver/curated/staking/silver__pool_events.yml rename to models/silver/staking/silver__pool_events.yml diff --git a/models/silver/curated/staking/silver__staking_actions_v2.sql b/models/silver/staking/silver__staking_actions_v2.sql similarity index 100% rename from models/silver/curated/staking/silver__staking_actions_v2.sql rename to models/silver/staking/silver__staking_actions_v2.sql diff --git a/models/silver/curated/staking/silver__staking_actions_v2.yml b/models/silver/staking/silver__staking_actions_v2.yml similarity index 100% rename from models/silver/curated/staking/silver__staking_actions_v2.yml rename to models/silver/staking/silver__staking_actions_v2.yml diff --git a/models/silver/curated/staking/silver__staking_epochs.sql b/models/silver/staking/silver__staking_epochs.sql similarity index 100% rename from models/silver/curated/staking/silver__staking_epochs.sql rename to models/silver/staking/silver__staking_epochs.sql diff --git a/models/silver/curated/staking/silver__staking_epochs.yml b/models/silver/staking/silver__staking_epochs.yml similarity index 100% rename from models/silver/curated/staking/silver__staking_epochs.yml rename to models/silver/staking/silver__staking_epochs.yml diff --git a/models/silver/curated/staking/silver__staking_pools_s3.sql b/models/silver/staking/silver__staking_pools_s3.sql similarity index 100% rename from models/silver/curated/staking/silver__staking_pools_s3.sql rename to models/silver/staking/silver__staking_pools_s3.sql diff --git a/models/silver/curated/staking/silver__staking_pools_s3.yml b/models/silver/staking/silver__staking_pools_s3.yml similarity index 100% rename from models/silver/curated/staking/silver__staking_pools_s3.yml rename to models/silver/staking/silver__staking_pools_s3.yml diff --git a/models/silver/curated/defi/bridge/silver__bridge_allbridge.sql b/models/silver/transfers/bridging/silver__bridge_allbridge.sql similarity index 100% rename from models/silver/curated/defi/bridge/silver__bridge_allbridge.sql rename to models/silver/transfers/bridging/silver__bridge_allbridge.sql diff --git a/models/silver/curated/defi/bridge/silver__bridge_allbridge.yml b/models/silver/transfers/bridging/silver__bridge_allbridge.yml similarity index 100% rename from models/silver/curated/defi/bridge/silver__bridge_allbridge.yml rename to models/silver/transfers/bridging/silver__bridge_allbridge.yml diff --git a/models/silver/curated/defi/bridge/silver__bridge_multichain.sql b/models/silver/transfers/bridging/silver__bridge_multichain.sql similarity index 100% rename from models/silver/curated/defi/bridge/silver__bridge_multichain.sql rename to models/silver/transfers/bridging/silver__bridge_multichain.sql diff --git a/models/silver/curated/defi/bridge/silver__bridge_multichain.yml b/models/silver/transfers/bridging/silver__bridge_multichain.yml similarity index 100% rename from models/silver/curated/defi/bridge/silver__bridge_multichain.yml rename to models/silver/transfers/bridging/silver__bridge_multichain.yml diff --git a/models/silver/curated/defi/bridge/silver__bridge_rainbow.sql b/models/silver/transfers/bridging/silver__bridge_rainbow.sql similarity index 100% rename from models/silver/curated/defi/bridge/silver__bridge_rainbow.sql rename to models/silver/transfers/bridging/silver__bridge_rainbow.sql diff --git a/models/silver/curated/defi/bridge/silver__bridge_rainbow.yml b/models/silver/transfers/bridging/silver__bridge_rainbow.yml similarity index 100% rename from models/silver/curated/defi/bridge/silver__bridge_rainbow.yml rename to models/silver/transfers/bridging/silver__bridge_rainbow.yml diff --git a/models/silver/curated/defi/bridge/silver__bridge_wormhole.sql b/models/silver/transfers/bridging/silver__bridge_wormhole.sql similarity index 100% rename from models/silver/curated/defi/bridge/silver__bridge_wormhole.sql rename to models/silver/transfers/bridging/silver__bridge_wormhole.sql diff --git a/models/silver/curated/defi/bridge/silver__bridge_wormhole.yml b/models/silver/transfers/bridging/silver__bridge_wormhole.yml similarity index 100% rename from models/silver/curated/defi/bridge/silver__bridge_wormhole.yml rename to models/silver/transfers/bridging/silver__bridge_wormhole.yml diff --git a/models/silver/curated/token_transfers/native/silver__token_transfer_deposit.sql b/models/silver/transfers/native/silver__token_transfer_deposit.sql similarity index 100% rename from models/silver/curated/token_transfers/native/silver__token_transfer_deposit.sql rename to models/silver/transfers/native/silver__token_transfer_deposit.sql diff --git a/models/silver/curated/token_transfers/native/silver__token_transfer_deposit.yml b/models/silver/transfers/native/silver__token_transfer_deposit.yml similarity index 100% rename from models/silver/curated/token_transfers/native/silver__token_transfer_deposit.yml rename to models/silver/transfers/native/silver__token_transfer_deposit.yml diff --git a/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql b/models/silver/transfers/native/silver__token_transfer_native.sql similarity index 100% rename from models/silver/curated/token_transfers/native/silver__token_transfer_native.sql rename to models/silver/transfers/native/silver__token_transfer_native.sql diff --git a/models/silver/curated/token_transfers/native/silver__token_transfer_native.yml b/models/silver/transfers/native/silver__token_transfer_native.yml similarity index 100% rename from models/silver/curated/token_transfers/native/silver__token_transfer_native.yml rename to models/silver/transfers/native/silver__token_transfer_native.yml diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_event.sql similarity index 100% rename from models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.sql rename to models/silver/transfers/non_native/silver__token_transfer_ft_transfers_event.sql diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.yml b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_event.yml similarity index 100% rename from models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_event.yml rename to models/silver/transfers/non_native/silver__token_transfer_ft_transfers_event.yml diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql similarity index 100% rename from models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.sql rename to models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.yml b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.yml similarity index 100% rename from models/silver/curated/token_transfers/non_native/silver__token_transfer_ft_transfers_method.yml rename to models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.yml diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql b/models/silver/transfers/non_native/silver__token_transfer_liquidity.sql similarity index 100% rename from models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.sql rename to models/silver/transfers/non_native/silver__token_transfer_liquidity.sql diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.yml b/models/silver/transfers/non_native/silver__token_transfer_liquidity.yml similarity index 100% rename from models/silver/curated/token_transfers/non_native/silver__token_transfer_liquidity.yml rename to models/silver/transfers/non_native/silver__token_transfer_liquidity.yml diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql b/models/silver/transfers/non_native/silver__token_transfer_mints.sql similarity index 100% rename from models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.sql rename to models/silver/transfers/non_native/silver__token_transfer_mints.sql diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.yml b/models/silver/transfers/non_native/silver__token_transfer_mints.yml similarity index 100% rename from models/silver/curated/token_transfers/non_native/silver__token_transfer_mints.yml rename to models/silver/transfers/non_native/silver__token_transfer_mints.yml diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql b/models/silver/transfers/non_native/silver__token_transfer_orders.sql similarity index 100% rename from models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.sql rename to models/silver/transfers/non_native/silver__token_transfer_orders.sql diff --git a/models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.yml b/models/silver/transfers/non_native/silver__token_transfer_orders.yml similarity index 100% rename from models/silver/curated/token_transfers/non_native/silver__token_transfer_orders.yml rename to models/silver/transfers/non_native/silver__token_transfer_orders.yml diff --git a/models/silver/curated/token_transfers/silver__token_transfers_complete.sql b/models/silver/transfers/silver__token_transfers_complete.sql similarity index 100% rename from models/silver/curated/token_transfers/silver__token_transfers_complete.sql rename to models/silver/transfers/silver__token_transfers_complete.sql diff --git a/models/silver/curated/token_transfers/silver__token_transfers_complete.yml b/models/silver/transfers/silver__token_transfers_complete.yml similarity index 100% rename from models/silver/curated/token_transfers/silver__token_transfers_complete.yml rename to models/silver/transfers/silver__token_transfers_complete.yml