diff --git a/models/bronze/bronze__streamline_FR_blocks.sql b/models/bronze/bronze__streamline_FR_blocks.sql index 1a14620..65873a9 100644 --- a/models/bronze/bronze__streamline_FR_blocks.sql +++ b/models/bronze/bronze__streamline_FR_blocks.sql @@ -12,7 +12,7 @@ SELECT inserted_timestamp FROM {{ ref('bronze__streamline_FR_blocks_v2') }} -UNION ALL + {# UNION ALL SELECT VALUE, _partition_by_block_id AS partition_key, @@ -22,3 +22,4 @@ SELECT _INSERTED_TIMESTAMP FROM {{ ref('bronze__streamline_FR_blocks_v1') }} + #} diff --git a/models/bronze/bronze__streamline_FR_blocks_v1.sql b/models/bronze/bronze__streamline_FR_blocks_v1.sql index 4a96db7..c82b993 100644 --- a/models/bronze/bronze__streamline_FR_blocks_v1.sql +++ b/models/bronze/bronze__streamline_FR_blocks_v1.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + enabled = false ) }} {{ streamline_external_table_FR_query( model = "blocks", diff --git a/models/bronze/bronze__streamline_FR_transactions.sql b/models/bronze/bronze__streamline_FR_transactions.sql index f751fec..2ec12fe 100644 --- a/models/bronze/bronze__streamline_FR_transactions.sql +++ b/models/bronze/bronze__streamline_FR_transactions.sql @@ -12,7 +12,7 @@ SELECT inserted_timestamp FROM {{ ref('bronze__streamline_FR_transactions_v2') }} -UNION ALL + {# UNION ALL SELECT VALUE, _partition_by_block_id AS partition_key, @@ -22,3 +22,4 @@ SELECT _INSERTED_TIMESTAMP FROM {{ ref('bronze__streamline_FR_transactions_v1') }} + #} diff --git a/models/bronze/bronze__streamline_FR_transactions_v1.sql b/models/bronze/bronze__streamline_FR_transactions_v1.sql index 0173f76..4cda593 100644 --- a/models/bronze/bronze__streamline_FR_transactions_v1.sql +++ b/models/bronze/bronze__streamline_FR_transactions_v1.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + enabled = false ) }} {% set model = this.identifier.split("_") [-1] %} diff --git a/models/sources.yml b/models/sources.yml index 3d9677d..bce033d 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -6,20 +6,23 @@ sources: schema: | {{ "COSMOS_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "COSMOS" }} tables: - - name: blocks - - name: transactions - identifier: tx_search - - name: validators + # - name: blocks + # - name: transactions + # identifier: tx_search + # - name: validators - name: blockchain - name: blocks_ch1 + identifier: migrated_blocks_ch1 - name: tx_search_ch1 - - name: validators_ch1 + identifier: migrated_tx_search_ch1 - name: blocks_ch2 + identifier: migrated_blocks_ch2 - name: tx_search_ch2 - - name: validators_ch2 + identifier: migrated_tx_search_ch2 - name: blocks_ch3 + identifier: migrated_blocks_ch3 - name: tx_search_ch3 - - name: validators_ch3 + identifier: migrated_tx_search_ch3 - name: blocks_v2 - name: txcount_v2 - name: transactions_v2 diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_validators_ch1.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_validators_ch1.sql deleted file mode 100644 index 5578264..0000000 --- a/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_validators_ch1.sql +++ /dev/null @@ -1,54 +0,0 @@ -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["id"] -) }} - -WITH meta AS ( - - SELECT - last_modified, - file_name - FROM - TABLE( - information_schema.external_table_files( - table_name => '{{ source( "bronze", "validators_ch1") }}' - ) - ) A -) - -{% if is_incremental() %}, -max_date AS ( - SELECT - COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP - FROM - {{ this }}) - {% endif %} - SELECT - {{ dbt_utils.generate_surrogate_key( - ['block_number'] - ) }} AS id, - block_number, - last_modified AS _inserted_timestamp - FROM - {{ source( - "bronze", - "validators_ch1" - ) }} - JOIN meta b - ON b.file_name = metadata$filename - -{% if is_incremental() %} -WHERE - b.last_modified > ( - SELECT - max_INSERTED_TIMESTAMP - FROM - max_date - ) -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_validators_ch2.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_validators_ch2.sql deleted file mode 100644 index aa8e034..0000000 --- a/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_validators_ch2.sql +++ /dev/null @@ -1,54 +0,0 @@ -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["id"] -) }} - -WITH meta AS ( - - SELECT - last_modified, - file_name - FROM - TABLE( - information_schema.external_table_files( - table_name => '{{ source( "bronze", "validators_ch2") }}' - ) - ) A -) - -{% if is_incremental() %}, -max_date AS ( - SELECT - COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP - FROM - {{ this }}) - {% endif %} - SELECT - {{ dbt_utils.generate_surrogate_key( - ['block_number'] - ) }} AS id, - block_number, - last_modified AS _inserted_timestamp - FROM - {{ source( - "bronze", - "validators_ch2" - ) }} - JOIN meta b - ON b.file_name = metadata$filename - -{% if is_incremental() %} -WHERE - b.last_modified > ( - SELECT - max_INSERTED_TIMESTAMP - FROM - max_date - ) -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_validators_ch3.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_validators_ch3.sql deleted file mode 100644 index e540898..0000000 --- a/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_validators_ch3.sql +++ /dev/null @@ -1,54 +0,0 @@ -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["id"] -) }} - -WITH meta AS ( - - SELECT - last_modified, - file_name - FROM - TABLE( - information_schema.external_table_files( - table_name => '{{ source( "bronze", "validators_ch3") }}' - ) - ) A -) - -{% if is_incremental() %}, -max_date AS ( - SELECT - COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP - FROM - {{ this }}) - {% endif %} - SELECT - {{ dbt_utils.generate_surrogate_key( - ['block_number'] - ) }} AS id, - block_number, - last_modified AS _inserted_timestamp - FROM - {{ source( - "bronze", - "validators_ch3" - ) }} - JOIN meta b - ON b.file_name = metadata$filename - -{% if is_incremental() %} -WHERE - b.last_modified > ( - SELECT - max_INSERTED_TIMESTAMP - FROM - max_date - ) -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/cosmos_hub_1/streamline__validators_genesis_backfill_ch1.sql b/models/streamline/genesis_backfill/cosmos_hub_1/streamline__validators_genesis_backfill_ch1.sql deleted file mode 100644 index 3798065..0000000 --- a/models/streamline/genesis_backfill/cosmos_hub_1/streamline__validators_genesis_backfill_ch1.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_get_cosmos_validators(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch1/rpc','call_type','non_batch','external_table','validators_ch1','producer_batch_size','256000','worker_batch_size','8000'))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -SELECT - {{ dbt_utils.generate_surrogate_key( - ['block_number'] - ) }} AS id, - block_number -FROM - {{ ref("streamline__blocks_ch1") }} -EXCEPT -SELECT - id, - block_number -FROM - {{ ref("streamline__complete_validators_ch1") }} -ORDER BY - block_number - diff --git a/models/streamline/genesis_backfill/cosmos_hub_2/streamline__validators_genesis_backfill_ch2.sql b/models/streamline/genesis_backfill/cosmos_hub_2/streamline__validators_genesis_backfill_ch2.sql deleted file mode 100644 index 512a04e..0000000 --- a/models/streamline/genesis_backfill/cosmos_hub_2/streamline__validators_genesis_backfill_ch2.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_get_cosmos_validators(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch2/rpc','call_type','non_batch','external_table','validators_ch2','producer_batch_size','256000','worker_batch_size','8000'))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -SELECT - {{ dbt_utils.generate_surrogate_key( - ['block_number'] - ) }} AS id, - block_number -FROM - {{ ref("streamline__blocks_ch2") }} -EXCEPT -SELECT - id, - block_number -FROM - {{ ref("streamline__complete_validators_ch2") }} -ORDER BY - block_number - diff --git a/models/streamline/genesis_backfill/cosmos_hub_3/streamline__validators_genesis_backfill_ch3.sql b/models/streamline/genesis_backfill/cosmos_hub_3/streamline__validators_genesis_backfill_ch3.sql deleted file mode 100644 index 08dbedb..0000000 --- a/models/streamline/genesis_backfill/cosmos_hub_3/streamline__validators_genesis_backfill_ch3.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_get_cosmos_validators(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch3/rpc','call_type','non_batch','external_table','validators_ch3','producer_batch_size','256000','worker_batch_size','8000'))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -SELECT - {{ dbt_utils.generate_surrogate_key( - ['block_number'] - ) }} AS id, - block_number -FROM - {{ ref("streamline__blocks_ch3") }} -EXCEPT -SELECT - id, - block_number -FROM - {{ ref("streamline__complete_validators_ch3") }} -ORDER BY - block_number -