diff --git a/Makefile b/Makefile index 2a695b8..eb8338c 100644 --- a/Makefile +++ b/Makefile @@ -3,4 +3,81 @@ SHELL := /bin/bash dbt-console: docker-compose run dbt_console -.PHONY: dbt-console \ No newline at end of file +.PHONY: dbt-console + +SHELL := /bin/bash + +# set default target +DBT_TARGET ?= dev +AWS_LAMBDA_ROLE ?= aws_lambda_cosmos_api_dev + +sl-cosmos-api: + dbt run-operation create_aws_cosmos_api \ + --profile cosmos \ + --target $(DBT_TARGET) \ + --profiles-dir ~/.dbt/ + +udfs: + dbt run-operation create_udfs \ + --vars '{"UPDATE_UDFS_AND_SPS":True}' \ + --profile cosmos \ + --target $(DBT_TARGET) \ + --profiles-dir ~/.dbt/ + +complete: + dbt run \ + --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + -m 1+models/silver/streamline/core/complete \ + --profile cosmos \ + --target $(DBT_TARGET) \ + --profiles-dir ~/.dbt + +streamline: sl-cosmos-api udfs + +blocks_history: + dbt run \ + --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + -m 1+models/streamline/genesis_backfill/cosmos_hub_1/streamline__blocks_genesis_backfill_ch1.sql \ + --profile cosmos \ + --target $(DBT_TARGET) \ + --profiles-dir ~/.dbt + +tx_history: + dbt run \ + --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + -m 1+models/streamline/genesis_backfill/cosmos_hub_1/streamline__transactions_genesis_backfill_ch1.sql \ + --profile cosmos \ + --target $(DBT_TARGET) \ + --profiles-dir ~/.dbt + +validators_history: + dbt run \ + --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + -m 1+models/streamline/genesis_backfill/cosmos_hub_1/streamline__validators_genesis_backfill_ch1.sql \ + --profile cosmos \ + --target $(DBT_TARGET) \ + --profiles-dir ~/.dbt + +blocks_realtime: + dbt run \ + --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + -m 1+models/streamline/genesis_backfill/tests/streamline__blocks_realtime_test.sql \ + --profile cosmos \ + --target dev \ + --profiles-dir ~/.dbt + +tx_realtime: + dbt run \ + --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + -m 1+models/streamline/genesis_backfill/tests/streamline__transactions_realtime_test.sql \ + --profile cosmos \ + --target dev \ + --profiles-dir ~/.dbt + +validators_realtime: + dbt run \ + --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + -m 1+models/streamline/genesis_backfill/tests/streamline__validators_realtime_test.sql \ + --profile cosmos \ + --target dev \ + --profiles-dir ~/.dbt \ No newline at end of file diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 956c4a2..9a83c97 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -14,7 +14,6 @@ {{ create_udf_get_cosmos_validators() }} {{ create_udf_get_cosmos_generic() }} {{ create_udf_get_cosmos_chainhead() }} - {% endset %} {% do run_query(sql) %} {% endif %} diff --git a/macros/streamline/api_integrations.sql b/macros/streamline/api_integrations.sql index 51a9994..69afccf 100644 --- a/macros/streamline/api_integrations.sql +++ b/macros/streamline/api_integrations.sql @@ -1,9 +1,8 @@ {% macro create_aws_cosmos_api() %} {% if target.name == "prod" %} {% set sql %} - CREATE api integration IF NOT EXISTS aws_cosmos_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/snowflake-api-cosmos' api_allowed_prefixes = ( - 'https://z97ik1b2d0.execute-api.us-east-1.amazonaws.com/dev/', - 'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/' + CREATE api integration IF NOT EXISTS aws_cosmos_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/cosmos-api-prod-rolesnowflakeudfsAF733095-14KYNLUQ3CWV2' api_allowed_prefixes = ( + 'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/' ) enabled = TRUE; {% endset %} {% do run_query(sql) %} diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql index 69e7223..2dab54a 100644 --- a/macros/streamline/streamline_udfs.sql +++ b/macros/streamline/streamline_udfs.sql @@ -2,10 +2,11 @@ CREATE OR REPLACE EXTERNAL FUNCTION streamline.udf_get_cosmos_blocks( json variant - ) returns text api_integration = aws_cosmos_api AS {% if target.name == "prod" %} - 'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_blocks' + ) returns text {% if target.name == "prod" %} + api_integration = aws_cosmos_api AS 'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_blocks' {% else %} - 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_blocks' + api_integration = aws_cosmos_api_dev AS 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_blocks' + {%- endif %}; {% endmacro %} @@ -13,10 +14,10 @@ CREATE OR REPLACE EXTERNAL FUNCTION streamline.udf_get_cosmos_transactions( json variant - ) returns text api_integration = aws_cosmos_api AS {% if target.name == "prod" %} - 'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_transactions' + ) returns text {% if target.name == "prod" %} + api_integration = aws_cosmos_api AS 'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_transactions' {% else %} - 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_transactions' + api_integration = aws_cosmos_api_dev AS 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_transactions' {%- endif %}; {% endmacro %} @@ -24,10 +25,10 @@ CREATE OR REPLACE EXTERNAL FUNCTION streamline.udf_get_cosmos_validators( json variant - ) returns text api_integration = aws_cosmos_api AS {% if target.name == "prod" %} - 'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_validators' + ) returns text {% if target.name == "prod" %} + api_integration = aws_cosmos_api AS 'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_validators' {% else %} - 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_validators' + api_integration = aws_cosmos_api_dev AS 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_validators' {%- endif %}; {% endmacro %} @@ -35,17 +36,19 @@ CREATE OR REPLACE EXTERNAL FUNCTION streamline.udf_get_cosmos_generic( json variant - ) returns text api_integration = aws_cosmos_api AS {% if target.name == "prod" %} - 'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_generic' + ) returns text{% if target.name == "prod" %} + api_integration = aws_cosmos_api AS 'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_generic' {% else %} - 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_generic' + api_integration = aws_cosmos_api_dev AS 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_generic' {%- endif %}; {% endmacro %} {% macro create_udf_get_cosmos_chainhead() %} - CREATE EXTERNAL FUNCTION IF NOT EXISTS streamline.udf_get_cosmos_chainhead() returns variant api_integration = aws_cosmos_api AS {% if target.name == "prod" %} - 'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/get_cosmos_chainhead' + CREATE + OR REPLACE EXTERNAL FUNCTION streamline.udf_get_cosmos_chainhead() + returns variant {% if target.name == "prod" %} + api_integration = aws_cosmos_api AS 'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/get_cosmos_chainhead' {% else %} - 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/get_cosmos_chainhead' + api_integration = aws_cosmos_api_dev AS'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/get_cosmos_chainhead' {%- endif %}; {% endmacro %} diff --git a/models/sources.yml b/models/sources.yml index 7be9a32..09351b9 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -11,6 +11,16 @@ sources: identifier: tx_search - name: validators - name: blockchain + - name: blocks_ch1 + - name: tx_search_ch1 + - name: validators_ch1 + - name: blocks_ch2 + - name: tx_search_ch2 + - name: validators_ch2 + - name: blocks_ch3 + - name: tx_search_ch3 + - name: validators_ch3 + - name: osmo database: osmosis schema: silver diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_blocks_ch1.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_blocks_ch1.sql new file mode 100644 index 0000000..2122e20 --- /dev/null +++ b/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_blocks_ch1.sql @@ -0,0 +1,54 @@ +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"] +) }} + +WITH meta AS ( + + SELECT + last_modified, + file_name + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "blocks_ch1") }}' + ) + ) A +) + +{% if is_incremental() %}, +max_date AS ( + SELECT + COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP + FROM + {{ this }}) + {% endif %} + SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number, + last_modified AS _inserted_timestamp + FROM + {{ source( + "bronze_streamline", + "blocks_ch1" + ) }} + JOIN meta b + ON b.file_name = metadata$filename + +{% if is_incremental() %} +WHERE + b.last_modified > ( + SELECT + max_INSERTED_TIMESTAMP + FROM + max_date + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_transactions_ch1.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_transactions_ch1.sql new file mode 100644 index 0000000..ad5e07a --- /dev/null +++ b/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_transactions_ch1.sql @@ -0,0 +1,54 @@ +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"] +) }} + +WITH meta AS ( + + SELECT + last_modified, + file_name + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "tx_search_ch1") }}' + ) + ) A +) + +{% if is_incremental() %}, +max_date AS ( + SELECT + COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP + FROM + {{ this }}) + {% endif %} + SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number, + last_modified AS _inserted_timestamp + FROM + {{ source( + "bronze_streamline", + "tx_search_ch1" + ) }} + JOIN meta b + ON b.file_name = metadata$filename + +{% if is_incremental() %} +WHERE + b.last_modified > ( + SELECT + max_INSERTED_TIMESTAMP + FROM + max_date + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_validators_ch1.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_validators_ch1.sql new file mode 100644 index 0000000..81dbadf --- /dev/null +++ b/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_validators_ch1.sql @@ -0,0 +1,54 @@ +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"] +) }} + +WITH meta AS ( + + SELECT + last_modified, + file_name + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "validators_ch1") }}' + ) + ) A +) + +{% if is_incremental() %}, +max_date AS ( + SELECT + COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP + FROM + {{ this }}) + {% endif %} + SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number, + last_modified AS _inserted_timestamp + FROM + {{ source( + "bronze_streamline", + "validators_ch1" + ) }} + JOIN meta b + ON b.file_name = metadata$filename + +{% if is_incremental() %} +WHERE + b.last_modified > ( + SELECT + max_INSERTED_TIMESTAMP + FROM + max_date + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_blocks_ch2.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_blocks_ch2.sql new file mode 100644 index 0000000..36431bf --- /dev/null +++ b/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_blocks_ch2.sql @@ -0,0 +1,54 @@ +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"] +) }} + +WITH meta AS ( + + SELECT + last_modified, + file_name + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "blocks_ch2") }}' + ) + ) A +) + +{% if is_incremental() %}, +max_date AS ( + SELECT + COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP + FROM + {{ this }}) + {% endif %} + SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number, + last_modified AS _inserted_timestamp + FROM + {{ source( + "bronze_streamline", + "blocks_ch2" + ) }} + JOIN meta b + ON b.file_name = metadata$filename + +{% if is_incremental() %} +WHERE + b.last_modified > ( + SELECT + max_INSERTED_TIMESTAMP + FROM + max_date + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_transactions_ch2.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_transactions_ch2.sql new file mode 100644 index 0000000..d742689 --- /dev/null +++ b/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_transactions_ch2.sql @@ -0,0 +1,54 @@ +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"] +) }} + +WITH meta AS ( + + SELECT + last_modified, + file_name + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "tx_search_ch2") }}' + ) + ) A +) + +{% if is_incremental() %}, +max_date AS ( + SELECT + COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP + FROM + {{ this }}) + {% endif %} + SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number, + last_modified AS _inserted_timestamp + FROM + {{ source( + "bronze_streamline", + "tx_search_ch2" + ) }} + JOIN meta b + ON b.file_name = metadata$filename + +{% if is_incremental() %} +WHERE + b.last_modified > ( + SELECT + max_INSERTED_TIMESTAMP + FROM + max_date + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_validators_ch2.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_validators_ch2.sql new file mode 100644 index 0000000..425585f --- /dev/null +++ b/models/streamline/genesis_backfill/complete/cosmos_hub_2/streamline__complete_validators_ch2.sql @@ -0,0 +1,54 @@ +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"] +) }} + +WITH meta AS ( + + SELECT + last_modified, + file_name + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "validators_ch2") }}' + ) + ) A +) + +{% if is_incremental() %}, +max_date AS ( + SELECT + COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP + FROM + {{ this }}) + {% endif %} + SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number, + last_modified AS _inserted_timestamp + FROM + {{ source( + "bronze_streamline", + "validators_ch2" + ) }} + JOIN meta b + ON b.file_name = metadata$filename + +{% if is_incremental() %} +WHERE + b.last_modified > ( + SELECT + max_INSERTED_TIMESTAMP + FROM + max_date + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_blocks_ch3.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_blocks_ch3.sql new file mode 100644 index 0000000..312d307 --- /dev/null +++ b/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_blocks_ch3.sql @@ -0,0 +1,54 @@ +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"] +) }} + +WITH meta AS ( + + SELECT + last_modified, + file_name + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "blocks_ch3") }}' + ) + ) A +) + +{% if is_incremental() %}, +max_date AS ( + SELECT + COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP + FROM + {{ this }}) + {% endif %} + SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number, + last_modified AS _inserted_timestamp + FROM + {{ source( + "bronze_streamline", + "blocks_ch3" + ) }} + JOIN meta b + ON b.file_name = metadata$filename + +{% if is_incremental() %} +WHERE + b.last_modified > ( + SELECT + max_INSERTED_TIMESTAMP + FROM + max_date + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_transactions_ch3.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_transactions_ch3.sql new file mode 100644 index 0000000..c82067e --- /dev/null +++ b/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_transactions_ch3.sql @@ -0,0 +1,54 @@ +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"] +) }} + +WITH meta AS ( + + SELECT + last_modified, + file_name + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "tx_search_ch3") }}' + ) + ) A +) + +{% if is_incremental() %}, +max_date AS ( + SELECT + COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP + FROM + {{ this }}) + {% endif %} + SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number, + last_modified AS _inserted_timestamp + FROM + {{ source( + "bronze_streamline", + "tx_search_ch3" + ) }} + JOIN meta b + ON b.file_name = metadata$filename + +{% if is_incremental() %} +WHERE + b.last_modified > ( + SELECT + max_INSERTED_TIMESTAMP + FROM + max_date + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_validators_ch3.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_validators_ch3.sql new file mode 100644 index 0000000..0a9646e --- /dev/null +++ b/models/streamline/genesis_backfill/complete/cosmos_hub_3/streamline__complete_validators_ch3.sql @@ -0,0 +1,54 @@ +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"] +) }} + +WITH meta AS ( + + SELECT + last_modified, + file_name + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "validators_ch3") }}' + ) + ) A +) + +{% if is_incremental() %}, +max_date AS ( + SELECT + COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP + FROM + {{ this }}) + {% endif %} + SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number, + last_modified AS _inserted_timestamp + FROM + {{ source( + "bronze_streamline", + "validators_ch3" + ) }} + JOIN meta b + ON b.file_name = metadata$filename + +{% if is_incremental() %} +WHERE + b.last_modified > ( + SELECT + max_INSERTED_TIMESTAMP + FROM + max_date + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/genesis_backfill/cosmos_hub_1/streamline__blocks_genesis_backfill_ch1.sql b/models/streamline/genesis_backfill/cosmos_hub_1/streamline__blocks_genesis_backfill_ch1.sql new file mode 100644 index 0000000..4d5df9a --- /dev/null +++ b/models/streamline/genesis_backfill/cosmos_hub_1/streamline__blocks_genesis_backfill_ch1.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_get_cosmos_blocks(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch1/rpc', 'call_type','non_batch','external_table','blocks_ch1','producer_batch_size','1024000','worker_batch_size','32000'))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number +FROM + {{ ref("streamline__blocks_ch1") }} +EXCEPT +SELECT + id, + block_number +FROM + {{ ref("streamline__complete_blocks_ch1") }} +ORDER BY + block_number \ No newline at end of file diff --git a/models/streamline/genesis_backfill/cosmos_hub_1/streamline__transactions_genesis_backfill_ch1.sql b/models/streamline/genesis_backfill/cosmos_hub_1/streamline__transactions_genesis_backfill_ch1.sql new file mode 100644 index 0000000..4385d65 --- /dev/null +++ b/models/streamline/genesis_backfill/cosmos_hub_1/streamline__transactions_genesis_backfill_ch1.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_get_cosmos_transactions(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch1/rpc','call_type','non_batch','external_table','tx_search_ch1','producer_batch_size','256000','worker_batch_size','8000'))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number +FROM + {{ ref("streamline__blocks_ch1") }} +EXCEPT +SELECT + id, + block_number +FROM + {{ ref("streamline__complete_transactions_ch1") }} +ORDER BY + block_number + diff --git a/models/streamline/genesis_backfill/cosmos_hub_1/streamline__validators_genesis_backfill_ch1.sql b/models/streamline/genesis_backfill/cosmos_hub_1/streamline__validators_genesis_backfill_ch1.sql new file mode 100644 index 0000000..3798065 --- /dev/null +++ b/models/streamline/genesis_backfill/cosmos_hub_1/streamline__validators_genesis_backfill_ch1.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_get_cosmos_validators(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch1/rpc','call_type','non_batch','external_table','validators_ch1','producer_batch_size','256000','worker_batch_size','8000'))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number +FROM + {{ ref("streamline__blocks_ch1") }} +EXCEPT +SELECT + id, + block_number +FROM + {{ ref("streamline__complete_validators_ch1") }} +ORDER BY + block_number + diff --git a/models/streamline/genesis_backfill/cosmos_hub_2/streamline__blocks_genesis_backfill_ch2.sql b/models/streamline/genesis_backfill/cosmos_hub_2/streamline__blocks_genesis_backfill_ch2.sql new file mode 100644 index 0000000..3712b4e --- /dev/null +++ b/models/streamline/genesis_backfill/cosmos_hub_2/streamline__blocks_genesis_backfill_ch2.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_get_cosmos_blocks(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch2/rpc', 'call_type','non_batch','external_table','blocks_ch2','producer_batch_size','1024000','worker_batch_size','32000'))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number +FROM + {{ ref("streamline__blocks_ch2") }} +EXCEPT +SELECT + id, + block_number +FROM + {{ ref("streamline__complete_blocks_ch2") }} +ORDER BY + block_number \ No newline at end of file diff --git a/models/streamline/genesis_backfill/cosmos_hub_2/streamline__transactions_genesis_backfill_ch2.sql b/models/streamline/genesis_backfill/cosmos_hub_2/streamline__transactions_genesis_backfill_ch2.sql new file mode 100644 index 0000000..67f726d --- /dev/null +++ b/models/streamline/genesis_backfill/cosmos_hub_2/streamline__transactions_genesis_backfill_ch2.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_get_cosmos_transactions(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch2/rpc','call_type','non_batch','external_table','tx_search_ch2','producer_batch_size','256000','worker_batch_size','8000'))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number +FROM + {{ ref("streamline__blocks_ch2") }} +EXCEPT +SELECT + id, + block_number +FROM + {{ ref("streamline__complete_transactions_ch2") }} +ORDER BY + block_number + diff --git a/models/streamline/genesis_backfill/cosmos_hub_2/streamline__validators_genesis_backfill_ch2.sql b/models/streamline/genesis_backfill/cosmos_hub_2/streamline__validators_genesis_backfill_ch2.sql new file mode 100644 index 0000000..512a04e --- /dev/null +++ b/models/streamline/genesis_backfill/cosmos_hub_2/streamline__validators_genesis_backfill_ch2.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_get_cosmos_validators(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch2/rpc','call_type','non_batch','external_table','validators_ch2','producer_batch_size','256000','worker_batch_size','8000'))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number +FROM + {{ ref("streamline__blocks_ch2") }} +EXCEPT +SELECT + id, + block_number +FROM + {{ ref("streamline__complete_validators_ch2") }} +ORDER BY + block_number + diff --git a/models/streamline/genesis_backfill/cosmos_hub_3/streamline__blocks_genesis_backfill_ch3.sql b/models/streamline/genesis_backfill/cosmos_hub_3/streamline__blocks_genesis_backfill_ch3.sql new file mode 100644 index 0000000..4ac2c83 --- /dev/null +++ b/models/streamline/genesis_backfill/cosmos_hub_3/streamline__blocks_genesis_backfill_ch3.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_get_cosmos_blocks(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch3/rpc', 'call_type','non_batch','external_table','blocks_ch3','producer_batch_size','1024000','worker_batch_size','32000'))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number +FROM + {{ ref("streamline__blocks_ch3") }} +EXCEPT +SELECT + id, + block_number +FROM + {{ ref("streamline__complete_blocks_ch3") }} +ORDER BY + block_number \ No newline at end of file diff --git a/models/streamline/genesis_backfill/cosmos_hub_3/streamline__transactions_genesis_backfill_ch3.sql b/models/streamline/genesis_backfill/cosmos_hub_3/streamline__transactions_genesis_backfill_ch3.sql new file mode 100644 index 0000000..109b98c --- /dev/null +++ b/models/streamline/genesis_backfill/cosmos_hub_3/streamline__transactions_genesis_backfill_ch3.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_get_cosmos_transactions(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch3/rpc','call_type','non_batch','external_table','tx_search_ch3','producer_batch_size','256000','worker_batch_size','8000'))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number +FROM + {{ ref("streamline__blocks_ch3") }} +EXCEPT +SELECT + id, + block_number +FROM + {{ ref("streamline__complete_transactions_ch3") }} +ORDER BY + block_number + diff --git a/models/streamline/genesis_backfill/cosmos_hub_3/streamline__validators_genesis_backfill_ch3.sql b/models/streamline/genesis_backfill/cosmos_hub_3/streamline__validators_genesis_backfill_ch3.sql new file mode 100644 index 0000000..08dbedb --- /dev/null +++ b/models/streamline/genesis_backfill/cosmos_hub_3/streamline__validators_genesis_backfill_ch3.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_get_cosmos_validators(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch3/rpc','call_type','non_batch','external_table','validators_ch3','producer_batch_size','256000','worker_batch_size','8000'))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number +FROM + {{ ref("streamline__blocks_ch3") }} +EXCEPT +SELECT + id, + block_number +FROM + {{ ref("streamline__complete_validators_ch3") }} +ORDER BY + block_number + diff --git a/models/streamline/genesis_backfill/streamline__blocks_ch1.sql b/models/streamline/genesis_backfill/streamline__blocks_ch1.sql new file mode 100644 index 0000000..23a883c --- /dev/null +++ b/models/streamline/genesis_backfill/streamline__blocks_ch1.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = "view", + tags = ['streamline_view'] +) }} + +SELECT + height as block_number +FROM + TABLE(streamline.udtf_get_base_table(500042)) +WHERE + height between 0 and 500042 // https://hub.cosmos.network/main/roadmap/#cosmos-hub-summary \ No newline at end of file diff --git a/models/streamline/genesis_backfill/streamline__blocks_ch2.sql b/models/streamline/genesis_backfill/streamline__blocks_ch2.sql new file mode 100644 index 0000000..a61f320 --- /dev/null +++ b/models/streamline/genesis_backfill/streamline__blocks_ch2.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = "view", + tags = ['streamline_view'] +) }} + +SELECT + height as block_number +FROM + TABLE(streamline.udtf_get_base_table(2901999)) +WHERE + height between 500043 and 2901999 // https://hub.cosmos.network/main/roadmap/#cosmos-hub-summary \ No newline at end of file diff --git a/models/streamline/genesis_backfill/streamline__blocks_ch3.sql b/models/streamline/genesis_backfill/streamline__blocks_ch3.sql new file mode 100644 index 0000000..a5f0493 --- /dev/null +++ b/models/streamline/genesis_backfill/streamline__blocks_ch3.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = "view", + tags = ['streamline_view'] +) }} + +SELECT + height as block_number +FROM + TABLE(streamline.udtf_get_base_table(5200790)) +WHERE + height between 2902000 and 5200790 // https://hub.cosmos.network/main/roadmap/#cosmos-hub-summary \ No newline at end of file diff --git a/packages.yml b/packages.yml index 5772608..14e0503 100644 --- a/packages.yml +++ b/packages.yml @@ -6,4 +6,4 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-utils.git revision: "v1.3.0" - package: get-select/dbt_snowflake_query_tags - version: [">=2.0.0", "<3.0.0"] + version: [">=2.0.0", "<3.0.0"] \ No newline at end of file