mirror of
https://github.com/FlipsideCrypto/cosmos-models.git
synced 2026-02-06 11:01:45 +00:00
Stream 417 network version genesis backfill (#44)
* added cdk migrated dev udf url * added genesis backfill models * added base udtf genesis backfill * added ch1,ch2,ch3 backfill block ranges * Stream 417 genesis backfill (#43) * added cdk migrated dev udf url * added genesis backfill models * added base udtf genesis backfill * added ch1,ch2,ch3 backfill block ranges --------- Co-authored-by: shah <info@shahnewazkhan.ca> * added validators genesis backfill * merged in dev * removed get-select/dbt_snowflake_query_tags * added query_tags package * fix newline * update packages newline * refactored genesis backfill complete folder * added complete except clause * removed realtime test models * added external_table udf param * added prod cdk udf api integrations * tuned to 300 rps * added ch2 & ch3 backfill models * removed debug logs from macros --------- Co-authored-by: shah <info@shahnewazkhan.ca>
This commit is contained in:
Shah Newaz Khan
GitHub
parent
a522e04c00
commit
09e85213c0
79
Makefile
79
Makefile
@ -3,4 +3,81 @@ SHELL := /bin/bash
|
||||
dbt-console:
|
||||
docker-compose run dbt_console
|
||||
|
||||
.PHONY: dbt-console
|
||||
.PHONY: dbt-console
|
||||
|
||||
SHELL := /bin/bash
|
||||
|
||||
# set default target
|
||||
DBT_TARGET ?= dev
|
||||
AWS_LAMBDA_ROLE ?= aws_lambda_cosmos_api_dev
|
||||
|
||||
sl-cosmos-api:
|
||||
dbt run-operation create_aws_cosmos_api \
|
||||
--profile cosmos \
|
||||
--target $(DBT_TARGET) \
|
||||
--profiles-dir ~/.dbt/
|
||||
|
||||
udfs:
|
||||
dbt run-operation create_udfs \
|
||||
--vars '{"UPDATE_UDFS_AND_SPS":True}' \
|
||||
--profile cosmos \
|
||||
--target $(DBT_TARGET) \
|
||||
--profiles-dir ~/.dbt/
|
||||
|
||||
complete:
|
||||
dbt run \
|
||||
--vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \
|
||||
-m 1+models/silver/streamline/core/complete \
|
||||
--profile cosmos \
|
||||
--target $(DBT_TARGET) \
|
||||
--profiles-dir ~/.dbt
|
||||
|
||||
streamline: sl-cosmos-api udfs
|
||||
|
||||
blocks_history:
|
||||
dbt run \
|
||||
--vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \
|
||||
-m 1+models/streamline/genesis_backfill/cosmos_hub_1/streamline__blocks_genesis_backfill_ch1.sql \
|
||||
--profile cosmos \
|
||||
--target $(DBT_TARGET) \
|
||||
--profiles-dir ~/.dbt
|
||||
|
||||
tx_history:
|
||||
dbt run \
|
||||
--vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \
|
||||
-m 1+models/streamline/genesis_backfill/cosmos_hub_1/streamline__transactions_genesis_backfill_ch1.sql \
|
||||
--profile cosmos \
|
||||
--target $(DBT_TARGET) \
|
||||
--profiles-dir ~/.dbt
|
||||
|
||||
validators_history:
|
||||
dbt run \
|
||||
--vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \
|
||||
-m 1+models/streamline/genesis_backfill/cosmos_hub_1/streamline__validators_genesis_backfill_ch1.sql \
|
||||
--profile cosmos \
|
||||
--target $(DBT_TARGET) \
|
||||
--profiles-dir ~/.dbt
|
||||
|
||||
blocks_realtime:
|
||||
dbt run \
|
||||
--vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \
|
||||
-m 1+models/streamline/genesis_backfill/tests/streamline__blocks_realtime_test.sql \
|
||||
--profile cosmos \
|
||||
--target dev \
|
||||
--profiles-dir ~/.dbt
|
||||
|
||||
tx_realtime:
|
||||
dbt run \
|
||||
--vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \
|
||||
-m 1+models/streamline/genesis_backfill/tests/streamline__transactions_realtime_test.sql \
|
||||
--profile cosmos \
|
||||
--target dev \
|
||||
--profiles-dir ~/.dbt
|
||||
|
||||
validators_realtime:
|
||||
dbt run \
|
||||
--vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \
|
||||
-m 1+models/streamline/genesis_backfill/tests/streamline__validators_realtime_test.sql \
|
||||
--profile cosmos \
|
||||
--target dev \
|
||||
--profiles-dir ~/.dbt
|
||||
@ -14,7 +14,6 @@
|
||||
{{ create_udf_get_cosmos_validators() }}
|
||||
{{ create_udf_get_cosmos_generic() }}
|
||||
{{ create_udf_get_cosmos_chainhead() }}
|
||||
|
||||
{% endset %}
|
||||
{% do run_query(sql) %}
|
||||
{% endif %}
|
||||
|
||||
@ -1,9 +1,8 @@
|
||||
{% macro create_aws_cosmos_api() %}
|
||||
{% if target.name == "prod" %}
|
||||
{% set sql %}
|
||||
CREATE api integration IF NOT EXISTS aws_cosmos_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/snowflake-api-cosmos' api_allowed_prefixes = (
|
||||
'https://z97ik1b2d0.execute-api.us-east-1.amazonaws.com/dev/',
|
||||
'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/'
|
||||
CREATE api integration IF NOT EXISTS aws_cosmos_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/cosmos-api-prod-rolesnowflakeudfsAF733095-14KYNLUQ3CWV2' api_allowed_prefixes = (
|
||||
'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/'
|
||||
) enabled = TRUE;
|
||||
{% endset %}
|
||||
{% do run_query(sql) %}
|
||||
|
||||
@ -2,10 +2,11 @@
|
||||
CREATE
|
||||
OR REPLACE EXTERNAL FUNCTION streamline.udf_get_cosmos_blocks(
|
||||
json variant
|
||||
) returns text api_integration = aws_cosmos_api AS {% if target.name == "prod" %}
|
||||
'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_blocks'
|
||||
) returns text {% if target.name == "prod" %}
|
||||
api_integration = aws_cosmos_api AS 'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_blocks'
|
||||
{% else %}
|
||||
'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_blocks'
|
||||
api_integration = aws_cosmos_api_dev AS 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_blocks'
|
||||
|
||||
{%- endif %};
|
||||
{% endmacro %}
|
||||
|
||||
@ -13,10 +14,10 @@
|
||||
CREATE
|
||||
OR REPLACE EXTERNAL FUNCTION streamline.udf_get_cosmos_transactions(
|
||||
json variant
|
||||
) returns text api_integration = aws_cosmos_api AS {% if target.name == "prod" %}
|
||||
'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_transactions'
|
||||
) returns text {% if target.name == "prod" %}
|
||||
api_integration = aws_cosmos_api AS 'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_transactions'
|
||||
{% else %}
|
||||
'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_transactions'
|
||||
api_integration = aws_cosmos_api_dev AS 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_transactions'
|
||||
{%- endif %};
|
||||
{% endmacro %}
|
||||
|
||||
@ -24,10 +25,10 @@
|
||||
CREATE
|
||||
OR REPLACE EXTERNAL FUNCTION streamline.udf_get_cosmos_validators(
|
||||
json variant
|
||||
) returns text api_integration = aws_cosmos_api AS {% if target.name == "prod" %}
|
||||
'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_validators'
|
||||
) returns text {% if target.name == "prod" %}
|
||||
api_integration = aws_cosmos_api AS 'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_validators'
|
||||
{% else %}
|
||||
'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_validators'
|
||||
api_integration = aws_cosmos_api_dev AS 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_validators'
|
||||
{%- endif %};
|
||||
{% endmacro %}
|
||||
|
||||
@ -35,17 +36,19 @@
|
||||
CREATE
|
||||
OR REPLACE EXTERNAL FUNCTION streamline.udf_get_cosmos_generic(
|
||||
json variant
|
||||
) returns text api_integration = aws_cosmos_api AS {% if target.name == "prod" %}
|
||||
'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_generic'
|
||||
) returns text{% if target.name == "prod" %}
|
||||
api_integration = aws_cosmos_api AS 'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/bulk_get_cosmos_generic'
|
||||
{% else %}
|
||||
'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_generic'
|
||||
api_integration = aws_cosmos_api_dev AS 'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/bulk_get_cosmos_generic'
|
||||
{%- endif %};
|
||||
{% endmacro %}
|
||||
|
||||
{% macro create_udf_get_cosmos_chainhead() %}
|
||||
CREATE EXTERNAL FUNCTION IF NOT EXISTS streamline.udf_get_cosmos_chainhead() returns variant api_integration = aws_cosmos_api AS {% if target.name == "prod" %}
|
||||
'https://dazi3rled6.execute-api.us-east-1.amazonaws.com/prod/get_cosmos_chainhead'
|
||||
CREATE
|
||||
OR REPLACE EXTERNAL FUNCTION streamline.udf_get_cosmos_chainhead()
|
||||
returns variant {% if target.name == "prod" %}
|
||||
api_integration = aws_cosmos_api AS 'https://bp6s0ib6fk.execute-api.us-east-1.amazonaws.com/prod/get_cosmos_chainhead'
|
||||
{% else %}
|
||||
'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/get_cosmos_chainhead'
|
||||
api_integration = aws_cosmos_api_dev AS'https://qkwbozz9l0.execute-api.us-east-1.amazonaws.com/dev/get_cosmos_chainhead'
|
||||
{%- endif %};
|
||||
{% endmacro %}
|
||||
|
||||
@ -11,6 +11,16 @@ sources:
|
||||
identifier: tx_search
|
||||
- name: validators
|
||||
- name: blockchain
|
||||
- name: blocks_ch1
|
||||
- name: tx_search_ch1
|
||||
- name: validators_ch1
|
||||
- name: blocks_ch2
|
||||
- name: tx_search_ch2
|
||||
- name: validators_ch2
|
||||
- name: blocks_ch3
|
||||
- name: tx_search_ch3
|
||||
- name: validators_ch3
|
||||
|
||||
- name: osmo
|
||||
database: osmosis
|
||||
schema: silver
|
||||
|
||||
@ -0,0 +1,54 @@
|
||||
{{ config (
|
||||
materialized = "incremental",
|
||||
unique_key = "id",
|
||||
cluster_by = "ROUND(block_number, -3)",
|
||||
merge_update_columns = ["id"]
|
||||
) }}
|
||||
|
||||
WITH meta AS (
|
||||
|
||||
SELECT
|
||||
last_modified,
|
||||
file_name
|
||||
FROM
|
||||
TABLE(
|
||||
information_schema.external_table_files(
|
||||
table_name => '{{ source( "bronze_streamline", "blocks_ch1") }}'
|
||||
)
|
||||
) A
|
||||
)
|
||||
|
||||
{% if is_incremental() %},
|
||||
max_date AS (
|
||||
SELECT
|
||||
COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
{{ this }})
|
||||
{% endif %}
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number,
|
||||
last_modified AS _inserted_timestamp
|
||||
FROM
|
||||
{{ source(
|
||||
"bronze_streamline",
|
||||
"blocks_ch1"
|
||||
) }}
|
||||
JOIN meta b
|
||||
ON b.file_name = metadata$filename
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
b.last_modified > (
|
||||
SELECT
|
||||
max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
max_date
|
||||
)
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over (PARTITION BY id
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
@ -0,0 +1,54 @@
|
||||
{{ config (
|
||||
materialized = "incremental",
|
||||
unique_key = "id",
|
||||
cluster_by = "ROUND(block_number, -3)",
|
||||
merge_update_columns = ["id"]
|
||||
) }}
|
||||
|
||||
WITH meta AS (
|
||||
|
||||
SELECT
|
||||
last_modified,
|
||||
file_name
|
||||
FROM
|
||||
TABLE(
|
||||
information_schema.external_table_files(
|
||||
table_name => '{{ source( "bronze_streamline", "tx_search_ch1") }}'
|
||||
)
|
||||
) A
|
||||
)
|
||||
|
||||
{% if is_incremental() %},
|
||||
max_date AS (
|
||||
SELECT
|
||||
COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
{{ this }})
|
||||
{% endif %}
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number,
|
||||
last_modified AS _inserted_timestamp
|
||||
FROM
|
||||
{{ source(
|
||||
"bronze_streamline",
|
||||
"tx_search_ch1"
|
||||
) }}
|
||||
JOIN meta b
|
||||
ON b.file_name = metadata$filename
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
b.last_modified > (
|
||||
SELECT
|
||||
max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
max_date
|
||||
)
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over (PARTITION BY id
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
@ -0,0 +1,54 @@
|
||||
{{ config (
|
||||
materialized = "incremental",
|
||||
unique_key = "id",
|
||||
cluster_by = "ROUND(block_number, -3)",
|
||||
merge_update_columns = ["id"]
|
||||
) }}
|
||||
|
||||
WITH meta AS (
|
||||
|
||||
SELECT
|
||||
last_modified,
|
||||
file_name
|
||||
FROM
|
||||
TABLE(
|
||||
information_schema.external_table_files(
|
||||
table_name => '{{ source( "bronze_streamline", "validators_ch1") }}'
|
||||
)
|
||||
) A
|
||||
)
|
||||
|
||||
{% if is_incremental() %},
|
||||
max_date AS (
|
||||
SELECT
|
||||
COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
{{ this }})
|
||||
{% endif %}
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number,
|
||||
last_modified AS _inserted_timestamp
|
||||
FROM
|
||||
{{ source(
|
||||
"bronze_streamline",
|
||||
"validators_ch1"
|
||||
) }}
|
||||
JOIN meta b
|
||||
ON b.file_name = metadata$filename
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
b.last_modified > (
|
||||
SELECT
|
||||
max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
max_date
|
||||
)
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over (PARTITION BY id
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
@ -0,0 +1,54 @@
|
||||
{{ config (
|
||||
materialized = "incremental",
|
||||
unique_key = "id",
|
||||
cluster_by = "ROUND(block_number, -3)",
|
||||
merge_update_columns = ["id"]
|
||||
) }}
|
||||
|
||||
WITH meta AS (
|
||||
|
||||
SELECT
|
||||
last_modified,
|
||||
file_name
|
||||
FROM
|
||||
TABLE(
|
||||
information_schema.external_table_files(
|
||||
table_name => '{{ source( "bronze_streamline", "blocks_ch2") }}'
|
||||
)
|
||||
) A
|
||||
)
|
||||
|
||||
{% if is_incremental() %},
|
||||
max_date AS (
|
||||
SELECT
|
||||
COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
{{ this }})
|
||||
{% endif %}
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number,
|
||||
last_modified AS _inserted_timestamp
|
||||
FROM
|
||||
{{ source(
|
||||
"bronze_streamline",
|
||||
"blocks_ch2"
|
||||
) }}
|
||||
JOIN meta b
|
||||
ON b.file_name = metadata$filename
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
b.last_modified > (
|
||||
SELECT
|
||||
max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
max_date
|
||||
)
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over (PARTITION BY id
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
@ -0,0 +1,54 @@
|
||||
{{ config (
|
||||
materialized = "incremental",
|
||||
unique_key = "id",
|
||||
cluster_by = "ROUND(block_number, -3)",
|
||||
merge_update_columns = ["id"]
|
||||
) }}
|
||||
|
||||
WITH meta AS (
|
||||
|
||||
SELECT
|
||||
last_modified,
|
||||
file_name
|
||||
FROM
|
||||
TABLE(
|
||||
information_schema.external_table_files(
|
||||
table_name => '{{ source( "bronze_streamline", "tx_search_ch2") }}'
|
||||
)
|
||||
) A
|
||||
)
|
||||
|
||||
{% if is_incremental() %},
|
||||
max_date AS (
|
||||
SELECT
|
||||
COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
{{ this }})
|
||||
{% endif %}
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number,
|
||||
last_modified AS _inserted_timestamp
|
||||
FROM
|
||||
{{ source(
|
||||
"bronze_streamline",
|
||||
"tx_search_ch2"
|
||||
) }}
|
||||
JOIN meta b
|
||||
ON b.file_name = metadata$filename
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
b.last_modified > (
|
||||
SELECT
|
||||
max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
max_date
|
||||
)
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over (PARTITION BY id
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
@ -0,0 +1,54 @@
|
||||
{{ config (
|
||||
materialized = "incremental",
|
||||
unique_key = "id",
|
||||
cluster_by = "ROUND(block_number, -3)",
|
||||
merge_update_columns = ["id"]
|
||||
) }}
|
||||
|
||||
WITH meta AS (
|
||||
|
||||
SELECT
|
||||
last_modified,
|
||||
file_name
|
||||
FROM
|
||||
TABLE(
|
||||
information_schema.external_table_files(
|
||||
table_name => '{{ source( "bronze_streamline", "validators_ch2") }}'
|
||||
)
|
||||
) A
|
||||
)
|
||||
|
||||
{% if is_incremental() %},
|
||||
max_date AS (
|
||||
SELECT
|
||||
COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
{{ this }})
|
||||
{% endif %}
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number,
|
||||
last_modified AS _inserted_timestamp
|
||||
FROM
|
||||
{{ source(
|
||||
"bronze_streamline",
|
||||
"validators_ch2"
|
||||
) }}
|
||||
JOIN meta b
|
||||
ON b.file_name = metadata$filename
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
b.last_modified > (
|
||||
SELECT
|
||||
max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
max_date
|
||||
)
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over (PARTITION BY id
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
@ -0,0 +1,54 @@
|
||||
{{ config (
|
||||
materialized = "incremental",
|
||||
unique_key = "id",
|
||||
cluster_by = "ROUND(block_number, -3)",
|
||||
merge_update_columns = ["id"]
|
||||
) }}
|
||||
|
||||
WITH meta AS (
|
||||
|
||||
SELECT
|
||||
last_modified,
|
||||
file_name
|
||||
FROM
|
||||
TABLE(
|
||||
information_schema.external_table_files(
|
||||
table_name => '{{ source( "bronze_streamline", "blocks_ch3") }}'
|
||||
)
|
||||
) A
|
||||
)
|
||||
|
||||
{% if is_incremental() %},
|
||||
max_date AS (
|
||||
SELECT
|
||||
COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
{{ this }})
|
||||
{% endif %}
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number,
|
||||
last_modified AS _inserted_timestamp
|
||||
FROM
|
||||
{{ source(
|
||||
"bronze_streamline",
|
||||
"blocks_ch3"
|
||||
) }}
|
||||
JOIN meta b
|
||||
ON b.file_name = metadata$filename
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
b.last_modified > (
|
||||
SELECT
|
||||
max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
max_date
|
||||
)
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over (PARTITION BY id
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
@ -0,0 +1,54 @@
|
||||
{{ config (
|
||||
materialized = "incremental",
|
||||
unique_key = "id",
|
||||
cluster_by = "ROUND(block_number, -3)",
|
||||
merge_update_columns = ["id"]
|
||||
) }}
|
||||
|
||||
WITH meta AS (
|
||||
|
||||
SELECT
|
||||
last_modified,
|
||||
file_name
|
||||
FROM
|
||||
TABLE(
|
||||
information_schema.external_table_files(
|
||||
table_name => '{{ source( "bronze_streamline", "tx_search_ch3") }}'
|
||||
)
|
||||
) A
|
||||
)
|
||||
|
||||
{% if is_incremental() %},
|
||||
max_date AS (
|
||||
SELECT
|
||||
COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
{{ this }})
|
||||
{% endif %}
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number,
|
||||
last_modified AS _inserted_timestamp
|
||||
FROM
|
||||
{{ source(
|
||||
"bronze_streamline",
|
||||
"tx_search_ch3"
|
||||
) }}
|
||||
JOIN meta b
|
||||
ON b.file_name = metadata$filename
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
b.last_modified > (
|
||||
SELECT
|
||||
max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
max_date
|
||||
)
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over (PARTITION BY id
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
@ -0,0 +1,54 @@
|
||||
{{ config (
|
||||
materialized = "incremental",
|
||||
unique_key = "id",
|
||||
cluster_by = "ROUND(block_number, -3)",
|
||||
merge_update_columns = ["id"]
|
||||
) }}
|
||||
|
||||
WITH meta AS (
|
||||
|
||||
SELECT
|
||||
last_modified,
|
||||
file_name
|
||||
FROM
|
||||
TABLE(
|
||||
information_schema.external_table_files(
|
||||
table_name => '{{ source( "bronze_streamline", "validators_ch3") }}'
|
||||
)
|
||||
) A
|
||||
)
|
||||
|
||||
{% if is_incremental() %},
|
||||
max_date AS (
|
||||
SELECT
|
||||
COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
{{ this }})
|
||||
{% endif %}
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number,
|
||||
last_modified AS _inserted_timestamp
|
||||
FROM
|
||||
{{ source(
|
||||
"bronze_streamline",
|
||||
"validators_ch3"
|
||||
) }}
|
||||
JOIN meta b
|
||||
ON b.file_name = metadata$filename
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
b.last_modified > (
|
||||
SELECT
|
||||
max_INSERTED_TIMESTAMP
|
||||
FROM
|
||||
max_date
|
||||
)
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over (PARTITION BY id
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
@ -0,0 +1,23 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
post_hook = if_data_call_function(
|
||||
func = "{{this.schema}}.udf_get_cosmos_blocks(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch1/rpc', 'call_type','non_batch','external_table','blocks_ch1','producer_batch_size','1024000','worker_batch_size','32000'))",
|
||||
target = "{{this.schema}}.{{this.identifier}}"
|
||||
)
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__blocks_ch1") }}
|
||||
EXCEPT
|
||||
SELECT
|
||||
id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__complete_blocks_ch1") }}
|
||||
ORDER BY
|
||||
block_number
|
||||
@ -0,0 +1,24 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
post_hook = if_data_call_function(
|
||||
func = "{{this.schema}}.udf_get_cosmos_transactions(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch1/rpc','call_type','non_batch','external_table','tx_search_ch1','producer_batch_size','256000','worker_batch_size','8000'))",
|
||||
target = "{{this.schema}}.{{this.identifier}}"
|
||||
)
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__blocks_ch1") }}
|
||||
EXCEPT
|
||||
SELECT
|
||||
id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__complete_transactions_ch1") }}
|
||||
ORDER BY
|
||||
block_number
|
||||
|
||||
@ -0,0 +1,24 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
post_hook = if_data_call_function(
|
||||
func = "{{this.schema}}.udf_get_cosmos_validators(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch1/rpc','call_type','non_batch','external_table','validators_ch1','producer_batch_size','256000','worker_batch_size','8000'))",
|
||||
target = "{{this.schema}}.{{this.identifier}}"
|
||||
)
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__blocks_ch1") }}
|
||||
EXCEPT
|
||||
SELECT
|
||||
id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__complete_validators_ch1") }}
|
||||
ORDER BY
|
||||
block_number
|
||||
|
||||
@ -0,0 +1,23 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
post_hook = if_data_call_function(
|
||||
func = "{{this.schema}}.udf_get_cosmos_blocks(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch2/rpc', 'call_type','non_batch','external_table','blocks_ch2','producer_batch_size','1024000','worker_batch_size','32000'))",
|
||||
target = "{{this.schema}}.{{this.identifier}}"
|
||||
)
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__blocks_ch2") }}
|
||||
EXCEPT
|
||||
SELECT
|
||||
id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__complete_blocks_ch2") }}
|
||||
ORDER BY
|
||||
block_number
|
||||
@ -0,0 +1,24 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
post_hook = if_data_call_function(
|
||||
func = "{{this.schema}}.udf_get_cosmos_transactions(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch2/rpc','call_type','non_batch','external_table','tx_search_ch2','producer_batch_size','256000','worker_batch_size','8000'))",
|
||||
target = "{{this.schema}}.{{this.identifier}}"
|
||||
)
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__blocks_ch2") }}
|
||||
EXCEPT
|
||||
SELECT
|
||||
id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__complete_transactions_ch2") }}
|
||||
ORDER BY
|
||||
block_number
|
||||
|
||||
@ -0,0 +1,24 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
post_hook = if_data_call_function(
|
||||
func = "{{this.schema}}.udf_get_cosmos_validators(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch2/rpc','call_type','non_batch','external_table','validators_ch2','producer_batch_size','256000','worker_batch_size','8000'))",
|
||||
target = "{{this.schema}}.{{this.identifier}}"
|
||||
)
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__blocks_ch2") }}
|
||||
EXCEPT
|
||||
SELECT
|
||||
id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__complete_validators_ch2") }}
|
||||
ORDER BY
|
||||
block_number
|
||||
|
||||
@ -0,0 +1,23 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
post_hook = if_data_call_function(
|
||||
func = "{{this.schema}}.udf_get_cosmos_blocks(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch3/rpc', 'call_type','non_batch','external_table','blocks_ch3','producer_batch_size','1024000','worker_batch_size','32000'))",
|
||||
target = "{{this.schema}}.{{this.identifier}}"
|
||||
)
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__blocks_ch3") }}
|
||||
EXCEPT
|
||||
SELECT
|
||||
id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__complete_blocks_ch3") }}
|
||||
ORDER BY
|
||||
block_number
|
||||
@ -0,0 +1,24 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
post_hook = if_data_call_function(
|
||||
func = "{{this.schema}}.udf_get_cosmos_transactions(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch3/rpc','call_type','non_batch','external_table','tx_search_ch3','producer_batch_size','256000','worker_batch_size','8000'))",
|
||||
target = "{{this.schema}}.{{this.identifier}}"
|
||||
)
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__blocks_ch3") }}
|
||||
EXCEPT
|
||||
SELECT
|
||||
id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__complete_transactions_ch3") }}
|
||||
ORDER BY
|
||||
block_number
|
||||
|
||||
@ -0,0 +1,24 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
post_hook = if_data_call_function(
|
||||
func = "{{this.schema}}.udf_get_cosmos_validators(object_construct('sql_source', '{{this.identifier}}','sm_node_path','prod/cosmos/allthatnode/mainnet_ch3/rpc','call_type','non_batch','external_table','validators_ch3','producer_batch_size','256000','worker_batch_size','8000'))",
|
||||
target = "{{this.schema}}.{{this.identifier}}"
|
||||
)
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__blocks_ch3") }}
|
||||
EXCEPT
|
||||
SELECT
|
||||
id,
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__complete_validators_ch3") }}
|
||||
ORDER BY
|
||||
block_number
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
tags = ['streamline_view']
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
height as block_number
|
||||
FROM
|
||||
TABLE(streamline.udtf_get_base_table(500042))
|
||||
WHERE
|
||||
height between 0 and 500042 // https://hub.cosmos.network/main/roadmap/#cosmos-hub-summary
|
||||
@ -0,0 +1,11 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
tags = ['streamline_view']
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
height as block_number
|
||||
FROM
|
||||
TABLE(streamline.udtf_get_base_table(2901999))
|
||||
WHERE
|
||||
height between 500043 and 2901999 // https://hub.cosmos.network/main/roadmap/#cosmos-hub-summary
|
||||
@ -0,0 +1,11 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
tags = ['streamline_view']
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
height as block_number
|
||||
FROM
|
||||
TABLE(streamline.udtf_get_base_table(5200790))
|
||||
WHERE
|
||||
height between 2902000 and 5200790 // https://hub.cosmos.network/main/roadmap/#cosmos-hub-summary
|
||||
@ -6,4 +6,4 @@ packages:
|
||||
- git: https://github.com/FlipsideCrypto/fsc-utils.git
|
||||
revision: "v1.3.0"
|
||||
- package: get-select/dbt_snowflake_query_tags
|
||||
version: [">=2.0.0", "<3.0.0"]
|
||||
version: [">=2.0.0", "<3.0.0"]
|
||||
Loading…
Reference in New Issue
Block a user