From c016b658ab07ed511ba799a00e8dfcd84c78bd97 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Sat, 30 Dec 2023 21:46:25 -0500 Subject: [PATCH] bronze & silver ch1,2,3 --- .../workflows/dbt_run_incremental_core.yml | 2 +- .github/workflows/dbt_test.yml | 2 +- .../bronze__streamline_FR_blocks_ch1.sql | 11 ++ .../bronze__streamline_FR_blocks_ch2.sql | 11 ++ .../bronze__streamline_FR_blocks_ch3.sql | 11 ++ ...bronze__streamline_FR_transactions_ch1.sql | 11 ++ ...bronze__streamline_FR_transactions_ch2.sql | 11 ++ ...bronze__streamline_FR_transactions_ch3.sql | 11 ++ .../genesis_backfill/silver__blocks_ch1.sql | 61 ++++++++ .../genesis_backfill/silver__blocks_ch1.yml | 69 +++++++++ .../genesis_backfill/silver__blocks_ch2.sql | 61 ++++++++ .../genesis_backfill/silver__blocks_ch2.yml | 69 +++++++++ .../genesis_backfill/silver__blocks_ch3.sql | 61 ++++++++ .../genesis_backfill/silver__blocks_ch3.yml | 69 +++++++++ .../silver__msg_attributes_ch1.sql | 37 +++++ .../genesis_backfill/silver__msgs_ch1.sql | 140 ++++++++++++++++++ .../genesis_backfill/silver__msgs_ch1.yml | 88 +++++++++++ .../silver__transactions_ch1.sql | 67 +++++++++ .../silver__transactions_ch1.yml | 73 +++++++++ .../silver__transactions_ch2.sql | 68 +++++++++ .../silver__transactions_ch2.yml | 73 +++++++++ .../silver__transactions_ch3.sql | 68 +++++++++ .../silver__transactions_ch3.yml | 73 +++++++++ 23 files changed, 1145 insertions(+), 2 deletions(-) create mode 100644 models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch1.sql create mode 100644 models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch2.sql create mode 100644 models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch3.sql create mode 100644 models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch1.sql create mode 100644 models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch2.sql create mode 100644 models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch3.sql create mode 100644 models/silver/core/genesis_backfill/silver__blocks_ch1.sql create mode 100644 models/silver/core/genesis_backfill/silver__blocks_ch1.yml create mode 100644 models/silver/core/genesis_backfill/silver__blocks_ch2.sql create mode 100644 models/silver/core/genesis_backfill/silver__blocks_ch2.yml create mode 100644 models/silver/core/genesis_backfill/silver__blocks_ch3.sql create mode 100644 models/silver/core/genesis_backfill/silver__blocks_ch3.yml create mode 100644 models/silver/core/genesis_backfill/silver__msg_attributes_ch1.sql create mode 100644 models/silver/core/genesis_backfill/silver__msgs_ch1.sql create mode 100644 models/silver/core/genesis_backfill/silver__msgs_ch1.yml create mode 100644 models/silver/core/genesis_backfill/silver__transactions_ch1.sql create mode 100644 models/silver/core/genesis_backfill/silver__transactions_ch1.yml create mode 100644 models/silver/core/genesis_backfill/silver__transactions_ch2.sql create mode 100644 models/silver/core/genesis_backfill/silver__transactions_ch2.yml create mode 100644 models/silver/core/genesis_backfill/silver__transactions_ch3.sql create mode 100644 models/silver/core/genesis_backfill/silver__transactions_ch3.yml diff --git a/.github/workflows/dbt_run_incremental_core.yml b/.github/workflows/dbt_run_incremental_core.yml index 9573157..cdd5c8b 100644 --- a/.github/workflows/dbt_run_incremental_core.yml +++ b/.github/workflows/dbt_run_incremental_core.yml @@ -28,7 +28,7 @@ jobs: uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main with: dbt_command: | - dbt run -m models/bronze models/silver/core models/gold/core + dbt run -m models/bronze models/silver/core models/gold/core --exclude models/bronze/genesis_backfill models/silver/core/genesis_backfill environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index f53089e..f98dce3 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -27,7 +27,7 @@ jobs: uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main with: dbt_command: | - dbt test -m models/bronze models/silver models/gold + dbt test -m models/bronze models/silver models/gold --exclude models/bronze/genesis_backfill models/silver/core/genesis_backfill environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} secrets: inherit diff --git a/models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch1.sql b/models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch1.sql new file mode 100644 index 0000000..153e068 --- /dev/null +++ b/models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch1.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_FR_query( + model = 'blocks_ch1', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch2.sql b/models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch2.sql new file mode 100644 index 0000000..b801035 --- /dev/null +++ b/models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch2.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_FR_query( + model = 'blocks_ch2', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch3.sql b/models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch3.sql new file mode 100644 index 0000000..ad889f8 --- /dev/null +++ b/models/bronze/genesis_backfill/bronze__streamline_FR_blocks_ch3.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_FR_query( + model = 'blocks_ch3', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch1.sql b/models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch1.sql new file mode 100644 index 0000000..c4154d1 --- /dev/null +++ b/models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch1.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_FR_query( + model = 'tx_search_ch1', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch2.sql b/models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch2.sql new file mode 100644 index 0000000..bb3325c --- /dev/null +++ b/models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch2.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_FR_query( + model = 'tx_search_ch2', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch3.sql b/models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch3.sql new file mode 100644 index 0000000..9cc2dd3 --- /dev/null +++ b/models/bronze/genesis_backfill/bronze__streamline_FR_transactions_ch3.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_FR_query( + model = 'tx_search_ch3', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/silver/core/genesis_backfill/silver__blocks_ch1.sql b/models/silver/core/genesis_backfill/silver__blocks_ch1.sql new file mode 100644 index 0000000..2ba147d --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__blocks_ch1.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'table', + cluster_by = ['block_timestamp::DATE'] +) }} + +SELECT + block_number AS block_id, + COALESCE( + DATA :result :block :header :time :: TIMESTAMP, + DATA :block :header :time :: TIMESTAMP, + DATA :result :block :header :timestamp :: TIMESTAMP, + DATA :block :header :timestamp :: TIMESTAMP + ) AS block_timestamp, + 'cosmos' AS blockchain, + COALESCE( + DATA :result :block :header :chain_id :: STRING, + DATA :block :header :chain_id :: STRING + ) AS chain_id, + COALESCE( + ARRAY_SIZE( + DATA :result :block :data :txs + ) :: NUMBER, + ARRAY_SIZE( + DATA :block :data :txs + ) :: NUMBER + ) AS tx_count, + COALESCE( + DATA :result :block :header :proposer_address :: STRING, + DATA :block :header :proposer_address :: STRING + ) AS proposer_address, + COALESCE( + DATA :result :block :header :validators_hash :: STRING, + DATA :block :header :validators_hash :: STRING + ) AS validator_hash, + COALESCE( + DATA :result :block :header, + DATA :block :header + ) AS header, + concat_ws( + '-', + chain_id, + block_id + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp :: timestamp_ntz AS _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('bronze__streamline_FR_blocks_ch1') }} +WHERE + VALUE :data :error IS NULL + AND DATA :error IS NULL + AND DATA :result :begin_block_events IS NULL qualify ROW_NUMBER() over ( + PARTITION BY chain_id, + block_id + ORDER BY + _inserted_timestamp DESC + ) = 1 diff --git a/models/silver/core/genesis_backfill/silver__blocks_ch1.yml b/models/silver/core/genesis_backfill/silver__blocks_ch1.yml new file mode 100644 index 0000000..489057f --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__blocks_ch1.yml @@ -0,0 +1,69 @@ +version: 2 +models: + - name: silver__blocks_ch1 + description: Records of all blocks that have occurred on Cosmos hub v1, dating back to the genesis block. Block range is from 0 to 500042) + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _UNIQUE_KEY + - sequence_gaps: + column_name: block_ID + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: TX_COUNT + description: "{{ doc('tx_count') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: PROPOSER_ADDRESS + description: "{{ doc('proposer_address') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: VALIDATOR_HASH + description: "{{ doc('validator_hash') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + diff --git a/models/silver/core/genesis_backfill/silver__blocks_ch2.sql b/models/silver/core/genesis_backfill/silver__blocks_ch2.sql new file mode 100644 index 0000000..db9fbbf --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__blocks_ch2.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'table', + cluster_by = ['block_timestamp::DATE'] +) }} + +SELECT + block_number AS block_id, + COALESCE( + DATA :result :block :header :time :: TIMESTAMP, + DATA :block :header :time :: TIMESTAMP, + DATA :result :block :header :timestamp :: TIMESTAMP, + DATA :block :header :timestamp :: TIMESTAMP + ) AS block_timestamp, + 'cosmos' AS blockchain, + COALESCE( + DATA :result :block :header :chain_id :: STRING, + DATA :block :header :chain_id :: STRING + ) AS chain_id, + COALESCE( + ARRAY_SIZE( + DATA :result :block :data :txs + ) :: NUMBER, + ARRAY_SIZE( + DATA :block :data :txs + ) :: NUMBER + ) AS tx_count, + COALESCE( + DATA :result :block :header :proposer_address :: STRING, + DATA :block :header :proposer_address :: STRING + ) AS proposer_address, + COALESCE( + DATA :result :block :header :validators_hash :: STRING, + DATA :block :header :validators_hash :: STRING + ) AS validator_hash, + COALESCE( + DATA :result :block :header, + DATA :block :header + ) AS header, + concat_ws( + '-', + chain_id, + block_id + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp :: timestamp_ntz AS _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('bronze__streamline_FR_blocks_ch2') }} +WHERE + VALUE :data :error IS NULL + AND DATA :error IS NULL + AND DATA :result :begin_block_events IS NULL qualify ROW_NUMBER() over ( + PARTITION BY chain_id, + block_id + ORDER BY + _inserted_timestamp DESC + ) = 1 diff --git a/models/silver/core/genesis_backfill/silver__blocks_ch2.yml b/models/silver/core/genesis_backfill/silver__blocks_ch2.yml new file mode 100644 index 0000000..672d89a --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__blocks_ch2.yml @@ -0,0 +1,69 @@ +version: 2 +models: + - name: silver__blocks_ch2 + description: Block range is from 500,043 to 2,901,999) + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _UNIQUE_KEY + - sequence_gaps: + column_name: block_ID + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: TX_COUNT + description: "{{ doc('tx_count') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: PROPOSER_ADDRESS + description: "{{ doc('proposer_address') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: VALIDATOR_HASH + description: "{{ doc('validator_hash') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + diff --git a/models/silver/core/genesis_backfill/silver__blocks_ch3.sql b/models/silver/core/genesis_backfill/silver__blocks_ch3.sql new file mode 100644 index 0000000..0334e59 --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__blocks_ch3.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'table', + cluster_by = ['block_timestamp::DATE'] +) }} + +SELECT + block_number AS block_id, + COALESCE( + DATA :result :block :header :time :: TIMESTAMP, + DATA :block :header :time :: TIMESTAMP, + DATA :result :block :header :timestamp :: TIMESTAMP, + DATA :block :header :timestamp :: TIMESTAMP + ) AS block_timestamp, + 'cosmos' AS blockchain, + COALESCE( + DATA :result :block :header :chain_id :: STRING, + DATA :block :header :chain_id :: STRING + ) AS chain_id, + COALESCE( + ARRAY_SIZE( + DATA :result :block :data :txs + ) :: NUMBER, + ARRAY_SIZE( + DATA :block :data :txs + ) :: NUMBER + ) AS tx_count, + COALESCE( + DATA :result :block :header :proposer_address :: STRING, + DATA :block :header :proposer_address :: STRING + ) AS proposer_address, + COALESCE( + DATA :result :block :header :validators_hash :: STRING, + DATA :block :header :validators_hash :: STRING + ) AS validator_hash, + COALESCE( + DATA :result :block :header, + DATA :block :header + ) AS header, + concat_ws( + '-', + chain_id, + block_id + ) AS _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['_unique_key'] + ) }} AS blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp :: timestamp_ntz AS _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('bronze__streamline_FR_blocks_ch3') }} +WHERE + VALUE :data :error IS NULL + AND DATA :error IS NULL + AND DATA :result :begin_block_events IS NULL qualify ROW_NUMBER() over ( + PARTITION BY chain_id, + block_id + ORDER BY + _inserted_timestamp DESC + ) = 1 diff --git a/models/silver/core/genesis_backfill/silver__blocks_ch3.yml b/models/silver/core/genesis_backfill/silver__blocks_ch3.yml new file mode 100644 index 0000000..8aad054 --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__blocks_ch3.yml @@ -0,0 +1,69 @@ +version: 2 +models: + - name: silver__blocks_ch3 + description: Block range is from 2,902,000 and 5,200,790) + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _UNIQUE_KEY + - sequence_gaps: + column_name: block_ID + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: TX_COUNT + description: "{{ doc('tx_count') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: PROPOSER_ADDRESS + description: "{{ doc('proposer_address') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: VALIDATOR_HASH + description: "{{ doc('validator_hash') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + diff --git a/models/silver/core/genesis_backfill/silver__msg_attributes_ch1.sql b/models/silver/core/genesis_backfill/silver__msg_attributes_ch1.sql new file mode 100644 index 0000000..0765349 --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__msg_attributes_ch1.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = 'table', + cluster_by = ['block_timestamp::DATE'], + enabled = false +) }} + +SELECT + block_id, + block_timestamp, + tx_id, + tx_succeeded, + msg_group, + msg_sub_group, + msg_index, + msg_type, + 0 AS attribute_index, + TRY_BASE64_DECODE_STRING( + msg :key :: STRING + ) AS attribute_key, + TRY_BASE64_DECODE_STRING( + msg :value :: STRING + ) AS attribute_value, + concat_ws( + '-', + tx_id, + msg_index, + attribute_index + ) AS unique_key, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','msg_index','attribute_index'] + ) }} AS msg_attributes_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('silver__msgs_ch1') }} A diff --git a/models/silver/core/genesis_backfill/silver__msgs_ch1.sql b/models/silver/core/genesis_backfill/silver__msgs_ch1.sql new file mode 100644 index 0000000..bf63a11 --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__msgs_ch1.sql @@ -0,0 +1,140 @@ +{{ config( + materialized = 'table', + cluster_by = ['block_timestamp::DATE'], + enabled = false +) }} + +WITH base_msgs AS ( + + SELECT + t.block_id, + t.block_timestamp, + t.tx_id, + t.gas_used, + t.gas_wanted, + t.tx_succeeded, + f.value AS msg, + f.index :: INT AS msg_index, + msg :type :: STRING AS msg_type, + IFF( + TRY_BASE64_DECODE_STRING( + msg :attributes [0] :key :: STRING + ) = 'action', + TRUE, + FALSE + ) AS is_action, + NULLIF( + (conditional_true_event(is_action) over (PARTITION BY tx_id + ORDER BY + msg_index ASC) -1), + -1 + ) AS msg_group, + IFF( + TRY_BASE64_DECODE_STRING( + msg :attributes [0] :key :: STRING + ) = 'module', + TRUE, + FALSE + ) AS is_module, + TRY_BASE64_DECODE_STRING( + msg :attributes [0] :key :: STRING + ) AS attribute_key, + TRY_BASE64_DECODE_STRING( + msg :attributes [0] :value :: STRING + ) AS attribute_value, + t._inserted_timestamp + FROM + {{ ref( + 'silver__transactions_ch1' + ) }} + t, + LATERAL FLATTEN( + input => msgs + ) f +), +exec_actions AS ( + SELECT + DISTINCT tx_id, + msg_group + FROM + base_msgs + WHERE + msg_type = 'message' + AND attribute_key = 'action' + AND LOWER(attribute_value) LIKE '%exec%' +), +GROUPING AS ( + SELECT + base_msgs.tx_id, + base_msgs.msg_index, + RANK() over( + PARTITION BY base_msgs.tx_id, + base_msgs.msg_group + ORDER BY + base_msgs.msg_index + ) -1 AS msg_sub_group + FROM + base_msgs + INNER JOIN exec_actions e + ON base_msgs.tx_id = e.tx_id + AND base_msgs.msg_group = e.msg_group + WHERE + base_msgs.is_module = 'TRUE' + AND base_msgs.msg_type = 'message' +), +msgs AS ( + SELECT + block_id, + block_timestamp, + A.tx_id, + tx_succeeded, + msg_group, + CASE + WHEN msg_group IS NULL THEN NULL + ELSE COALESCE( + LAST_VALUE( + b.msg_sub_group ignore nulls + ) over( + PARTITION BY A.tx_id, + msg_group + ORDER BY + A.msg_index DESC rows unbounded preceding + ), + 0 + ) + END AS msg_sub_group, + A.msg_index, + msg_type, + msg, + concat_ws( + '-', + A.tx_id, + A.msg_index + ) AS unique_key, + _inserted_timestamp + FROM + base_msgs A + LEFT JOIN GROUPING b + ON A.tx_id = b.tx_id + AND A.msg_index = b.msg_index +) +SELECT + block_id, + block_timestamp, + tx_id, + tx_succeeded, + msg_group, + msg_sub_group, + msg_index, + msg_type, + msg :: OBJECT AS msg, + unique_key, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','msg_index'] + ) }} AS msgs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + msgs diff --git a/models/silver/core/genesis_backfill/silver__msgs_ch1.yml b/models/silver/core/genesis_backfill/silver__msgs_ch1.yml new file mode 100644 index 0000000..68be01f --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__msgs_ch1.yml @@ -0,0 +1,88 @@ +version: 2 +models: + - name: silver__msgs_ch1 + description: Records of all messages associated to transactions that have occurred on Cosmos, dating back to the genesis block. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - MSG_INDEX + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: MSG_GROUP + description: "{{ doc('msg_group') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSG_SUB_GROUP + description: "{{ doc('msg_sub_group') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSG_INDEX + description: "{{ doc('msg_index') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSG_TYPE + description: "{{ doc('msg_type') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: MSG + description: "{{ doc('msg') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - OBJECT + - VARIENT + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + \ No newline at end of file diff --git a/models/silver/core/genesis_backfill/silver__transactions_ch1.sql b/models/silver/core/genesis_backfill/silver__transactions_ch1.sql new file mode 100644 index 0000000..7611f59 --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__transactions_ch1.sql @@ -0,0 +1,67 @@ +{{ config( + materialized = 'table', + cluster_by = 'block_timestamp::DATE' +) }} + +WITH base_transactions AS ( + + SELECT + block_number AS block_id, + t.value :hash :: STRING AS tx_id, + t.value :tx_result :codespace AS codespace, + COALESCE( + t.value :tx_result :gas_used, + t.value :tx_result :gasUsed + ) :: NUMBER AS gas_used, + COALESCE( + t.value :tx_result :gas_wanted, + t.value :tx_result :gasWanted + ) :: NUMBER AS gas_wanted, + CASE + WHEN t.value :tx_result :code :: NUMBER = 0 THEN TRUE + ELSE FALSE + END AS tx_succeeded, + t.value :tx_result :code :: NUMBER AS tx_code, + COALESCE( + t.value :tx_result :events, + t.value :tx_result :tags + ) AS msgs, + t.value :tx_result :log :: STRING AS tx_log, + t.value AS full_tx, + _inserted_timestamp :: timestamp_ntz AS _inserted_timestamp + FROM + {{ ref('bronze__streamline_FR_transactions_ch1') }}, + TABLE(FLATTEN(DATA :result :txs)) t +) +SELECT + t.block_id, + b.block_timestamp, + tx_id, + codespace, + gas_used, + gas_wanted, + tx_succeeded, + tx_code, + msgs, + tx_log, + concat_ws( + '-', + t.block_id, + tx_id + ) AS unique_key, + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} AS transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + t._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + base_transactions t + JOIN {{ ref('silver__blocks_ch1') }} + b + ON t.block_id = b.block_id qualify ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + t._inserted_timestamp DESC + ) = 1 diff --git a/models/silver/core/genesis_backfill/silver__transactions_ch1.yml b/models/silver/core/genesis_backfill/silver__transactions_ch1.yml new file mode 100644 index 0000000..c89fbb1 --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__transactions_ch1.yml @@ -0,0 +1,73 @@ +version: 2 +models: + - name: silver__transactions_ch1 + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - unique + - name: CODESPACE + description: "{{ doc('codespace') }}" + - name: GAS_USED + description: "{{ doc('gas_used') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_WANTED + description: "{{ doc('gas_wanted') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: TX_CODE + description: "{{ doc('tx_code') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSGS + description: "{{ doc('msgs') }}" + tests: + - not_null: + where: tx_code is null + - name: TX_LOG + description: "{{ doc('tx_log') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null diff --git a/models/silver/core/genesis_backfill/silver__transactions_ch2.sql b/models/silver/core/genesis_backfill/silver__transactions_ch2.sql new file mode 100644 index 0000000..2a52926 --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__transactions_ch2.sql @@ -0,0 +1,68 @@ +{{ config( + materialized = 'table', + cluster_by = 'block_timestamp::DATE' +) }} + +WITH base_transactions AS ( + + SELECT + block_number AS block_id, + t.value :hash :: STRING AS tx_id, + t.value :tx_result :codespace AS codespace, + COALESCE( + t.value :tx_result :gas_used, + t.value :tx_result :gasUsed + ) :: NUMBER AS gas_used, + COALESCE( + t.value :tx_result :gas_wanted, + t.value :tx_result :gasWanted + ) :: NUMBER AS gas_wanted, + CASE + WHEN t.value :tx_result :code :: NUMBER = 0 THEN TRUE + ELSE FALSE + END AS tx_succeeded, + t.value :tx_result :code :: NUMBER AS tx_code, + COALESCE( + t.value :tx_result :events, + t.value :tx_result :tags + ) AS msgs, + t.value :tx_result :log :: STRING AS tx_log, + t.value AS full_tx, + _inserted_timestamp :: timestamp_ntz AS _inserted_timestamp + FROM + {{ ref('bronze__streamline_FR_transactions_ch2') }}, + TABLE(FLATTEN(DATA :result :txs)) t +) +SELECT + t.block_id, + b.block_timestamp, + tx_id, + codespace, + gas_used, + gas_wanted, + tx_succeeded, + tx_code, + msgs, + tx_log, + full_tx, + concat_ws( + '-', + t.block_id, + tx_id + ) AS unique_key, + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} AS transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + t._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + base_transactions t + JOIN {{ ref('silver__blocks_ch2') }} + b + ON t.block_id = b.block_id qualify ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + t._inserted_timestamp DESC + ) = 1 diff --git a/models/silver/core/genesis_backfill/silver__transactions_ch2.yml b/models/silver/core/genesis_backfill/silver__transactions_ch2.yml new file mode 100644 index 0000000..f1fd2c1 --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__transactions_ch2.yml @@ -0,0 +1,73 @@ +version: 2 +models: + - name: silver__transactions_ch2 + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - unique + - name: CODESPACE + description: "{{ doc('codespace') }}" + - name: GAS_USED + description: "{{ doc('gas_used') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_WANTED + description: "{{ doc('gas_wanted') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: TX_CODE + description: "{{ doc('tx_code') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSGS + description: "{{ doc('msgs') }}" + tests: + - not_null: + where: tx_code is null + - name: TX_LOG + description: "{{ doc('tx_log') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null diff --git a/models/silver/core/genesis_backfill/silver__transactions_ch3.sql b/models/silver/core/genesis_backfill/silver__transactions_ch3.sql new file mode 100644 index 0000000..9311b48 --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__transactions_ch3.sql @@ -0,0 +1,68 @@ +{{ config( + materialized = 'table', + cluster_by = 'block_timestamp::DATE' +) }} + +WITH base_transactions AS ( + + SELECT + block_number AS block_id, + t.value :hash :: STRING AS tx_id, + t.value :tx_result :codespace AS codespace, + COALESCE( + t.value :tx_result :gas_used, + t.value :tx_result :gasUsed + ) :: NUMBER AS gas_used, + COALESCE( + t.value :tx_result :gas_wanted, + t.value :tx_result :gasWanted + ) :: NUMBER AS gas_wanted, + CASE + WHEN t.value :tx_result :code :: NUMBER = 0 THEN TRUE + ELSE FALSE + END AS tx_succeeded, + t.value :tx_result :code :: NUMBER AS tx_code, + COALESCE( + t.value :tx_result :events, + t.value :tx_result :tags + ) AS msgs, + t.value :tx_result :log :: STRING AS tx_log, + t.value AS full_tx, + _inserted_timestamp :: timestamp_ntz AS _inserted_timestamp + FROM + {{ ref('bronze__streamline_FR_transactions_ch3') }}, + TABLE(FLATTEN(DATA :result :txs)) t +) +SELECT + t.block_id, + b.block_timestamp, + tx_id, + codespace, + gas_used, + gas_wanted, + tx_succeeded, + tx_code, + msgs, + tx_log, + full_tx, + concat_ws( + '-', + t.block_id, + tx_id + ) AS unique_key, + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} AS transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + t._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + base_transactions t + JOIN {{ ref('silver__blocks_ch3') }} + b + ON t.block_id = b.block_id qualify ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + t._inserted_timestamp DESC + ) = 1 diff --git a/models/silver/core/genesis_backfill/silver__transactions_ch3.yml b/models/silver/core/genesis_backfill/silver__transactions_ch3.yml new file mode 100644 index 0000000..a5406e4 --- /dev/null +++ b/models/silver/core/genesis_backfill/silver__transactions_ch3.yml @@ -0,0 +1,73 @@ +version: 2 +models: + - name: silver__transactions_ch3 + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - unique + - name: CODESPACE + description: "{{ doc('codespace') }}" + - name: GAS_USED + description: "{{ doc('gas_used') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_WANTED + description: "{{ doc('gas_wanted') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: TX_CODE + description: "{{ doc('tx_code') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSGS + description: "{{ doc('msgs') }}" + tests: + - not_null: + where: tx_code is null + - name: TX_LOG + description: "{{ doc('tx_log') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null