diff --git a/models/descriptions/value.md b/models/descriptions/raw_metadata.md similarity index 64% rename from models/descriptions/value.md rename to models/descriptions/raw_metadata.md index f676676..539e992 100644 --- a/models/descriptions/value.md +++ b/models/descriptions/raw_metadata.md @@ -1,4 +1,4 @@ -{% docs value %} +{% docs raw_metadata %} Additional details about the validator or token in json format. diff --git a/models/silver/silver__asset_metadata.sql b/models/silver/silver__asset_metadata.sql index d46c8f2..a0afc89 100644 --- a/models/silver/silver__asset_metadata.sql +++ b/models/silver/silver__asset_metadata.sql @@ -8,7 +8,7 @@ WITH base AS ( base AS address, NAME AS label, symbol AS project_name, - denom_units AS VALUE + denom_units AS raw_metadata FROM {{ source( 'osmosis_external', @@ -28,6 +28,6 @@ SELECT 'token_contract' AS label_subtype, label, project_name, - VALUE + raw_metadata FROM base diff --git a/models/silver/silver__asset_metadata.yml b/models/silver/silver__asset_metadata.yml index 16dc3c3..899eba0 100644 --- a/models/silver/silver__asset_metadata.yml +++ b/models/silver/silver__asset_metadata.yml @@ -40,7 +40,7 @@ models: description: "{{ doc('project_name') }}" tests: - not_null - - name: VALUE - description: "{{ doc('value') }}" + - name: RAW_METADATA + description: "{{ doc('raw_metadata') }}" tests: - not_null \ No newline at end of file diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql index 882440e..75a35c1 100644 --- a/models/silver/silver__blocks.sql +++ b/models/silver/silver__blocks.sql @@ -1,24 +1,26 @@ {{ config( materialized = 'incremental', - unique_key = "block_id", + unique_key = "CONCAT_WS('-', chain_id, block_id)", incremental_strategy = 'delete+insert', - cluster_by = ['ingested_at::DATE'], + cluster_by = ['_ingested_at::DATE'], ) }} -SELECT - block_id, - block_timestamp, - chain_id, - tx_count, - header:proposer_address :: STRING AS proposer_address, - header:validators_hash :: STRING AS validator_hash, - ingested_at -FROM {{ ref('bronze__blocks') }} +SELECT + block_id, + block_timestamp, + chain_id, + tx_count, + header :proposer_address :: STRING AS proposer_address, + header :validators_hash :: STRING AS validator_hash, + ingested_at AS _ingested_at +FROM + {{ ref('bronze__blocks') }} {% if is_incremental() %} - WHERE ingested_at :: DATE >= getdate() - INTERVAL '2 days' +WHERE + ingested_at :: DATE >= CURRENT_DATE -2 {% endif %} -qualify(ROW_NUMBER() over(PARTITION BY block_id +qualify(ROW_NUMBER() over(PARTITION BY chain_id, block_id ORDER BY - ingested_at DESC)) = 1 \ No newline at end of file + ingested_at DESC)) = 1 diff --git a/models/silver/silver__blocks.yml b/models/silver/silver__blocks.yml index 6b3108f..a1273ba 100644 --- a/models/silver/silver__blocks.yml +++ b/models/silver/silver__blocks.yml @@ -5,6 +5,7 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: + - CHAIN_ID - BLOCK_ID columns: - name: BLOCK_ID @@ -34,6 +35,10 @@ models: description: "{{ doc('validator_hash') }}" tests: - not_null - - name: INGESTED_AT + - name: _INGESTED_AT + description: "{{ doc('ingested_at') }}" tests: - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ diff --git a/models/silver/silver__msg_attributes.sql b/models/silver/silver__msg_attributes.sql index eb63fe6..a6b41f5 100644 --- a/models/silver/silver__msg_attributes.sql +++ b/models/silver/silver__msg_attributes.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = "CONCAT_WS('-', chain_id, block_id, tx_id, msg_index, attribute_index)", incremental_strategy = 'delete+insert', - cluster_by = ['ingested_at::DATE'], + cluster_by = ['_ingested_at::DATE'], ) }} SELECT @@ -20,7 +20,7 @@ SELECT TRY_BASE64_DECODE_STRING( b.value :value :: STRING ) AS attribute_value, - ingested_at + _ingested_at FROM {{ ref('silver__msgs') }} A, LATERAL FLATTEN( @@ -30,5 +30,5 @@ FROM {% if is_incremental() %} WHERE - ingested_at :: DATE >= CURRENT_DATE -2 + _ingested_at :: DATE >= CURRENT_DATE -2 {% endif %} diff --git a/models/silver/silver__msg_attributes.yml b/models/silver/silver__msg_attributes.yml index 61c4c52..c661ca2 100644 --- a/models/silver/silver__msg_attributes.yml +++ b/models/silver/silver__msg_attributes.yml @@ -49,7 +49,7 @@ models: description: "{{ doc('msg_type') }}" tests: - not_null - - name: INGESTED_AT + - name: _INGESTED_AT description: "{{ doc('ingested_at') }}" tests: - not_null diff --git a/models/silver/silver__msgs.sql b/models/silver/silver__msgs.sql index 77daa09..312511b 100644 --- a/models/silver/silver__msgs.sql +++ b/models/silver/silver__msgs.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = "CONCAT_WS('-', chain_id, block_id, tx_id, msg_index)", incremental_strategy = 'delete+insert', - cluster_by = ['ingested_at::DATE'], + cluster_by = ['_ingested_at::DATE'], ) }} SELECT @@ -15,12 +15,12 @@ SELECT INDEX AS msg_index, VALUE :type :: STRING AS msg_type, VALUE AS msg, - ingested_at + _ingested_at FROM {{ ref('silver__transactions') }} A, LATERAL FLATTEN(input => A.msgs) {% if is_incremental() %} WHERE - ingested_at :: DATE >= getdate() - INTERVAL '2 days' + _ingested_at :: DATE >= CURRENT_DATE - 2 {% endif %} diff --git a/models/silver/silver__msgs.yml b/models/silver/silver__msgs.yml index 5d07dc6..4ded1de 100644 --- a/models/silver/silver__msgs.yml +++ b/models/silver/silver__msgs.yml @@ -9,7 +9,6 @@ models: - BLOCK_ID - TX_ID - MSG_INDEX - - ATTRIBUTE_INDEX columns: - name: BLOCK_ID description: "{{ doc('block_id') }}" @@ -53,9 +52,7 @@ models: - not_null - name: MSG description: "The underlying json from the message or event within the transactions" - tests: - - not_null - - name: INGESTED_AT + - name: _INGESTED_AT description: "{{ doc('ingested_at') }}" tests: - not_null diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql index 94251fd..4489746 100644 --- a/models/silver/silver__transactions.sql +++ b/models/silver/silver__transactions.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = "CONCAT_WS('-', chain_id, block_id, tx_id)", incremental_strategy = 'delete+insert', - cluster_by = ['ingested_at::DATE'], + cluster_by = ['_ingested_at::DATE'], ) }} SELECT @@ -18,18 +18,17 @@ SELECT WHEN tx :tx_result :code :: INT = 0 THEN 'SUCCEEDED' ELSE 'FAILED' END AS tx_status, - tx_block_index, tx :tx_result :code :: INT tx_code, tx :tx_result :events AS msgs, - ingested_at + ingested_at AS _ingested_at FROM {{ ref('bronze__transactions') }} {% if is_incremental() %} WHERE - ingested_at :: DATE >= getdate() - INTERVAL '2 days' + ingested_at :: DATE >= CURRENT_DATE - 2 {% endif %} -qualify(ROW_NUMBER() over(PARTITION BY tx_id +qualify(ROW_NUMBER() over(PARTITION BY chain_id, block_id, tx_id ORDER BY ingested_at DESC)) = 1 diff --git a/models/silver/silver__transactions.yml b/models/silver/silver__transactions.yml index 0dada8d..dc5f50e 100644 --- a/models/silver/silver__transactions.yml +++ b/models/silver/silver__transactions.yml @@ -57,14 +57,6 @@ models: description: "{{ doc('tx_status') }}" tests: - not_null - - name: TX_BLOCK_INDEX - description: "{{ doc('tx_block_index') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - name: TX_CODE description: "{{ doc('tx_code') }}" tests: @@ -77,7 +69,7 @@ models: description: "The underlying json from the messages or events within the transactions" tests: - not_null - - name: INGESTED_AT + - name: _INGESTED_AT description: "{{ doc('ingested_at') }}" tests: - not_null diff --git a/models/silver/silver__validator_metadata.sql b/models/silver/silver__validator_metadata.sql index f737e88..7d98fd9 100644 --- a/models/silver/silver__validator_metadata.sql +++ b/models/silver/silver__validator_metadata.sql @@ -2,18 +2,17 @@ materialized = 'table' ) }} -SELECT - operator_address AS address, - 'osmosis' AS blockchain, - 'flipside' AS creator, - 'operator' AS label_type, - 'validator' AS label_subtype, - moniker AS label, - identity AS project_name, - value -FROM - {{ source( - 'osmosis_external', - 'validator_metadata_api' - ) }} - \ No newline at end of file +SELECT + operator_address AS address, + 'osmosis' AS blockchain, + 'flipside' AS creator, + 'operator' AS label_type, + 'validator' AS label_subtype, + moniker AS label, + identity AS project_name, + VALUE AS raw_metadata +FROM + {{ source( + 'osmosis_external', + 'validator_metadata_api' + ) }} diff --git a/models/silver/silver__validator_metadata.yml b/models/silver/silver__validator_metadata.yml index 73ab7ec..6e3a1ee 100644 --- a/models/silver/silver__validator_metadata.yml +++ b/models/silver/silver__validator_metadata.yml @@ -44,7 +44,7 @@ models: - not_null tests: - not_null - - name: VALUE - description: "{{ doc('value') }}" + - name: RAW_METADATA + description: "{{ doc('raw_metadata') }}" tests: - not_null \ No newline at end of file