From 72747fdd6dc9732e112b2f3da01ddb33f1eeb25b Mon Sep 17 00:00:00 2001 From: Jessica Huhnke <91915469+jhuhnke@users.noreply.github.com> Date: Tue, 10 May 2022 14:53:35 -0500 Subject: [PATCH] An 1113 core views (#12) * core views v1 - transactions in the works * transactions - have uncle block issue * updated transactions model and ymls * updates to docs to reflect new column names * update to joins in tx (now correct number of txs) and test * added clustering by block_timestamp::date * changes --- models/core/core__dim_labels.sql | 29 ++++++++++ models/core/core__dim_labels.yml | 37 +++++++++++++ models/core/core__fact_blocks.sql | 13 +++++ models/core/core__fact_blocks.yml | 29 ++++++++++ models/core/core__fact_msg_attributes.sql | 17 ++++++ models/core/core__fact_msg_attributes.yml | 42 +++++++++++++++ models/core/core__fact_msgs.sql | 16 ++++++ models/core/core__fact_msgs.yml | 42 +++++++++++++++ models/core/core__fact_transactions.sql | 64 +++++++++++++++++++++++ models/core/core__fact_transactions.yml | 62 ++++++++++++++++++++++ models/descriptions/fee.md | 5 ++ models/descriptions/tx_from.md | 5 ++ 12 files changed, 361 insertions(+) create mode 100644 models/core/core__dim_labels.sql create mode 100644 models/core/core__dim_labels.yml create mode 100644 models/core/core__fact_blocks.sql create mode 100644 models/core/core__fact_blocks.yml create mode 100644 models/core/core__fact_msg_attributes.sql create mode 100644 models/core/core__fact_msg_attributes.yml create mode 100644 models/core/core__fact_msgs.sql create mode 100644 models/core/core__fact_msgs.yml create mode 100644 models/core/core__fact_transactions.sql create mode 100644 models/core/core__fact_transactions.yml create mode 100644 models/descriptions/fee.md create mode 100644 models/descriptions/tx_from.md diff --git a/models/core/core__dim_labels.sql b/models/core/core__dim_labels.sql new file mode 100644 index 0000000..021bb6e --- /dev/null +++ b/models/core/core__dim_labels.sql @@ -0,0 +1,29 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + blockchain, + address, + creator, + label_type, + label_subtype, + label, + project_name, + raw_metadata +FROM + {{ ref('silver__asset_metadata') }} + +UNION ALL + +SELECT + blockchain, + address, + creator, + label_type, + label_subtype, + label, + project_name, + raw_metadata +FROM + {{ ref('silver__validator_metadata') }} diff --git a/models/core/core__dim_labels.yml b/models/core/core__dim_labels.yml new file mode 100644 index 0000000..711f2c3 --- /dev/null +++ b/models/core/core__dim_labels.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: core__dim_labels + description: Contains token and validator labels for Osmosis. + columns: + - name: ADDRESS + description: "{{ doc('address') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CREATOR + description: "{{ doc('creator') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LABEL_TYPE + description: "{{ doc('label_type') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LABEL_SUBTYPE + description: "{{ doc('label_subtype') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LABEL + description: "{{ doc('label') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: PROJECT_NAME + description: "{{ doc('project_name') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: RAW_METADATA + description: "{{ doc('raw_metadata') }}" + tests: + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/core/core__fact_blocks.sql b/models/core/core__fact_blocks.sql new file mode 100644 index 0000000..9a6fc9c --- /dev/null +++ b/models/core/core__fact_blocks.sql @@ -0,0 +1,13 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_id, + block_timestamp, + chain_id, + tx_count, + proposer_address, + validator_hash +FROM + {{ ref('silver__blocks') }} \ No newline at end of file diff --git a/models/core/core__fact_blocks.yml b/models/core/core__fact_blocks.yml new file mode 100644 index 0000000..5b35092 --- /dev/null +++ b/models/core/core__fact_blocks.yml @@ -0,0 +1,29 @@ +version: 2 +models: + - name: core__fact_blocks + description: Records of all blocks that have occurred on Osmosis, dating back to the genesis block. + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_COUNT + description: "{{ doc('tx_count') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: PROPOSER_ADDRESS + description: "{{ doc('proposer_address') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: VALIDATOR_HASH + description: "{{ doc('validator_hash') }}" + tests: + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/core/core__fact_msg_attributes.sql b/models/core/core__fact_msg_attributes.sql new file mode 100644 index 0000000..87eb072 --- /dev/null +++ b/models/core/core__fact_msg_attributes.sql @@ -0,0 +1,17 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_id, + block_timestamp, + blockchain, + chain_id, + tx_id, + msg_index, + msg_type, + attribute_index, + attribute_key, + attribute_value +FROM + {{ ref('silver__msg_attributes') }} \ No newline at end of file diff --git a/models/core/core__fact_msg_attributes.yml b/models/core/core__fact_msg_attributes.yml new file mode 100644 index 0000000..344f1f9 --- /dev/null +++ b/models/core/core__fact_msg_attributes.yml @@ -0,0 +1,42 @@ +version: 2 +models: + - name: core__fact_msg_attributes + description: Records of all message attributes associated to messages that have occurred on Osmosis, dating back to the genesis block. + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_INDEX + description: "{{ doc('msg_index') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_TYPE + description: "{{ doc('msg_type') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: ATTRIBUTE_KEY + description: "They key from the key-value pair from the message attribute" + tests: + - dbt_expectations.expect_column_to_exist + - name: ATTRIBUTE_VALUE + description: "They value from the key-value pair from the message attribute" + tests: + - dbt_expectations.expect_column_to_exist + diff --git a/models/core/core__fact_msgs.sql b/models/core/core__fact_msgs.sql new file mode 100644 index 0000000..12ecc05 --- /dev/null +++ b/models/core/core__fact_msgs.sql @@ -0,0 +1,16 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_id, + block_timestamp, + blockchain, + chain_id, + tx_id, + tx_status, + msg_index, + msg_type, + msg +FROM + {{ ref('silver__msgs') }} \ No newline at end of file diff --git a/models/core/core__fact_msgs.yml b/models/core/core__fact_msgs.yml new file mode 100644 index 0000000..580a1c9 --- /dev/null +++ b/models/core/core__fact_msgs.yml @@ -0,0 +1,42 @@ +version: 2 +models: + - name: core__fact_msgs + description: Records of all message attributes associated to messages that have occurred on Osmosis, dating back to the genesis block. + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_STATUS + description: "{{ doc('tx_status') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_INDEX + description: "{{ doc('msg_index') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_TYPE + description: "{{ doc('msg_type') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG + description: "A block of json that contains the message attributes in base64 encoding." + tests: + - dbt_expectations.expect_column_to_exist + diff --git a/models/core/core__fact_transactions.sql b/models/core/core__fact_transactions.sql new file mode 100644 index 0000000..d864c60 --- /dev/null +++ b/models/core/core__fact_transactions.sql @@ -0,0 +1,64 @@ +{{ config( + materialized = 'incremental', + unique_key = "tx_id", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], +) }} + +WITH fee AS ( + SELECT + tx_id, + attribute_value AS fee + + FROM {{ ref('silver__msg_attributes') }} + WHERE attribute_key = 'fee' + + {% if is_incremental() %} + AND + _ingested_at :: DATE >= CURRENT_DATE -2 + {% endif %} + +), + +spender AS ( + SELECT + tx_id, + split_part(attribute_value, '/', 0) AS tx_from + + FROM {{ ref('silver__msg_attributes') }} + WHERE attribute_key = 'acc_seq' + + {% if is_incremental() %} + AND + _ingested_at :: DATE >= CURRENT_DATE -2 + {% endif %} +) + +SELECT + t.block_id, + t.block_timestamp, + t.blockchain, + t.chain_id, + t.tx_id, + s.tx_from, + tx_status, + codespace, + COALESCE( fee, + '0uosmo') AS fee, + gas_used, + gas_wanted, + tx_code, + msgs +FROM {{ ref('silver__transactions') }} t + +LEFT OUTER JOIN fee f +ON t.tx_id = f.tx_id + +LEFT OUTER JOIN spender s +ON t.tx_id = s.tx_id + +{% if is_incremental() %} + WHERE + _ingested_at :: DATE >= CURRENT_DATE -2 +{% endif %} + diff --git a/models/core/core__fact_transactions.yml b/models/core/core__fact_transactions.yml new file mode 100644 index 0000000..2cea528 --- /dev/null +++ b/models/core/core__fact_transactions.yml @@ -0,0 +1,62 @@ +version: 2 +models: + - name: core__fact_transactions + description: Records of all transactions that have occurred on Osmosis, dating back to the genesis block. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_FROM + description: "{{ doc('tx_from') }}" + tests: + - not_null: + where: TX_STATUS = 'SUCCEEDED' + - name: TX_STATUS + description: "{{ doc('tx_status') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CODESPACE + description: "{{ doc('codespace') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: FEE + description: "{{ doc('fee') }}" + tests: + - not_null + - name: GAS_USED + description: "{{ doc('gas_used') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: GAS_WANTED + description: "{{ doc('gas_wanted') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_CODE + description: "{{ doc('tx_code') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSGS + description: "The underlying json from the messages or events within the transactions" + tests: + - dbt_expectations.expect_column_to_exist diff --git a/models/descriptions/fee.md b/models/descriptions/fee.md new file mode 100644 index 0000000..8247bd4 --- /dev/null +++ b/models/descriptions/fee.md @@ -0,0 +1,5 @@ +{% docs fee %} + +The fee is paid by the initiator of the transaction. Fee = gas * gas price and is given in micro-OSMO. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_from.md b/models/descriptions/tx_from.md new file mode 100644 index 0000000..d9fd624 --- /dev/null +++ b/models/descriptions/tx_from.md @@ -0,0 +1,5 @@ +{% docs tx_from %} + +The wallet address of the individual who initiated the transaction + +{% enddocs %} \ No newline at end of file