Gold views (#16)

* gold and docs

* gold and docs_

* fee change

* fee

* docs

* wip

* fix unique key test in transactions

* dox

Co-authored-by: jhuhnke <jessica@flipsidecrypto.com>

models/descriptions/_unique_key.md

@@ -0,0 +1,5 @@
+{% docs _unique_key %}
+
+The unique key for the table. 
+
+{% enddocs %}

models/descriptions/attribute_index.md

@@ -0,0 +1,5 @@
+{% docs attribute_index %}
+
+The position in which attributes occur within a message
+
+{% enddocs %}

models/descriptions/attribute_key.md

@@ -0,0 +1,5 @@
+{% docs attribute_key %}
+
+The key from the key-value pair from the message attribute
+
+{% enddocs %}

models/descriptions/attribute_value.md

@@ -0,0 +1,5 @@
+{% docs attribute_value %}
+
+The value from the key-value pair from the message attribute
+
+{% enddocs %}

models/descriptions/decimal.md

@@ -0,0 +1,5 @@
+{% docs decimal %}
+
+Divide amount by decimal to get the actual amount of currency being transferred. 
+
+{% enddocs %}

models/descriptions/fee.md

@@ -0,0 +1,5 @@
+{% docs fee %}
+
+The fee is paid by the initiator of the transaction. Fee = gas * gas price.
+
+{% enddocs %}

models/descriptions/fee_denom.md

@@ -0,0 +1,5 @@
+{% docs fee_denom %}
+
+The denominator of the transaction fee.
+
+{% enddocs %}

models/descriptions/tx_from.md

@@ -0,0 +1,5 @@
+{% docs tx_from %}
+
+The wallet address of the individual who initiated the transaction
+
+{% enddocs %}

models/gold/core__fact_blocks.sql

@@ -0,0 +1,14 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+SELECT 
+    block_id, 
+    block_timestamp, 
+    blockchain,
+    chain_id, 
+    tx_count, 
+    proposer_address, 
+    validator_hash
+FROM 
+    {{ ref('silver__blocks') }}

models/gold/core__fact_blocks.yml

@@ -0,0 +1,33 @@
+version: 2
+models:
+  - name: core__fact_blocks
+    description: Records of all blocks that have occurred on Cosmos, dating back to the genesis block. 
+    columns:
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: BLOCKCHAIN
+        description: "{{ doc('blockchain') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: CHAIN_ID
+        description: "{{ doc('chain_id') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: TX_COUNT
+        description: "{{ doc('tx_count') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: PROPOSER_ADDRESS
+        description: "{{ doc('proposer_address') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: VALIDATOR_HASH
+        description: "{{ doc('validator_hash') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist

models/gold/core__fact_msg_attributes.sql

@@ -0,0 +1,22 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+SELECT
+    block_id,
+    block_timestamp,
+    tx_id,
+    tx_succeeded,
+    CONCAT(
+        msg_group,
+        ':',
+        msg_sub_group
+    ) AS msg_group,
+    msg_index,
+    msg_type,
+    attribute_index,
+    attribute_key,
+    attribute_value,
+    unique_key
+FROM
+    {{ ref('silver__msg_attributes') }}

models/gold/core__fact_msg_attributes.yml

@@ -0,0 +1,50 @@
+version: 2
+models:
+  - name: core__fact_msg_attributes
+    description: Records of all message attributes associated to messages that have occurred on Cosmos, dating back to the genesis block. 
+    columns:
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: TX_SUCCEEDED
+        description: "{{ doc('tx_succeeded') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: MSG_GROUP
+        description: "{{ doc('msg_group') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist 
+      - name: MSG_INDEX
+        description: "{{ doc('msg_index') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist 
+      - name: MSG_TYPE
+        description: "{{ doc('msg_type') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: ATTRIBUTE_INDEX
+        description: "{{ doc('attribute_index') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: ATTRIBUTE_KEY
+        description: "{{ doc('attribute_key') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: ATTRIBUTE_VALUE
+        description: "{{ doc('attribute_value') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: UNIQUE_KEY
+        description: "{{ doc('_unique_key') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      

models/gold/core__fact_msgs.sql

@@ -0,0 +1,21 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+SELECT
+    block_id,
+    block_timestamp,
+    tx_id,
+    tx_succeeded,
+    CONCAT(
+        msg_group,
+        ':',
+        msg_sub_group
+    ) AS msg_group,
+    msg_index,
+    msg_type,
+    msg,
+    unique_key
+
+FROM
+    {{ ref('silver__msgs') }}

models/gold/core__fact_msgs.yml

@@ -0,0 +1,42 @@
+version: 2
+models:
+  - name: core__fact_msgs
+    description: Records of all message attributes associated to messages that have occurred on Cosmos, dating back to the genesis block. 
+    columns:
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: TX_SUCCEEDED
+        description: "{{ doc('tx_succeeded') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: MSG_GROUP
+        description: "{{ doc('msg_group') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist 
+      - name: MSG_INDEX
+        description: "{{ doc('msg_index') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist 
+      - name: MSG_TYPE
+        description: "{{ doc('msg_type') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: MSG
+        description: "{{ doc('msg') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: UNIQUE_KEY
+        description: "{{ doc('_unique_key') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      

models/gold/core__fact_transactions.sql

@@ -0,0 +1,192 @@
+{{ config(
+  materialized = 'incremental',
+  unique_key = "unique_key",
+  incremental_strategy = 'delete+insert',
+  cluster_by = ['block_timestamp::DATE'],
+) }}
+
+
+WITH
+
+{% if is_incremental() %}
+max_block_partition AS (
+
+  SELECT
+    MAX(
+      _partition_by_block_id
+    ) AS _partition_by_block_id_max
+  FROM
+    {{ ref('silver__transactions') }}
+),
+{% endif %}
+
+fee AS (
+  SELECT
+    block_id, 
+    tx_id,
+    attribute_value AS fee
+  FROM
+    {{ ref('silver__msg_attributes') }}
+  WHERE
+    attribute_key = 'fee'
+    AND msg_type = 'tx'
+    
+    {% if is_incremental() %}
+AND _partition_by_block_id >= (
+  SELECT
+    _partition_by_block_id_max 
+  FROM
+    max_block_partition
+)
+{% endif %}
+),
+
+spender AS (
+  SELECT
+    block_id, 
+    tx_id,
+    SPLIT_PART(
+      attribute_value,
+      '/',
+      0
+    ) AS tx_from
+  FROM
+    {{ ref('silver__msg_attributes') }}
+  WHERE
+    attribute_key = 'acc_seq'
+
+{% if is_incremental() %}
+AND _partition_by_block_id >= (
+  SELECT
+    _partition_by_block_id_max
+  FROM
+    max_block_partition
+)
+{% endif %}
+
+qualify(ROW_NUMBER() over(PARTITION BY tx_id
+ORDER BY
+  msg_index)) = 1
+),
+
+no_fee_tx_raw as (
+ select  
+    block_id, 
+    tx_id, 
+    f.index,
+    f.value: type ::string as event_type,
+    try_base64_decode_string( f.value:attributes[0]:value) as recipient,
+    try_base64_decode_string( f.value:attributes[1]:value) as sender ,
+    try_base64_decode_string( f.value:attributes[2]:value) as amount_raw ,
+    CASE 
+      WHEN amount_raw like '%uatom' then amount_raw 
+      ELSE '0uatom' 
+    END as amount 
+
+    FROM
+        {{ ref('silver__transactions') }} , 
+        TABLE (flatten (input => msgs)) f
+          
+    WHERE 
+        tx_id NOT IN 
+            (select 
+                tx_id 
+                from fee
+                )
+     AND event_type = 'transfer'
+    
+     qualify row_number() over (partition by tx_id order by f.index asc) = 1 
+    ),
+    
+no_fee_transactions AS (
+SELECT
+  t.block_id,
+  t.block_timestamp,
+  t.tx_id,
+  f.sender as tx_from,
+  tx_succeeded,
+  codespace,
+  COALESCE(
+    amount,
+    '0uatom'
+  ) AS fee_raw,
+  regexp_substr(fee_raw, '[0-9]+') as fee, 
+  regexp_substr(fee_raw, '[a-z]+') as fee_denom, 
+  gas_used,
+  gas_wanted,
+  tx_code,
+  tx_log,
+  msgs,
+  _partition_by_block_id, 
+  unique_key
+FROM
+  {{ ref('silver__transactions') }} 
+  t
+  INNER JOIN no_fee_tx_raw f
+  ON t.tx_id = f.tx_id
+  AND t.block_id = f.block_id
+    
+
+),
+
+fee_transactions AS (
+SELECT
+  t.block_id,
+  t.block_timestamp,
+  t.tx_id,
+  s.tx_from,
+  tx_succeeded,
+  codespace,
+  COALESCE(
+    fee,
+    '0uatom'
+  ) AS fee_raw,
+  regexp_substr(fee_raw, '[0-9]+') as fee, 
+  regexp_substr(fee_raw, '[a-z]+') as fee_denom, 
+  gas_used,
+  gas_wanted,
+  tx_code,
+  tx_log,
+  msgs,
+  _partition_by_block_id, 
+  unique_key
+FROM
+  {{ ref('silver__transactions') }} 
+  t
+  INNER JOIN fee f
+  ON t.tx_id = f.tx_id
+  AND t.block_id = f.block_id
+  INNER JOIN spender s
+  ON t.tx_id = s.tx_id
+  AND t.block_id = s.block_id
+), 
+
+final_transactions as (
+select * 
+    from no_fee_transactions
+
+union all 
+
+select * 
+    from fee_transactions
+)
+
+
+SELECT 
+  block_id,
+  block_timestamp,
+  tx_id,
+  tx_from,
+  tx_succeeded,
+  codespace,
+  fee, 
+  fee_denom, 
+  gas_used,
+  gas_wanted,
+  tx_code,
+  tx_log,
+  msgs,
+  _partition_by_block_id, 
+  unique_key
+
+FROM final_transactions

models/gold/core__fact_transactions.yml

@@ -0,0 +1,66 @@
+version: 2
+models:
+  - name: core__fact_transactions
+    description: Records of all transactions that have occurred on Cosmos, dating back to the genesis block. 
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - UNIQUE_KEY
+    columns:
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: TX_FROM
+        description: "{{ doc('tx_from') }}"
+        tests: 
+          - not_null: 
+              where: TX_SUCCEEDED = 'TRUE'
+      - name: TX_SUCCEEDED
+        description: "{{ doc('tx_succeeded') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: CODESPACE
+        description: "{{ doc('codespace') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: FEE
+        description: "{{ doc('fee') }}"
+        tests: 
+          - not_null
+      - name: FEE_DENOM
+        description: "{{ doc('fee_denom') }}"
+        tests: 
+          - not_null
+      - name: GAS_USED
+        description: "{{ doc('gas_used') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist  
+      - name: GAS_WANTED
+        description: "{{ doc('gas_wanted') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist  
+      - name: TX_CODE
+        description: "{{ doc('tx_code') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist 
+      - name: TX_LOG
+        description: "{{ doc('tx_log') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist 
+      - name: MSGS
+        description: "The underlying json from the messages or events within the transactions"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: UNIQUE_KEY
+        description: "{{ doc('_unique_key') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist

models/gold/core__fact_transfers.sql

@@ -0,0 +1,17 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+SELECT 
+    block_id, 
+    block_timestamp, 
+    tx_id, 
+    tx_succeeded, 
+    transfer_type, 
+    sender, 
+    amount, 
+    currency,
+    receiver,
+    unique_key
+
+FROM {{ ref('silver__transfers') }}

models/gold/core__fact_transfers.yml

@@ -0,0 +1,45 @@
+version: 2
+models:
+  - name: core__fact_transfers
+    description: Records of all transfers on Cosmos
+    columns:
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: TX_SUCCEEDED
+        description: "{{ doc('tx_succeeded') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: TRANSFER_TYPE
+        description: "{{ doc('transfer_type') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: SENDER
+        description: "{{ doc('sender') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: AMOUNT
+        description: "{{ doc('amount') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: CURRENCY
+        description: "{{ doc('currency') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: RECEIVER
+        description: "{{ doc('receiver') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist
+      - name: UNIQUE_KEY
+        description: "{{ doc('_unique_key') }}"
+        tests: 
+          - dbt_expectations.expect_column_to_exist

models/silver/silver__msg_attributes.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = "_unique_key",
+    unique_key = "unique_key",
     incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE'],
 ) }}
@@ -27,7 +27,7 @@ SELECT
         tx_id,
         msg_index,
         attribute_index
-    ) AS _unique_key,
+    ) AS unique_key,
     _partition_by_block_id
 FROM
     {{ ref('silver__msgs') }} A,

models/silver/silver__msg_attributes.yml

@@ -5,7 +5,7 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
-            - _UNIQUE_KEY
+            - UNIQUE_KEY
     columns:
       - name: BLOCK_ID
         description: "{{ doc('block_id') }}"

models/silver/silver__msgs.sql

@@ -1,6 +1,6 @@
 {{ config(
   materialized = 'incremental',
-  unique_key = "_unique_key",
+  unique_key = "unique_key",
   incremental_strategy = 'delete+insert',
   cluster_by = ['block_timestamp::DATE'],
 ) }}
@@ -120,7 +120,7 @@ msgs AS (
       block_id,
       A.tx_id,
       A.msg_index
-    ) AS _unique_key,
+    ) AS unique_key,
     _partition_by_block_id
   FROM
     base_msgs A
@@ -138,7 +138,7 @@ SELECT
   msg_index,
   msg_type,
   msg :: OBJECT as msg,
-  _unique_key,
+  unique_key,
   _partition_by_block_id
 FROM
   msgs

models/silver/silver__msgs.yml

@@ -5,7 +5,7 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
-            - _UNIQUE_KEY
+            - UNIQUE_KEY
     columns:
       - name: BLOCK_ID
         description: "{{ doc('block_id') }}"

models/silver/silver__transactions.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = "_unique_key",
+    unique_key = "unique_key",
     incremental_strategy = 'delete+insert',
     cluster_by = 'block_timestamp::DATE',
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
@@ -52,7 +52,7 @@ SELECT
         '-',
         t.block_id,
         tx_id
-    ) AS _unique_key
+    ) AS unique_key
 FROM
     base_transactions t
     JOIN {{ ref('silver__blocks') }}

models/silver/silver__transactions.yml

@@ -5,7 +5,7 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
-            - _UNIQUE_KEY
+            - UNIQUE_KEY
     columns:
       - name: BLOCK_ID
         description: "{{ doc('block_id') }}"

models/silver/silver__transfers.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = "_unique_key",
+    unique_key = "unique_key",
     incremental_strategy = 'delete+insert',
     cluster_by = 'block_timestamp::DATE',
 ) }}
@@ -193,7 +193,7 @@ cosmos_txs_final AS (
             r.tx_id,
             r.msg_index,
             currency
-        ) AS _unique_key
+        ) AS unique_key
     FROM
         receiver r
         LEFT OUTER JOIN amount C
@@ -239,7 +239,7 @@ ibc_in_tx AS (
             tx_id,
             msg_index,
             currency
-        ) AS _unique_key
+        ) AS unique_key
     FROM
         {{ ref('silver__msg_attributes') }}
     WHERE
@@ -295,7 +295,7 @@ ibc_out_tx AS (
             tx_id,
             msg_index,
             currency
-        ) AS _unique_key
+        ) AS unique_key
     FROM
         {{ ref('silver__msg_attributes') }}
     WHERE
@@ -341,7 +341,7 @@ ibc_tx_final AS (
         i.receiver,
         msg_index,
         _partition_by_block_id,
-        _unique_key
+        unique_key
     FROM
         ibc_transfers_agg i
 )
@@ -357,7 +357,7 @@ SELECT
     receiver,
     msg_index,
     _partition_by_block_id,
-    _unique_key
+    unique_key
 FROM
     ibc_tx_final
 UNION ALL
@@ -373,6 +373,6 @@ SELECT
     receiver,
     msg_index,
     _partition_by_block_id,
-    _unique_key
+    unique_key
 FROM
     cosmos_txs_final

models/silver/silver__transfers.yml

@@ -5,7 +5,7 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
-            - _UNIQUE_KEY
+            - UNIQUE_KEY
     columns:
       - name: BLOCK_ID
         description: "{{ doc('block_id') }}"