An 2745/burn and mint actions (#233)

* initial model

* add test cases

* add tests

* add signer/authority

* add mint_authority/signer cols

* tests and cleanup

* exclude model from incremental

* grab entire signers array

* change col name

* add not-null test for col

* load start date

.github/workflows/dbt_run_incremental.yml

@@ -44,6 +44,6 @@ jobs:
           dbt run-operation run_sp_refresh_external_tables_full
           dbt run -s models/silver/silver__transactions.sql models/silver/silver__blocks.sql models/silver/silver__votes.sql models/silver/silver___inner_instructions.sql models/silver/silver___instructions.sql models/silver/silver__events.sql models/silver/silver___all_undecoded_instructions_data.sql
           dbt run-operation run_sp_refresh_external_tables_full
-          dbt run -s ./models --exclude models/core models/silver/silver__transactions.sql models/silver/silver__blocks.sql models/silver/silver__votes.sql models/silver/silver___inner_instructions.sql models/silver/silver___instructions.sql models/silver/silver__events.sql models/silver/silver___all_undecoded_instructions_data.sql tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql models/silver/liquidity_pool/silver__initialization_pools_orca.sql
+          dbt run -s ./models --exclude models/core models/silver/silver__transactions.sql models/silver/silver__blocks.sql models/silver/silver__votes.sql models/silver/silver___inner_instructions.sql models/silver/silver___instructions.sql models/silver/silver__events.sql models/silver/silver___all_undecoded_instructions_data.sql tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql models/silver/liquidity_pool/silver__initialization_pools_orca.sql models/silver/silver__burn_actions.sql
           dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s ./models/core --exclude models/core/core__ez_signers.sql
           

data/testing__burn_actions.csv

@@ -0,0 +1,7 @@
+tx_id, event_type, mint, burn_authority, burn_amount
+4KrSQZq32joS7Cs4CXRq3yqpq94sBkvMZDYJA1uUSyzDH26qDnUdfxhPy4Skb8d1HBQvanuhkwMSiAWRS5ueStf5,burnChecked,Hm2HFCdJN3UkpkTgWcHpXTCdA4nwmSyVv2QW7mDQUMUb,DrtJrDZ4bhjP442xq13mogSz7ewiGmWkKsjbsQrTzETj,1
+5Gw9kgQi1Qh2fs1QCz9gt3ZTvL6bYqJ2pmbEgWP6cC5mYfPpSkppvZBBuxfMqWt7HQBw26dJGQjWzMQEyyhXBEHj,burn,VVWAy5U2KFd1p8AdchjUxqaJbZPBeP5vUQRZtAy8hyc,6BjXbcYT5Bmft4ZYCYNfwqwQVwmCk1epKKrQAgFXeSEZ,5439000000000
+5Gw9kgQi1Qh2fs1QCz9gt3ZTvL6bYqJ2pmbEgWP6cC5mYfPpSkppvZBBuxfMqWt7HQBw26dJGQjWzMQEyyhXBEHj,burn,Hz4EuZnooWRJxMY8VTBaeXfoSgFHaJqbJqK4fwF7ptHr,6BjXbcYT5Bmft4ZYCYNfwqwQVwmCk1epKKrQAgFXeSEZ,2
+4q64pGRsu79AyCjF8Ty2hkJfPinxm7JJhqWrefw8WLpkvW8n5Eg4EXxpgmDZGPEjoCXsCYGz5EphjQ4bmmbjMf8M,burn,8odYyYV8AfevKVrFjzoiizowAXrDgdbEB5CQ7wNa36G4,6qFyJTaH8yehBRwAHs8fgJxfyrDY15QEo3ic3sJAexyY,9
+5MBgE9rq527vcZK5fHzYE3XD5qQnzTmHZ1R4FnLCQGi3VK9LvroFNnKGpF7tpDtbYGZTUibJQbQXA1PquBq8rDHT,burn,6VBTGNqKUbeQbMD42hENjMJTYbqZpeZDKa7JXqvKgEdv,9uDFnWc9JCg9e23d2LjFa9NEH4upyoPV6ZPpTbP9jeZ9,17004
+4oEuNB3GMvPPrhQDVmF52dkh2kHXdEpZC29xYN1H24Cq8apHEUk92VHQsLzCNhcEude8VJ2Z9tAWa8pXr9tBP27H,burn,78EQHRXfw6TM89F1CzwbsTYXhLcpqKErEzNPG8qjqMQp,9uDFnWc9JCg9e23d2LjFa9NEH4upyoPV6ZPpTbP9jeZ9,352015

models/silver/silver__burn_actions.sql

@@ -0,0 +1,98 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = "CONCAT_WS('-', tx_id, event_type, mint)",
+    incremental_strategy = 'delete+insert',
+    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
+    full_refresh = false
+) }}
+
+WITH base_events AS (
+
+    SELECT
+        *
+    FROM
+        {{ ref('silver__events') }}
+{% if is_incremental() and env_var(
+    'DBT_IS_BATCH_LOAD',
+    "false"
+) == "true" %}
+WHERE
+    block_id BETWEEN (
+        SELECT
+            LEAST(COALESCE(MAX(block_id), 31310775)+1,175418104)
+        FROM
+            {{ this }}
+        )
+        AND (
+        SELECT
+            LEAST(COALESCE(MAX(block_id), 31310775)+4000000,175418104)
+        FROM
+            {{ this }}
+        ) 
+{% elif is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(_inserted_timestamp)
+        FROM
+            {{ this }}
+    )
+{% else %}
+WHERE
+    block_id between 31310775 and 32310775
+{% endif %}
+)
+SELECT
+    block_id,
+    block_timestamp,
+    tx_id,
+    succeeded,
+    index,
+    null as inner_index,
+    event_type,
+    instruction :parsed :info :mint :: STRING AS mint,
+    COALESCE(
+        instruction :parsed :info :amount :: INTEGER,
+        instruction :parsed :info :tokenAmount: amount :: INTEGER
+    ) AS burn_amount,
+    COALESCE(
+        instruction :parsed :info :authority :: string,
+        instruction :parsed :info :multisigAuthority :: string
+    ) AS burn_authority,
+    instruction :parsed :info :signers :: string AS signers,
+    _inserted_timestamp
+FROM
+    base_events
+WHERE
+    event_type IN (
+       'burn',
+        'burnChecked'
+    )
+UNION
+SELECT
+    block_id,
+    block_timestamp,
+    tx_id,
+    succeeded,
+    e.index,
+    i.index as inner_index,
+    i.value :parsed :type :: STRING AS event_type,
+    i.value :parsed :info :mint :: STRING AS mint,
+    COALESCE(
+        i.value :parsed :info :amount :: INTEGER,
+        i.value :parsed :info :tokenAmount: amount :: INTEGER
+    ) AS burn_amount,
+    COALESCE(
+        i.value :parsed :info :authority :: string,
+        i.value :parsed :info :multisigAuthority :: string
+    ) AS burn_authority,
+    instruction :parsed :info :signers :: string AS signers,
+    _inserted_timestamp
+FROM
+    base_events e,
+    TABLE(FLATTEN(inner_instruction :instructions)) i
+WHERE
+    i.value :parsed :type :: STRING IN (
+       'burn',
+        'burnChecked'
+    )

models/silver/silver__burn_actions.yml

@@ -0,0 +1,75 @@
+version: 2
+models:
+  - name: silver__burn_actions
+    description: table holding token burn events
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TX_ID
+            - INDEX
+            - INNER_INDEX
+            - EVENT_TYPE
+            - MINT
+      - compare_model_subset:
+          name: silver__burn_actions_logic_test
+          compare_model: ref('testing__burn_actions')
+          compare_columns: 
+            - TX_ID
+            - EVENT_TYPE
+            - MINT
+            - BURN_AUTHORITY
+            - BURN_AMOUNT
+          model_condition: "where tx_id in ('4KrSQZq32joS7Cs4CXRq3yqpq94sBkvMZDYJA1uUSyzDH26qDnUdfxhPy4Skb8d1HBQvanuhkwMSiAWRS5ueStf5',
+          '5Gw9kgQi1Qh2fs1QCz9gt3ZTvL6bYqJ2pmbEgWP6cC5mYfPpSkppvZBBuxfMqWt7HQBw26dJGQjWzMQEyyhXBEHj',
+          '4q64pGRsu79AyCjF8Ty2hkJfPinxm7JJhqWrefw8WLpkvW8n5Eg4EXxpgmDZGPEjoCXsCYGz5EphjQ4bmmbjMf8M',
+          '5MBgE9rq527vcZK5fHzYE3XD5qQnzTmHZ1R4FnLCQGi3VK9LvroFNnKGpF7tpDtbYGZTUibJQbQXA1PquBq8rDHT',
+          '4oEuNB3GMvPPrhQDVmF52dkh2kHXdEpZC29xYN1H24Cq8apHEUk92VHQsLzCNhcEude8VJ2Z9tAWa8pXr9tBP27H')"
+    columns:
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - not_null:
+              where: block_id > 39824213 and _inserted_timestamp::date < current_date
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 3
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - not_null
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests:
+          - not_null
+      - name: SUCCEEDED
+        description: "{{ doc('tx_succeeded') }}"
+        tests:
+          - not_null
+      - name: INDEX
+        description: "{{ doc('event_index') }}"
+        tests:
+          - not_null
+      - name: INNER_INDEX
+        description: Location of the inner instruction within an instruction
+      - name: EVENT_TYPE 
+        description: "{{ doc('event_type') }}"
+        tests: 
+          - not_null
+      - name: MINT
+        description: "{{ doc('mint') }}"
+        tests: 
+          - not_null
+      - name: BURN_AMOUNT
+        description: Number of tokens burned
+        tests: 
+          - not_null
+      - name: BURN_AUTHORITY
+        description: Account address authorizing burn
+        tests: 
+          - not_null
+      - name: SIGNERS
+        description: Account address authorizing burn
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('_inserted_timestamp') }}"
+        tests: 
+          - not_null

models/silver/silver__mint_actions.sql

@@ -56,6 +56,11 @@ SELECT
         instruction :parsed :info :amount :: INTEGER,
         instruction :parsed :info :tokenAmount: amount :: INTEGER
     ) AS mint_amount,
+    COALESCE(
+        instruction :parsed :info :mintAuthority :: string,
+        instruction :parsed :info :multisigAuthority :: string
+    ) AS mint_authority,
+    instruction :parsed :info :signers :: string AS signers,
     _inserted_timestamp
 FROM
     base_events
@@ -81,6 +86,11 @@ SELECT
         i.value :parsed :info :amount :: INTEGER,
         i.value :parsed :info :tokenAmount: amount :: INTEGER
     ) AS mint_amount,
+    COALESCE(
+        i.value :parsed :info :mintAuthority :: string,
+        i.value :parsed :info :multisigAuthority :: string
+    ) AS mint_authority,
+    i.value :parsed :info :signers :: string AS signers,
     _inserted_timestamp
 FROM
     base_events e,

models/silver/silver__mint_actions.yml

@@ -55,6 +55,10 @@ models:
         tests: 
           - not_null:
               where: event_type in ('mintTo','mintToChecked')
+      - name: MINT_AUTHORITY
+        description: "Account address authorizing mint"
+      - name: SIGNERS
+        description: "Account address signing off on mint"
       - name: _INSERTED_TIMESTAMP
         description: "{{ doc('_inserted_timestamp') }}"
         tests: