diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml index 6e412e4a..45f74b2c 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -44,6 +44,6 @@ jobs: dbt run-operation run_sp_refresh_external_tables_full dbt run -s models/silver/silver__transactions.sql models/silver/silver__blocks.sql models/silver/silver__votes.sql models/silver/silver___inner_instructions.sql models/silver/silver___instructions.sql models/silver/silver__events.sql models/silver/silver___all_undecoded_instructions_data.sql dbt run-operation run_sp_refresh_external_tables_full - dbt run -s ./models --exclude models/core models/silver/silver__transactions.sql models/silver/silver__blocks.sql models/silver/silver__votes.sql models/silver/silver___inner_instructions.sql models/silver/silver___instructions.sql models/silver/silver__events.sql models/silver/silver___all_undecoded_instructions_data.sql tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql models/silver/liquidity_pool/silver__initialization_pools_orca.sql + dbt run -s ./models --exclude models/core models/silver/silver__transactions.sql models/silver/silver__blocks.sql models/silver/silver__votes.sql models/silver/silver___inner_instructions.sql models/silver/silver___instructions.sql models/silver/silver__events.sql models/silver/silver___all_undecoded_instructions_data.sql tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql models/silver/liquidity_pool/silver__initialization_pools_orca.sql models/silver/silver__burn_actions.sql dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s ./models/core --exclude models/core/core__ez_signers.sql diff --git a/data/testing__burn_actions.csv b/data/testing__burn_actions.csv new file mode 100644 index 00000000..009ec754 --- /dev/null +++ b/data/testing__burn_actions.csv @@ -0,0 +1,7 @@ +tx_id, event_type, mint, burn_authority, burn_amount +4KrSQZq32joS7Cs4CXRq3yqpq94sBkvMZDYJA1uUSyzDH26qDnUdfxhPy4Skb8d1HBQvanuhkwMSiAWRS5ueStf5,burnChecked,Hm2HFCdJN3UkpkTgWcHpXTCdA4nwmSyVv2QW7mDQUMUb,DrtJrDZ4bhjP442xq13mogSz7ewiGmWkKsjbsQrTzETj,1 +5Gw9kgQi1Qh2fs1QCz9gt3ZTvL6bYqJ2pmbEgWP6cC5mYfPpSkppvZBBuxfMqWt7HQBw26dJGQjWzMQEyyhXBEHj,burn,VVWAy5U2KFd1p8AdchjUxqaJbZPBeP5vUQRZtAy8hyc,6BjXbcYT5Bmft4ZYCYNfwqwQVwmCk1epKKrQAgFXeSEZ,5439000000000 +5Gw9kgQi1Qh2fs1QCz9gt3ZTvL6bYqJ2pmbEgWP6cC5mYfPpSkppvZBBuxfMqWt7HQBw26dJGQjWzMQEyyhXBEHj,burn,Hz4EuZnooWRJxMY8VTBaeXfoSgFHaJqbJqK4fwF7ptHr,6BjXbcYT5Bmft4ZYCYNfwqwQVwmCk1epKKrQAgFXeSEZ,2 +4q64pGRsu79AyCjF8Ty2hkJfPinxm7JJhqWrefw8WLpkvW8n5Eg4EXxpgmDZGPEjoCXsCYGz5EphjQ4bmmbjMf8M,burn,8odYyYV8AfevKVrFjzoiizowAXrDgdbEB5CQ7wNa36G4,6qFyJTaH8yehBRwAHs8fgJxfyrDY15QEo3ic3sJAexyY,9 +5MBgE9rq527vcZK5fHzYE3XD5qQnzTmHZ1R4FnLCQGi3VK9LvroFNnKGpF7tpDtbYGZTUibJQbQXA1PquBq8rDHT,burn,6VBTGNqKUbeQbMD42hENjMJTYbqZpeZDKa7JXqvKgEdv,9uDFnWc9JCg9e23d2LjFa9NEH4upyoPV6ZPpTbP9jeZ9,17004 +4oEuNB3GMvPPrhQDVmF52dkh2kHXdEpZC29xYN1H24Cq8apHEUk92VHQsLzCNhcEude8VJ2Z9tAWa8pXr9tBP27H,burn,78EQHRXfw6TM89F1CzwbsTYXhLcpqKErEzNPG8qjqMQp,9uDFnWc9JCg9e23d2LjFa9NEH4upyoPV6ZPpTbP9jeZ9,352015 diff --git a/models/silver/silver__burn_actions.sql b/models/silver/silver__burn_actions.sql new file mode 100644 index 00000000..6e360ce0 --- /dev/null +++ b/models/silver/silver__burn_actions.sql @@ -0,0 +1,98 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', tx_id, event_type, mint)", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + full_refresh = false +) }} + +WITH base_events AS ( + + SELECT + * + FROM + {{ ref('silver__events') }} +{% if is_incremental() and env_var( + 'DBT_IS_BATCH_LOAD', + "false" +) == "true" %} +WHERE + block_id BETWEEN ( + SELECT + LEAST(COALESCE(MAX(block_id), 31310775)+1,175418104) + FROM + {{ this }} + ) + AND ( + SELECT + LEAST(COALESCE(MAX(block_id), 31310775)+4000000,175418104) + FROM + {{ this }} + ) +{% elif is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) +{% else %} +WHERE + block_id between 31310775 and 32310775 +{% endif %} +) +SELECT + block_id, + block_timestamp, + tx_id, + succeeded, + index, + null as inner_index, + event_type, + instruction :parsed :info :mint :: STRING AS mint, + COALESCE( + instruction :parsed :info :amount :: INTEGER, + instruction :parsed :info :tokenAmount: amount :: INTEGER + ) AS burn_amount, + COALESCE( + instruction :parsed :info :authority :: string, + instruction :parsed :info :multisigAuthority :: string + ) AS burn_authority, + instruction :parsed :info :signers :: string AS signers, + _inserted_timestamp +FROM + base_events +WHERE + event_type IN ( + 'burn', + 'burnChecked' + ) +UNION +SELECT + block_id, + block_timestamp, + tx_id, + succeeded, + e.index, + i.index as inner_index, + i.value :parsed :type :: STRING AS event_type, + i.value :parsed :info :mint :: STRING AS mint, + COALESCE( + i.value :parsed :info :amount :: INTEGER, + i.value :parsed :info :tokenAmount: amount :: INTEGER + ) AS burn_amount, + COALESCE( + i.value :parsed :info :authority :: string, + i.value :parsed :info :multisigAuthority :: string + ) AS burn_authority, + instruction :parsed :info :signers :: string AS signers, + _inserted_timestamp +FROM + base_events e, + TABLE(FLATTEN(inner_instruction :instructions)) i +WHERE + i.value :parsed :type :: STRING IN ( + 'burn', + 'burnChecked' + ) diff --git a/models/silver/silver__burn_actions.yml b/models/silver/silver__burn_actions.yml new file mode 100644 index 00000000..01f52a52 --- /dev/null +++ b/models/silver/silver__burn_actions.yml @@ -0,0 +1,75 @@ +version: 2 +models: + - name: silver__burn_actions + description: table holding token burn events + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - INDEX + - INNER_INDEX + - EVENT_TYPE + - MINT + - compare_model_subset: + name: silver__burn_actions_logic_test + compare_model: ref('testing__burn_actions') + compare_columns: + - TX_ID + - EVENT_TYPE + - MINT + - BURN_AUTHORITY + - BURN_AMOUNT + model_condition: "where tx_id in ('4KrSQZq32joS7Cs4CXRq3yqpq94sBkvMZDYJA1uUSyzDH26qDnUdfxhPy4Skb8d1HBQvanuhkwMSiAWRS5ueStf5', + '5Gw9kgQi1Qh2fs1QCz9gt3ZTvL6bYqJ2pmbEgWP6cC5mYfPpSkppvZBBuxfMqWt7HQBw26dJGQjWzMQEyyhXBEHj', + '4q64pGRsu79AyCjF8Ty2hkJfPinxm7JJhqWrefw8WLpkvW8n5Eg4EXxpgmDZGPEjoCXsCYGz5EphjQ4bmmbjMf8M', + '5MBgE9rq527vcZK5fHzYE3XD5qQnzTmHZ1R4FnLCQGi3VK9LvroFNnKGpF7tpDtbYGZTUibJQbQXA1PquBq8rDHT', + '4oEuNB3GMvPPrhQDVmF52dkh2kHXdEpZC29xYN1H24Cq8apHEUk92VHQsLzCNhcEude8VJ2Z9tAWa8pXr9tBP27H')" + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: block_id > 39824213 and _inserted_timestamp::date < current_date + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - name: INDEX + description: "{{ doc('event_index') }}" + tests: + - not_null + - name: INNER_INDEX + description: Location of the inner instruction within an instruction + - name: EVENT_TYPE + description: "{{ doc('event_type') }}" + tests: + - not_null + - name: MINT + description: "{{ doc('mint') }}" + tests: + - not_null + - name: BURN_AMOUNT + description: Number of tokens burned + tests: + - not_null + - name: BURN_AUTHORITY + description: Account address authorizing burn + tests: + - not_null + - name: SIGNERS + description: Account address authorizing burn + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__mint_actions.sql b/models/silver/silver__mint_actions.sql index 784da51e..7c278e81 100644 --- a/models/silver/silver__mint_actions.sql +++ b/models/silver/silver__mint_actions.sql @@ -56,6 +56,11 @@ SELECT instruction :parsed :info :amount :: INTEGER, instruction :parsed :info :tokenAmount: amount :: INTEGER ) AS mint_amount, + COALESCE( + instruction :parsed :info :mintAuthority :: string, + instruction :parsed :info :multisigAuthority :: string + ) AS mint_authority, + instruction :parsed :info :signers :: string AS signers, _inserted_timestamp FROM base_events @@ -81,6 +86,11 @@ SELECT i.value :parsed :info :amount :: INTEGER, i.value :parsed :info :tokenAmount: amount :: INTEGER ) AS mint_amount, + COALESCE( + i.value :parsed :info :mintAuthority :: string, + i.value :parsed :info :multisigAuthority :: string + ) AS mint_authority, + i.value :parsed :info :signers :: string AS signers, _inserted_timestamp FROM base_events e, diff --git a/models/silver/silver__mint_actions.yml b/models/silver/silver__mint_actions.yml index 51024d33..c8ae3ec8 100644 --- a/models/silver/silver__mint_actions.yml +++ b/models/silver/silver__mint_actions.yml @@ -55,6 +55,10 @@ models: tests: - not_null: where: event_type in ('mintTo','mintToChecked') + - name: MINT_AUTHORITY + description: "Account address authorizing mint" + - name: SIGNERS + description: "Account address signing off on mint" - name: _INSERTED_TIMESTAMP description: "{{ doc('_inserted_timestamp') }}" tests: