diff --git a/models/descriptions/burn_amount.md b/models/descriptions/burn_amount.md new file mode 100644 index 0000000..2d936ff --- /dev/null +++ b/models/descriptions/burn_amount.md @@ -0,0 +1,5 @@ +{% docs burn_amount %} + +The number of tokens burned + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/burn_authority.md b/models/descriptions/burn_authority.md new file mode 100644 index 0000000..56b48f8 --- /dev/null +++ b/models/descriptions/burn_authority.md @@ -0,0 +1,5 @@ +{% docs burn_authority %} + +The account address that is authorizing the burn action + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/mint_amount.md b/models/descriptions/mint_amount.md new file mode 100644 index 0000000..4a7a2c1 --- /dev/null +++ b/models/descriptions/mint_amount.md @@ -0,0 +1,5 @@ +{% docs mint_amount %} + +The number of tokens minted + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/mint_authority.md b/models/descriptions/mint_authority.md new file mode 100644 index 0000000..8581f28 --- /dev/null +++ b/models/descriptions/mint_authority.md @@ -0,0 +1,5 @@ +{% docs mint_authority %} + +The account address that is authorizing the mint action + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/mint_standard_type.md b/models/descriptions/mint_standard_type.md new file mode 100644 index 0000000..295867a --- /dev/null +++ b/models/descriptions/mint_standard_type.md @@ -0,0 +1,5 @@ +{% docs mint_standard_type %} + +The type of mint following Metaplex mint standards + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/mint_type.md b/models/descriptions/mint_type.md new file mode 100644 index 0000000..a002b39 --- /dev/null +++ b/models/descriptions/mint_type.md @@ -0,0 +1,5 @@ +{% docs mint_type %} + +The general identification of a mint: NFT or Token + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/token_account.md b/models/descriptions/token_account.md new file mode 100644 index 0000000..217337c --- /dev/null +++ b/models/descriptions/token_account.md @@ -0,0 +1,5 @@ +{% docs token_account %} + +The account address that has ownership of a specific token/nft + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/defi/defi__fact_token_burn_actions.sql b/models/gold/defi/defi__fact_token_burn_actions.sql new file mode 100644 index 0000000..efece10 --- /dev/null +++ b/models/gold/defi/defi__fact_token_burn_actions.sql @@ -0,0 +1,47 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'Token' }}}, + unique_key = ["fact_token_burn_actions_id"], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE'], + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id, program_id, tx_from, tx_to, mint, fact_transfers_id)'), + tags = ['scheduled_non_core'] +) }} + +{% if execute %} + {% if is_incremental() %} + {% set max_modified_query %} + SELECT + MAX(modified_timestamp) AS modified_timestamp + FROM + {{ this }} + {% endset %} + {% set max_modified_timestamp = run_query(max_modified_query)[0][0] %} + {% endif %} +{% endif %} + +SELECT + block_id, + block_timestamp, + tx_id, + succeeded, + INDEX, + inner_index, + event_type, + mint, + burn_amount, + burn_authority, + token_account, + signers, + DECIMAL, + mint_standard_type, + token_burn_actions_id AS fact_token_burn_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__token_burn_actions') }} +{% if is_incremental() %} +WHERE + modified_timestamp > '{{ max_modified_timestamp }}' +{% endif %} diff --git a/models/gold/defi/defi__fact_token_burn_actions.yml b/models/gold/defi/defi__fact_token_burn_actions.yml new file mode 100644 index 0000000..8bcce28 --- /dev/null +++ b/models/gold/defi/defi__fact_token_burn_actions.yml @@ -0,0 +1,97 @@ +version: 2 +models: + - name: defi__fact_token_burn_actions + description: Contains all burn events for tokens + recent_date_filter: &recent_date_filter + config: + where: modified_timestamp >= current_date - 7 + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - INDEX + - INNER_INDEX + - MINT + where: block_timestamp::date > current_date - 30 + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: TX_ID + description: "{{ doc('tx_id') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: INDEX + description: "{{ doc('event_index') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: INNER_INDEX + description: "{{ doc('inner_instruction_index') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - name: EVENT_TYPE + description: "{{ doc('event_type') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: MINT + description: "{{ doc('mint') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: BURN_AMOUNT + description: "{{ doc('burn_amount') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: BURN_AUTHORITY + description: "{{ doc('burn_authority') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: TOKEN_ACCOUNT + description: "{{ doc('token_account') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - name: SIGNERS + description: "{{ doc('signers') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - name: DECIMAL + description: "{{ doc('decimal') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - name: MINT_STANDARD_TYPE + description: "{{ doc('mint_standard_type') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - name: FACT_TOKEN_BURN_ACTIONS_ID + description: '{{ doc("pk") }}' + data_tests: + - not_null: *recent_date_filter + - unique + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + data_tests: + - not_null: *recent_date_filter + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + data_tests: + - not_null diff --git a/models/gold/defi/defi__fact_token_mint_actions.sql b/models/gold/defi/defi__fact_token_mint_actions.sql new file mode 100644 index 0000000..c5c351f --- /dev/null +++ b/models/gold/defi/defi__fact_token_mint_actions.sql @@ -0,0 +1,47 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'Token' }}}, + unique_key = ["fact_token_mint_actions_id"], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE'], + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id, program_id, tx_from, tx_to, mint, fact_transfers_id)'), + tags = ['scheduled_non_core'] +) }} + +{% if execute %} + {% if is_incremental() %} + {% set max_modified_query %} + SELECT + MAX(modified_timestamp) AS modified_timestamp + FROM + {{ this }} + {% endset %} + {% set max_modified_timestamp = run_query(max_modified_query)[0][0] %} + {% endif %} +{% endif %} + +SELECT + block_id, + block_timestamp, + tx_id, + succeeded, + INDEX, + inner_index, + event_type, + mint, + mint_amount, + mint_authority, + token_account, + signers, + DECIMAL, + mint_standard_type, + token_mint_actions_id AS fact_token_mint_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__token_mint_actions') }} +{% if is_incremental() %} +WHERE + modified_timestamp > '{{ max_modified_timestamp }}' +{% endif %} diff --git a/models/gold/defi/defi__fact_token_mint_actions.sql.yml b/models/gold/defi/defi__fact_token_mint_actions.sql.yml new file mode 100644 index 0000000..a302032 --- /dev/null +++ b/models/gold/defi/defi__fact_token_mint_actions.sql.yml @@ -0,0 +1,102 @@ +version: 2 +models: + - name: defi__fact_token_mint_actions + description: Contains all mint events for tokens + recent_date_filter: &recent_date_filter + config: + where: modified_timestamp >= current_date - 7 + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - MINT + - INDEX + - INNER_INDEX + where: block_timestamp::date > current_date - 30 + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: TX_ID + description: "{{ doc('tx_id') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null: *recent_date_filter + - name: INDEX + description: "{{ doc('event_index') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: INNER_INDEX + description: "{{ doc('inner_instruction_index') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - name: EVENT_TYPE + description: "{{ doc('event_type') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: MINT + description: "{{ doc('mint') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: TOKEN_ACCOUNT + description: "{{ doc('token_account') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - name: DECIMAL + description: "{{ doc('decimal') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: + config: + where: > + modified_timestamp >= current_date - 7 + AND event_type in ('initializeMint','initializeMint2') + - name: MINT_AMOUNT + description: "{{ doc('mint_amount') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: + config: + where: > + modified_timestamp >= current_date - 7 + AND event_type in ('mintTo','mintToChecked') + - name: MINT_AUTHORITY + description: "{{ doc('mint_authority') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - name: SIGNERS + description: "{{ doc('signers') }}" + data_tests: + - dbt_expectations.expect_column_to_exist + - name: FACT_TOKEN_MINT_ACTIONS_ID + description: '{{ doc("pk") }}' + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + data_tests: + - dbt_expectations.expect_column_to_exist + - not_null \ No newline at end of file diff --git a/models/silver/core/silver__burn_actions.sql b/models/silver/core/silver__burn_actions.sql new file mode 100644 index 0000000..becbc1a --- /dev/null +++ b/models/silver/core/silver__burn_actions.sql @@ -0,0 +1,95 @@ +{{ config( + materialized = 'incremental', + unique_key = ["burn_actions_id"], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + merge_exclude_columns = ["inserted_timestamp"], + tags = ['scheduled_non_core'] +) }} + +{% if execute %} + {% if is_incremental() %} + {% set max_inserted_query %} + SELECT + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + {{ this }} + {% endset %} + {% set max_inserted_timestamp = run_query(max_inserted_query)[0][0] %} + {% endif %} +{% endif %} + +WITH base AS ( + SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + INDEX, + NULL AS inner_index, + program_id, + event_type, + instruction, + _inserted_timestamp + FROM + {{ ref('silver__events') }} + WHERE + event_type IN ('burn','burnChecked') + {% if is_incremental() %} + AND _inserted_timestamp >= '{{ max_inserted_timestamp }}' + {% else %} + AND _inserted_timestamp :: DATE = '2024-09-12' + {% endif %} + UNION ALL + SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + instruction_index AS INDEX, + inner_index, + program_id, + event_type, + instruction, + _inserted_timestamp + FROM + {{ ref('silver__events_inner') }} + WHERE + event_type IN ( + 'burn', + 'burnChecked' + ) + {% if is_incremental() %} + AND _inserted_timestamp >= '{{ max_inserted_timestamp }}' + {% else %} + AND _inserted_timestamp :: DATE = '2024-09-12' + {% endif %} +) +SELECT + block_id, + block_timestamp, + tx_id, + succeeded, + INDEX, + inner_index, + event_type, + instruction :parsed :info :mint :: STRING AS mint, + instruction :parsed :info :account :: STRING AS token_account, + COALESCE( + instruction :parsed :info :amount :: INTEGER, + instruction :parsed :info :tokenAmount: amount :: INTEGER + ) AS burn_amount, + COALESCE( + instruction :parsed :info :authority :: STRING, + instruction :parsed :info :multisigAuthority :: STRING + ) AS burn_authority, + instruction :parsed :info :signers :: STRING AS signers, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_id', 'mint', 'index', 'inner_index'] + ) }} AS burn_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + base diff --git a/models/silver/core/silver__burn_actions.yml b/models/silver/core/silver__burn_actions.yml new file mode 100644 index 0000000..25f2c3b --- /dev/null +++ b/models/silver/core/silver__burn_actions.yml @@ -0,0 +1,82 @@ +version: 2 +models: + - name: silver__burn_actions + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - MINT + - INDEX + - INNER_INDEX + where: block_timestamp::date > current_date - 30 + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + data_tests: + - not_null: *recent_date_filter + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + data_tests: + - not_null: *recent_date_filter + - name: TX_ID + description: "{{ doc('tx_id') }}" + data_tests: + - not_null: *recent_date_filter + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null: *recent_date_filter + - name: INDEX + description: "{{ doc('event_index') }}" + data_tests: + - not_null: *recent_date_filter + - name: INNER_INDEX + description: "{{ doc('inner_instruction_index') }}" + - name: EVENT_TYPE + description: "{{ doc('event_type') }}" + data_tests: + - not_null: *recent_date_filter + - name: MINT + description: "{{ doc('mint') }}" + data_tests: + - not_null: *recent_date_filter + - name: TOKEN_ACCOUNT + description: "{{ doc('token_account') }}" + - name: BURN_AMOUNT + description: "{{ doc('burn_amount') }}" + data_tests: + - not_null: *recent_date_filter + - name: BURN_AUTHORITY + description: "{{ doc('burn_authority') }}" + data_tests: + - not_null: *recent_date_filter + - name: SIGNERS + description: "{{ doc('signers') }}" + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + data_tests: + - not_null + - name: BURN_ACTIONS_ID + description: '{{ doc("pk") }}' + data_tests: + - not_null: *recent_date_filter + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + data_tests: + - not_null + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + data_tests: + - not_null: *recent_date_filter + - name: _INVOCATION_ID + description: '{{ doc("_invocation_id") }}' + data_tests: + - not_null: + name: test_silver__not_null_burn_actions_invocation_id + <<: *recent_date_filter \ No newline at end of file diff --git a/models/silver/core/silver__mint_actions.sql b/models/silver/core/silver__mint_actions.sql new file mode 100644 index 0000000..37f824c --- /dev/null +++ b/models/silver/core/silver__mint_actions.sql @@ -0,0 +1,141 @@ +{{ config( + materialized = 'incremental', + unique_key = ["mint_actions_id"], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + merge_exclude_columns = ["inserted_timestamp"], + tags = ['scheduled_non_core'] +) }} + +{% if execute %} + {% if is_incremental() %} + {% set max_inserted_query %} + SELECT + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + {{ this }} + {% endset %} + {% set max_inserted_timestamp = run_query(max_inserted_query)[0][0] %} + {% endif %} +{% endif %} + +WITH base AS ( + SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + INDEX, + NULL AS inner_index, + program_id, + event_type, + instruction, + _inserted_timestamp + FROM + {{ ref('silver__events') }} + WHERE + event_type IN ( + 'mintTo', + 'initializeMint', + 'mintToChecked', + 'initializeMint2', + 'initializeConfidentialTransferMint', + 'initializeNonTransferableMint' + ) + {% if is_incremental() %} + AND _inserted_timestamp >= '{{ max_inserted_timestamp }}' + {% else %} + AND _inserted_timestamp :: DATE = '2024-09-12' + {% endif %} + UNION ALL + SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + instruction_index AS INDEX, + inner_index, + program_id, + event_type, + instruction, + _inserted_timestamp + FROM + {{ ref('silver__events_inner') }} + WHERE + event_type IN ( + 'mintTo', + 'initializeMint', + 'mintToChecked', + 'initializeMint2', + 'initializeConfidentialTransferMint', + 'initializeNonTransferableMint' + ) + {% if is_incremental() %} + AND _inserted_timestamp >= '{{ max_inserted_timestamp }}' + {% else %} + AND _inserted_timestamp :: DATE = '2024-09-12' + {% endif %} +), +prefinal AS ( + SELECT + block_id, + block_timestamp, + tx_id, + succeeded, + INDEX, + inner_index, + event_type, + instruction :parsed :info :mint :: STRING AS mint, + instruction :parsed :info :account :: STRING AS token_account, + instruction :parsed :info :decimals :: INTEGER AS DECIMAL, + COALESCE( + instruction :parsed :info :amount :: INTEGER, + instruction :parsed :info :tokenAmount: amount :: INTEGER + ) AS mint_amount, + COALESCE( + instruction :parsed :info :mintAuthority :: STRING, + instruction :parsed :info :multisigMintAuthority :: STRING, + instruction :parsed :info :authority :: STRING + ) AS mint_authority, + instruction :parsed :info :signers :: STRING AS signers, + _inserted_timestamp + FROM + base +) +SELECT + block_id, + block_timestamp, + tx_id, + succeeded, + INDEX, + inner_index, + event_type, + mint, + token_account, + DECIMAL, + mint_amount, + CASE + WHEN event_type IN ( + 'initializeConfidentialTransferMint', + 'initializeNonTransferableMint' + ) THEN FIRST_VALUE(mint_authority) ignore nulls over ( + PARTITION BY tx_id + ORDER BY + CASE + WHEN mint_authority IS NOT NULL THEN 0 + ELSE 1 + END, + INDEX + ) + ELSE mint_authority + END AS mint_authority, + signers, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_id', 'mint', 'index', 'inner_index'] + ) }} AS mint_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + prefinal diff --git a/models/silver/core/silver__mint_actions.yml b/models/silver/core/silver__mint_actions.yml new file mode 100644 index 0000000..39674d6 --- /dev/null +++ b/models/silver/core/silver__mint_actions.yml @@ -0,0 +1,92 @@ +version: 2 +models: + - name: silver__mint_actions + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - MINT + - INDEX + - INNER_INDEX + where: block_timestamp::date > current_date - 30 + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + data_tests: + - not_null: *recent_date_filter + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + data_tests: + - not_null: *recent_date_filter + - name: TX_ID + description: "{{ doc('tx_id') }}" + data_tests: + - not_null: *recent_date_filter + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null: *recent_date_filter + - name: INDEX + description: "{{ doc('event_index') }}" + data_tests: + - not_null: *recent_date_filter + - name: INNER_INDEX + description: "{{ doc('inner_instruction_index') }}" + - name: EVENT_TYPE + description: "{{ doc('event_type') }}" + data_tests: + - not_null: *recent_date_filter + - name: MINT + description: "{{ doc('mint') }}" + data_tests: + - not_null: *recent_date_filter + - name: TOKEN_ACCOUNT + description: "{{ doc('token_account') }}" + - name: DECIMAL + description: "{{ doc('decimal') }}" + data_tests: + - not_null: + config: + where: > + _inserted_timestamp >= current_date - 7 + AND event_type in ('initializeMint','initializeMint2') + - name: MINT_AMOUNT + description: "{{ doc('mint_amount') }}" + data_tests: + - not_null: + config: + where: > + _inserted_timestamp >= current_date - 7 + AND event_type in ('mintTo','mintToChecked') + - name: MINT_AUTHORITY + description: "{{ doc('mint_authority') }}" + - name: SIGNERS + description: "{{ doc('signers') }}" + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + data_tests: + - not_null + - name: MINT_ACTIONS_ID + description: '{{ doc("pk") }}' + data_tests: + - not_null: *recent_date_filter + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + data_tests: + - not_null: *recent_date_filter + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + data_tests: + - not_null: *recent_date_filter + - name: _INVOCATION_ID + description: '{{ doc("_invocation_id") }}' + data_tests: + - not_null: + name: test_silver__not_null_mint_actions_invocation_id + <<: *recent_date_filter \ No newline at end of file diff --git a/models/silver/labels/silver__mint_types.sql b/models/silver/labels/silver__mint_types.sql new file mode 100644 index 0000000..fbeca7f --- /dev/null +++ b/models/silver/labels/silver__mint_types.sql @@ -0,0 +1,241 @@ +{{ config( + materialized = 'incremental', + unique_key = ["mint_types_id"], + tags = ['scheduled_non_core'], +) }} + +{% if execute %} + {% if is_incremental() %} + {% set max_inserted_query %} + SELECT + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + {{ this }} + {% endset %} + {% set max_inserted_timestamp = run_query(max_inserted_query)[0][0] %} + {% endif %} +{% endif %} + +WITH initialization AS ( + SELECT + *, + CASE + WHEN DECIMAL = 0 THEN 'nft' + WHEN DECIMAL > 0 THEN 'token' + ELSE NULL + END AS mint_type + FROM + {{ ref('silver__mint_actions') }} + WHERE + event_type IN ( + 'initializeMint', + 'initializeMint2' + ) + AND succeeded + {% if is_incremental() %} + AND _inserted_timestamp >= '{{ max_inserted_timestamp }}' + {% endif %} +), +base_metaplex_events AS ( + SELECT + block_timestamp, + block_id, + tx_id, + INDEX, + NULL AS inner_index, + program_id, + instruction :accounts AS accounts, + ARRAY_SIZE(accounts) AS num_accounts + FROM + {{ ref('silver__events') }} + WHERE + program_id = 'metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s' + AND succeeded + {% if is_incremental() %} + AND _inserted_timestamp >= '{{ max_inserted_timestamp }}' + {% endif %} + UNION + SELECT + block_timestamp, + block_id, + tx_id, + instruction_index AS INDEX, + inner_index, + program_id, + instruction :accounts AS accounts, + ARRAY_SIZE(accounts) AS num_accounts + FROM + {{ ref('silver__events_inner') }} + WHERE + program_id = 'metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s' + AND succeeded + {% if is_incremental() %} + AND _inserted_timestamp >= '{{ max_inserted_timestamp }}' + {% endif %} +), +metaplex_mint_events AS ( + SELECT + *, + CASE + WHEN num_accounts = 9 + AND accounts [6] = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' + AND accounts [7] = '11111111111111111111111111111111' + AND accounts [8] = 'SysvarRent111111111111111111111111111111111' THEN 'Create Master Edition' + WHEN num_accounts = 8 + AND accounts [6] = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' + AND accounts [7] = '11111111111111111111111111111111' THEN 'Create Master Edition V3' + WHEN num_accounts = 11 + AND accounts [9] = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' + AND accounts [10] = '11111111111111111111111111111111' + AND accounts [11] = 'SysvarRent111111111111111111111111111111111' THEN 'Create Master Edition Deprecated' + WHEN num_accounts = 9 + AND accounts [6] = '11111111111111111111111111111111' + AND accounts [7] = 'Sysvar1nstructions1111111111111111111111111' + AND accounts [8] = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' THEN 'Create' + WHEN num_accounts IN ( + 13, + 14 + ) + AND accounts [11] :: STRING = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' + AND accounts [12] :: STRING = '11111111111111111111111111111111' THEN 'Edition' + WHEN num_accounts = 6 + AND accounts [5] = '11111111111111111111111111111111' THEN 'Create Metadata Account V3' + WHEN num_accounts = 7 + AND accounts [5] = '11111111111111111111111111111111' + AND accounts [6] = 'SysvarRent111111111111111111111111111111111' THEN 'Create Metadata Account' + END AS metaplex_event_type, + CASE + WHEN metaplex_event_type = 'Edition' THEN accounts [3] :: STRING + WHEN metaplex_event_type = 'Create' THEN accounts [2] :: STRING + WHEN metaplex_event_type IS NOT NULL THEN accounts [1] :: STRING + ELSE NULL + END AS mint + FROM + base_metaplex_events + WHERE + metaplex_event_type IS NOT NULL +), +ranked AS ( + SELECT + *, + ROW_NUMBER() over ( + PARTITION BY mint + ORDER BY + CASE + WHEN metaplex_event_type IN ( + 'Create Master Edition Deprecated', + 'Create Master Edition V3', + 'Create Master Edition' + ) THEN 1 + WHEN metaplex_event_type = 'Edition' THEN 2 + WHEN metaplex_event_type IN ( + 'Create Metadata account', + 'Create', + 'Create Metadata Account V3' + ) THEN 3 + ELSE 4 + END + ) AS rn + FROM + metaplex_mint_events +), +nonfungibles AS ( + SELECT + A.tx_id, + A.mint, + A.decimal, + A.mint_type, + CASE + WHEN b.metaplex_event_type = 'Edition' + AND A.decimal = 0 THEN 'Edition' + WHEN b.metaplex_event_type IN ( + 'Create Master Edition Deprecated', + 'Create Master Edition V3', + 'Create Master Edition' + ) + AND A.decimal = 0 THEN 'NonFungible' + WHEN b.metaplex_event_type IN ('Create') + AND accounts [1] <> 'metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s' + AND A.decimal = 0 THEN 'NonFungible' + END AS mint_standard_type, + A._inserted_timestamp + FROM + initialization A + LEFT JOIN ranked b + ON A.mint = b.mint + WHERE + b.rn = 1 + AND mint_standard_type IS NOT NULL +), +fungibles_and_others AS ( + SELECT + A.mint, + A.decimal, + A.mint_type, + CASE + WHEN b.metaplex_event_type IN ('Create') + AND accounts [1] = 'metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s' + AND A.decimal = 0 THEN 'FungibleAsset' + WHEN b.metaplex_event_type IN ( + 'Create Metadata Account', + 'Create', + 'Create Metadata Account V3' + ) + AND A.decimal = 0 THEN 'FungibleAsset' + WHEN b.metaplex_event_type IN ( + 'Create Metadata Account', + 'Create', + 'Create Metadata Account V3' + ) + AND A.decimal > 0 THEN 'Fungible' + END AS mint_standard_type, + A._inserted_timestamp + FROM + initialization A + LEFT JOIN ranked b + ON A.mint = b.mint + AND b.rn = 1 + WHERE + A.mint NOT IN ( + SELECT + mint + FROM + nonfungibles + ) + AND A.mint_type IS NOT NULL qualify(ROW_NUMBER() over (PARTITION BY A.mint + ORDER BY + A._inserted_timestamp DESC)) = 1 +), +prefinal AS ( + SELECT + mint, + DECIMAL, + mint_type, + mint_standard_type, + _inserted_timestamp + FROM + nonfungibles + UNION ALL + SELECT + mint, + DECIMAL, + mint_type, + mint_standard_type, + _inserted_timestamp + FROM + fungibles_and_others +) +SELECT + mint, + DECIMAL, + mint_type, + mint_standard_type, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['mint'] + ) }} AS mint_types_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + prefinal diff --git a/models/silver/labels/silver__mint_types.yml b/models/silver/labels/silver__mint_types.yml new file mode 100644 index 0000000..8b822c9 --- /dev/null +++ b/models/silver/labels/silver__mint_types.yml @@ -0,0 +1,43 @@ +version: 2 +models: + - name: silver__mint_types + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 + columns: + - name: MINT + description: "{{ doc('mint') }}" + data_tests: + - not_null: *recent_date_filter + - name: DECIMAL + description: "{{ doc('decimal') }}" + tests: + - not_null: *recent_date_filter + - name: MINT_TYPE + description: "{{ doc('mint_type') }}" + data_tests: + - not_null: *recent_date_filter + - name: MINT_STANDARD_TYPE + description: "{{ doc('mint_standard_type') }}" + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null + - name: MINT_TYPES_ID + description: '{{ doc("pk") }}' + data_tests: + - not_null: *recent_date_filter + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + data_tests: + - not_null: *recent_date_filter + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + data_tests: + - not_null: *recent_date_filter + - name: _INVOCATION_ID + description: '{{ doc("_invocation_id") }}' + data_tests: + - not_null: + name: test_silver__not_null_mint_types_invocation_id + <<: *recent_date_filter \ No newline at end of file diff --git a/models/silver/token/silver__token_burn_actions.sql b/models/silver/token/silver__token_burn_actions.sql new file mode 100644 index 0000000..b0ac58c --- /dev/null +++ b/models/silver/token/silver__token_burn_actions.sql @@ -0,0 +1,58 @@ +{{ config( + materialized = 'incremental', + unique_key = ["token_burn_actions_id"], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], + merge_exclude_columns = ["inserted_timestamp"], + tags = ['scheduled_non_core'] +) }} + +{% if execute %} + {% if is_incremental() %} + {% set max_inserted_query %} + SELECT + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + {{ this }} + {% endset %} + {% set max_inserted_timestamp = run_query(max_inserted_query)[0][0] %} + {% endif %} +{% endif %} + +WITH base_burn_actions AS ( + SELECT + * + FROM + {{ ref('silver__burn_actions') }} + {% if is_incremental() %} + WHERE _inserted_timestamp >= '{{ max_inserted_timestamp }}' + {% endif %} +) +SELECT + A.block_id, + A.block_timestamp, + A.tx_id, + A.succeeded, + A.index, + COALESCE(A.inner_index, -1) as inner_index, + A.event_type, + A.mint, + A.burn_amount, + A.burn_authority, + A.token_account, + A.signers, + b.decimal, + b.mint_standard_type, + A._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['A.tx_id', 'A.index', 'inner_index', 'A.mint'] + ) }} AS token_burn_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + base_burn_actions A + INNER JOIN {{ ref('silver__mint_types') }} b + ON A.mint = b.mint +WHERE + b.mint_type = 'token' diff --git a/models/silver/token/silver__token_burn_actions.yml b/models/silver/token/silver__token_burn_actions.yml new file mode 100644 index 0000000..208efc4 --- /dev/null +++ b/models/silver/token/silver__token_burn_actions.yml @@ -0,0 +1,89 @@ +version: 2 +models: + - name: silver__token_burn_actions + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - INDEX + - INNER_INDEX + - MINT + where: block_timestamp::date > current_date - 30 + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + data_tests: + - not_null: *recent_date_filter + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + data_tests: + - not_null: *recent_date_filter + - name: TX_ID + description: "{{ doc('tx_id') }}" + data_tests: + - not_null: *recent_date_filter + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + data_tests: + - not_null: *recent_date_filter + - name: INDEX + description: "{{ doc('event_index') }}" + data_tests: + - not_null: *recent_date_filter + - name: INNER_INDEX + description: "{{ doc('inner_instruction_index') }}" + - name: EVENT_TYPE + description: "{{ doc('event_type') }}" + data_tests: + - not_null: *recent_date_filter + - name: MINT + description: "{{ doc('mint') }}" + data_tests: + - not_null: *recent_date_filter + - name: BURN_AMOUNT + description: "{{ doc('burn_amount') }}" + data_tests: + - not_null: *recent_date_filter + - name: BURN_AUTHORITY + description: "{{ doc('burn_authority') }}" + data_tests: + - not_null: *recent_date_filter + - name: TOKEN_ACCOUNT + description: "{{ doc('token_account') }}" + - name: SIGNERS + description: Account address authorizing burn + - name: DECIMAL + description: "{{ doc('decimal') }}" + data_tests: + - not_null: *recent_date_filter + - name: MINT_STANDARD_TYPE + description: "{{ doc('mint_standard_type') }}" + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + data_tests: + - not_null + - name: TOKEN_BURN_ACTIONS_ID + description: '{{ doc("pk") }}' + data_tests: + - not_null: *recent_date_filter + - unique + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + data_tests: + - not_null: *recent_date_filter + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + data_tests: + - not_null: *recent_date_filter + - name: _INVOCATION_ID + description: '{{ doc("_invocation_id") }}' + data_tests: + - not_null: + name: test_silver__not_null_token_burn_actions_invocation_id + <<: *recent_date_filter \ No newline at end of file diff --git a/models/silver/token/silver__token_mint_actions.sql b/models/silver/token/silver__token_mint_actions.sql new file mode 100644 index 0000000..db680bf --- /dev/null +++ b/models/silver/token/silver__token_mint_actions.sql @@ -0,0 +1,58 @@ +{{ config( + materialized = 'incremental', + unique_key = ["token_mint_actions_id"], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], + merge_exclude_columns = ["inserted_timestamp"], + tags = ['scheduled_non_core'] +) }} + +{% if execute %} + {% if is_incremental() %} + {% set max_inserted_query %} + SELECT + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + {{ this }} + {% endset %} + {% set max_inserted_timestamp = run_query(max_inserted_query)[0][0] %} + {% endif %} +{% endif %} + +WITH base_mint_actions AS ( + SELECT + * + FROM + {{ ref('silver__mint_actions') }} + {% if is_incremental() %} + WHERE _inserted_timestamp >= '{{ max_inserted_timestamp }}' + {% endif %} +) +SELECT + A.block_id, + A.block_timestamp, + A.tx_id, + A.succeeded, + A.index, + COALESCE(A.inner_index, -1) as inner_index, + A.event_type, + A.mint, + A.mint_amount, + A.mint_authority, + A.token_account, + A.signers, + b.decimal, + b.mint_standard_type, + A._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['A.tx_id', 'A.index', 'inner_index', 'A.mint'] + ) }} AS token_mint_actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + base_mint_actions A + INNER JOIN {{ ref('silver__mint_types') }} b + ON A.mint = b.mint +WHERE + b.mint_type = 'token' diff --git a/models/silver/token/silver__token_mint_actions.yml b/models/silver/token/silver__token_mint_actions.yml new file mode 100644 index 0000000..167da39 --- /dev/null +++ b/models/silver/token/silver__token_mint_actions.yml @@ -0,0 +1,90 @@ +version: 2 +models: + - name: silver__token_mint_actions + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - INDEX + - INNER_INDEX + - MINT + where: block_timestamp::date > current_date - 30 + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + data_tests: + - not_null: *recent_date_filter + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + data_tests: + - not_null: *recent_date_filter + - name: TX_ID + description: "{{ doc('tx_id') }}" + data_tests: + - not_null: *recent_date_filter + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + data_tests: + - not_null: *recent_date_filter + - name: INDEX + description: "{{ doc('event_index') }}" + data_tests: + - not_null: *recent_date_filter + - name: INNER_INDEX + description: "{{ doc('inner_instruction_index') }}" + - name: EVENT_TYPE + description: "{{ doc('event_type') }}" + data_tests: + - not_null: *recent_date_filter + - name: MINT + description: "{{ doc('mint') }}" + data_tests: + - not_null: *recent_date_filter + - name: MINT_AMOUNT + description: "{{ doc('mint_amount') }}" + data_tests: + - not_null: + where: event_type in ('mintToChecked', 'mintTo') + - name: MINT_AUTHORITY + description: "{{ doc('mint_authority') }}" + data_tests: + - not_null: *recent_date_filter + - name: TOKEN_ACCOUNT + description: "{{ doc('token_account') }}" + - name: SIGNERS + description: "{{ doc('signers') }}" + - name: DECIMAL + description: "{{ doc('decimal') }}" + data_tests: + - not_null: *recent_date_filter + - name: MINT_STANDARD_TYPE + description: "{{ doc('mint_standard_type') }}" + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + data_tests: + - not_null + - name: TOKEN_MINT_ACTIONS_ID + description: '{{ doc("pk") }}' + data_tests: + - not_null: *recent_date_filter + - unique + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + data_tests: + - not_null: *recent_date_filter + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + data_tests: + - not_null: *recent_date_filter + - name: _INVOCATION_ID + description: '{{ doc("_invocation_id") }}' + data_tests: + - not_null: + name: test_silver__not_null_token_mint_actions_invocation_id + <<: *recent_date_filter \ No newline at end of file