From d9cabceeb33478c0248dcb0c6bee991c8d04b591 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 2 Jun 2025 13:59:29 -0600 Subject: [PATCH] added silver fungible token intermediate models, modified gold transfers, added docs and tests --- models/descriptions/block_number_first.md | 5 + models/descriptions/block_timestamp_first.md | 5 + models/descriptions/is_fungible.md | 5 + models/descriptions/metadata_address.md | 5 + models/descriptions/owner_address.md | 5 + models/descriptions/store_address.md | 5 + models/descriptions/version.md | 2 +- models/gold/core/core__fact_transfers.sql | 35 ++++ models/gold/core/gold_core.yml | 4 + .../core/silver__fungiblestore_metadata.sql | 42 +++++ .../core/silver__fungiblestore_owners.sql | 40 ++++ .../core/silver__transfers_fungible.sql | 171 ++++++++++++++++++ models/silver/core/silver_core.yml | 97 ++++++++++ 13 files changed, 420 insertions(+), 1 deletion(-) create mode 100644 models/descriptions/block_number_first.md create mode 100644 models/descriptions/block_timestamp_first.md create mode 100644 models/descriptions/is_fungible.md create mode 100644 models/descriptions/metadata_address.md create mode 100644 models/descriptions/owner_address.md create mode 100644 models/descriptions/store_address.md create mode 100644 models/silver/core/silver__fungiblestore_metadata.sql create mode 100644 models/silver/core/silver__fungiblestore_owners.sql create mode 100644 models/silver/core/silver__transfers_fungible.sql diff --git a/models/descriptions/block_number_first.md b/models/descriptions/block_number_first.md new file mode 100644 index 0000000..6b9a0a9 --- /dev/null +++ b/models/descriptions/block_number_first.md @@ -0,0 +1,5 @@ +{% docs block_number_first %} + +Block number of the first time the store metadata was posted to the chain + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_timestamp_first.md b/models/descriptions/block_timestamp_first.md new file mode 100644 index 0000000..339cd2d --- /dev/null +++ b/models/descriptions/block_timestamp_first.md @@ -0,0 +1,5 @@ +{% docs block_timestamp_first %} + +Block timestamp of the first time the store metadata was posted to the chain + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/is_fungible.md b/models/descriptions/is_fungible.md new file mode 100644 index 0000000..ab90cb9 --- /dev/null +++ b/models/descriptions/is_fungible.md @@ -0,0 +1,5 @@ +{% docs is_fungible %} + +Boolean indicating whether the transfer was conducted using the legacy coin transfer mechanism (a simpler, original method for transferring coins) or the fungible_asset module (a newer, more flexible system for managing fungible assets). + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/metadata_address.md b/models/descriptions/metadata_address.md new file mode 100644 index 0000000..bbf92dc --- /dev/null +++ b/models/descriptions/metadata_address.md @@ -0,0 +1,5 @@ +{% docs metadata_address %} + +Metadata address of the token in the store + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/owner_address.md b/models/descriptions/owner_address.md new file mode 100644 index 0000000..cd44f3e --- /dev/null +++ b/models/descriptions/owner_address.md @@ -0,0 +1,5 @@ +{% docs owner_address %} + +Address of the store owner + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/store_address.md b/models/descriptions/store_address.md new file mode 100644 index 0000000..dfa4a99 --- /dev/null +++ b/models/descriptions/store_address.md @@ -0,0 +1,5 @@ +{% docs store_address %} + +Address of the fungible store + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/version.md b/models/descriptions/version.md index ac9836e..39e6f83 100644 --- a/models/descriptions/version.md +++ b/models/descriptions/version.md @@ -1,5 +1,5 @@ {% docs version %} -The version also know as the height of the transactions that have been executed on the Aptos blockchain. The first transaction in the blockchain has a version of 0. The version number is incremented by 1 for each transaction that is executed on the blockchain. +The version also know as the height of the transactions that have been executed on the Movement blockchain. The first transaction in the blockchain has a version of 0. The version number is incremented by 1 for each transaction that is executed on the blockchain. {% enddocs %} \ No newline at end of file diff --git a/models/gold/core/core__fact_transfers.sql b/models/gold/core/core__fact_transfers.sql index c458977..40c7ba0 100644 --- a/models/gold/core/core__fact_transfers.sql +++ b/models/gold/core/core__fact_transfers.sql @@ -21,6 +21,8 @@ SELECT account_address, amount, token_address, + FALSE AS is_fungible, + NULL :: STRING AS store_address, {{ dbt_utils.generate_surrogate_key( ['tx_hash','event_index','version'] ) }} AS fact_transfers_id, @@ -41,3 +43,36 @@ AND modified_timestamp >= ( {{ this }} ) {% endif %} + +UNION ALL + +SELECT + block_number, + block_timestamp, + tx_hash, + version, + success, + event_index, + NULL AS creation_number, + transfer_event, + owner_address AS account_address, + amount, + metadata_address AS token_address, + TRUE AS is_fungible, + store_address, + transfers_fungible_id AS fact_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, +FROM + {{ ref('silver__transfers_fungible') }} +WHERE + amount <> 0 + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} \ No newline at end of file diff --git a/models/gold/core/gold_core.yml b/models/gold/core/gold_core.yml index 25088d8..16939e2 100644 --- a/models/gold/core/gold_core.yml +++ b/models/gold/core/gold_core.yml @@ -318,6 +318,10 @@ models: description: '{{ doc("amount") }}' - name: TOKEN_ADDRESS description: '{{ doc("token_address") }}' + - name: IS_FUNGIBLE + description: '{{ doc("is_fungible") }}' + - name: STORE_ADDRESS + description: '{{ doc("store_address") }}' - name: FACT_TRANSFERS_ID description: '{{ doc("pk") }}' - name: INSERTED_TIMESTAMP diff --git a/models/silver/core/silver__fungiblestore_metadata.sql b/models/silver/core/silver__fungiblestore_metadata.sql new file mode 100644 index 0000000..9d72b00 --- /dev/null +++ b/models/silver/core/silver__fungiblestore_metadata.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = 'incremental', + unique_key = ['store_address'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp","block_timestamp_first","block_number_first"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(store_address);", + tags = ['core'] +) }} + +-- depends_on: {{ ref('core__fact_changes') }} + +SELECT + block_timestamp AS block_timestamp_first, + block_number AS block_number_first, + address AS store_address, + change_data :metadata :inner :: STRING AS metadata_address, + {{ dbt_utils.generate_surrogate_key( + ['store_address'] + ) }} AS fungiblestore_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('core__fact_changes') }} +WHERE + change_module = 'fungible_asset' + AND change_resource = 'FungibleStore' + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} + +qualify ROW_NUMBER() over ( + PARTITION BY address + ORDER BY + block_number +) = 1 diff --git a/models/silver/core/silver__fungiblestore_owners.sql b/models/silver/core/silver__fungiblestore_owners.sql new file mode 100644 index 0000000..ba38ee5 --- /dev/null +++ b/models/silver/core/silver__fungiblestore_owners.sql @@ -0,0 +1,40 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_hash','change_index'], + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(store_address);", + tags = ['core'] +) }} + +-- depends_on: {{ ref('core__fact_changes') }} + +SELECT + block_timestamp, + block_number, + version, + tx_hash, + change_index, + address AS store_address, + change_data :owner :: STRING AS owner_address, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','change_index'] + ) }} AS fungiblestore_owners_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('core__fact_changes') }} +WHERE + change_address = '0x1' + AND change_module = 'object' + AND change_resource = 'ObjectCore' + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} diff --git a/models/silver/core/silver__transfers_fungible.sql b/models/silver/core/silver__transfers_fungible.sql new file mode 100644 index 0000000..8aad4b4 --- /dev/null +++ b/models/silver/core/silver__transfers_fungible.sql @@ -0,0 +1,171 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_hash','event_index','block_timestamp::DATE'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], + tags = ['core','full_test'] +) }} +-- depends_on: {{ ref('core__fact_events') }} +-- depends_on: {{ ref('silver__fungiblestore_owners') }} + +{% if execute %} + {% set base_query %} + CREATE + OR REPLACE temporary TABLE silver.transfers_fungible__intermediate_tmp AS + + SELECT + block_number, + version, + success, + block_timestamp, + tx_hash, + event_index, + event_resource, + event_data :amount :: bigint AS amount, + event_data :store :: STRING AS store_address + FROM + {{ ref('core__fact_events') }} + WHERE + event_module = 'fungible_asset' + AND event_resource IN ( + 'WithdrawEvent', + 'DepositEvent', + 'Withdraw', + 'Deposit' + ) + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} + +{% endset %} +{% do run_query(base_query) %} +{% set owner_query %} +CREATE +OR REPLACE temporary TABLE silver.transfers_fungible_store_owners__intermediate_tmp AS +SELECT + version, + block_timestamp, + store_address, + owner_address, + COUNT( + DISTINCT owner_address + ) over( + PARTITION BY store_address + ) owner_cnt +FROM + {{ ref('silver__fungiblestore_owners') }} +WHERE + store_address IN ( + SELECT + store_address + FROM + silver.transfers_fungible__intermediate_tmp + ) +ORDER BY + block_timestamp {% endset %} + {% do run_query(owner_query) %} + {% set owner_query_single %} + CREATE + OR REPLACE temporary TABLE silver.transfers_fungible_store_owners_single__intermediate_tmp AS +SELECT + DISTINCT store_address, + owner_address +FROM + silver.transfers_fungible_store_owners__intermediate_tmp +WHERE + owner_cnt = 1 {% endset %} + {% do run_query(owner_query_single) %} + {% set owner_query_many %} + CREATE + OR REPLACE temporary TABLE silver.transfers_fungible_store_owners_many__intermediate_tmp AS WITH base AS ( + SELECT + store_address, + owner_address, + block_timestamp, + conditional_change_event(owner_address) over ( + PARTITION BY store_address + ORDER BY + block_timestamp + ) AS change_event, + ROW_NUMBER() over ( + PARTITION BY store_address + ORDER BY + block_timestamp + ) AS rn + FROM + silver.transfers_fungible_store_owners__intermediate_tmp + WHERE + owner_cnt > 1 + ) +SELECT + store_address, + owner_address, + block_timestamp +FROM + base qualify ROW_NUMBER() over( + PARTITION BY store_address, + change_event + ORDER BY + rn + ) = 1; +{% endset %} + {% do run_query(owner_query_many) %} +{% endif %} + +WITH md AS ( + SELECT + store_address, + metadata_address + FROM + {{ ref('silver__fungiblestore_metadata') }} +) +SELECT + e.block_number, + e.block_timestamp, + e.tx_hash, + e.version, + e.success, + e.event_index, + CASE + WHEN event_resource IN ( + 'WithdrawEvent', + 'Withdraw' + ) THEN 'WithdrawEvent' + WHEN event_resource IN ( + 'DepositEvent', + 'Deposit' + ) THEN 'DepositEvent' + END AS transfer_event, + e.store_address, + COALESCE( + os.owner_address, + om.owner_address + ) AS owner_address, + m.metadata_address, + e.amount, + os.owner_address os, + om.owner_address om, + om.block_timestamp om_block_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['e.tx_hash','e.event_index'] + ) }} AS transfers_fungible_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + silver.transfers_fungible__intermediate_tmp e asof + JOIN silver.transfers_fungible_store_owners_many__intermediate_tmp om match_condition( + e.block_timestamp >= om.block_timestamp + ) + ON e.store_address = om.store_address + LEFT JOIN silver.transfers_fungible_store_owners_single__intermediate_tmp os + ON e.store_address = os.store_address + LEFT JOIN md m + ON e.store_address = m.store_address diff --git a/models/silver/core/silver_core.yml b/models/silver/core/silver_core.yml index fd7e929..ff98ba7 100644 --- a/models/silver/core/silver_core.yml +++ b/models/silver/core/silver_core.yml @@ -230,6 +230,61 @@ models: - name: _invocation_id data_type: VARCHAR + - name: silver__fungiblestore_metadata + config: + contract: + enforced: true + columns: + - name: block_timestamp_first + data_type: TIMESTAMP_NTZ + - name: block_number_first + data_type: NUMBER + - name: store_address + data_type: VARCHAR + - name: metadata_address + data_type: VARCHAR + tests: + - not_null: + tags: ['test_quality'] + - name: fungiblestore_metadata_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__fungiblestore_owners + config: + contract: + enforced: true + columns: + - name: block_timestamp + data_type: TIMESTAMP_NTZ + - name: block_number + data_type: NUMBER + - name: version + data_type: NUMBER + - name: tx_hash + data_type: VARCHAR + - name: change_index + data_type: NUMBER + - name: store_address + data_type: VARCHAR + - name: owner_address + data_type: VARCHAR + tests: + - not_null: + tags: ['test_quality'] + - name: fungiblestore_owners_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR - name: silver__transactions config: contract: @@ -387,5 +442,47 @@ models: data_type: TIMESTAMP_NTZ - name: modified_timestamp data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__transfers_fungible + config: + contract: + enforced: true + columns: + - name: block_number + data_type: NUMBER + - name: block_timestamp + data_type: TIMESTAMP_NTZ + - name: tx_hash + data_type: VARCHAR + - name: version + data_type: NUMBER + - name: success + data_type: BOOLEAN + - name: event_index + data_type: NUMBER + - name: transfer_event + data_type: VARCHAR + - name: store_address + data_type: VARCHAR + - name: owner_address + data_type: VARCHAR + - name: metadata_address + data_type: VARCHAR + - name: amount + data_type: NUMBER + - name: os + data_type: VARCHAR + - name: om + data_type: VARCHAR + - name: om_block_timestamp + data_type: TIMESTAMP_NTZ + - name: transfers_fungible_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ - name: _invocation_id data_type: VARCHAR \ No newline at end of file