From d269fbbcdb74fa88c6e733bc81fd044c9af90468 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Fri, 30 May 2025 10:19:02 -0600 Subject: [PATCH] add silver and gold transfers + docs --- models/descriptions/__overview__.md | 4 +- .../tables/core__fact_transfers.md | 5 + models/gold/core/core__fact_transfers.sql | 40 +++++ models/gold/core/gold_core.yml | 39 +++++ models/silver/core/silver__transfers.sql | 148 ++++++++++++++++++ models/silver/core/silver_core.yml | 57 +++++++ 6 files changed, 291 insertions(+), 2 deletions(-) create mode 100644 models/descriptions/tables/core__fact_transfers.md create mode 100644 models/gold/core/core__fact_transfers.sql create mode 100644 models/silver/core/silver__transfers.sql diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index f8b4700..818a2f8 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -24,6 +24,7 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_transactions](#!/model/model.movement_models.core__fact_transactions) - [fact_transactions_block_metadata](#!/model/model.movement_models.core__fact_transactions_block_metadata) - [fact_transactions_state_checkpoint](#!/model/model.movement_models.core__fact_transactions_state_checkpoint) +- [fact_transfers](#!/model/model.movement_models.core__fact_transfers) ### Price Tables (`movement`.`PRICE`.``) @@ -73,8 +74,7 @@ Note that you can also right-click on models to interactively filter and explore ### **More information** - [Flipside](https://flipsidecrypto.xyz/) -- [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) -- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) +- [Community](https://docs.flipsidecrypto.xyz/welcome-to-flipside/flipside-community-overview) - [Github](https://github.com/FlipsideCrypto/movement-models) - [What is dbt?](https://docs.getdbt.com/docs/introduction) diff --git a/models/descriptions/tables/core__fact_transfers.md b/models/descriptions/tables/core__fact_transfers.md new file mode 100644 index 0000000..ba3b931 --- /dev/null +++ b/models/descriptions/tables/core__fact_transfers.md @@ -0,0 +1,5 @@ +{% docs core__fact_transactions %} + +This table contains Deposit and Withdraw events on the Movement blockchain. Note: transfers with a 0 amount are excluded. + +{% enddocs %} diff --git a/models/gold/core/core__fact_transfers.sql b/models/gold/core/core__fact_transfers.sql new file mode 100644 index 0000000..dae0e2c --- /dev/null +++ b/models/gold/core/core__fact_transfers.sql @@ -0,0 +1,40 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_hash','event_index','block_timestamp::DATE'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash, version, account_address,token_address);", + tags = ['core'] +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + version, + success, + event_index, + creation_number, + transfer_event, + account_address, + amount, + token_address, + transfers_id AS fact_transfers_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref( + 'silver__transfers' + ) }} +WHERE + amount <> 0 + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} diff --git a/models/gold/core/gold_core.yml b/models/gold/core/gold_core.yml index 79f6206..25088d8 100644 --- a/models/gold/core/gold_core.yml +++ b/models/gold/core/gold_core.yml @@ -285,3 +285,42 @@ models: description: '{{ doc("inserted_timestamp") }}' - name: MODIFIED_TIMESTAMP description: '{{ doc("modified_timestamp") }}' + + - name: core__fact_transfers + description: '{{ doc("core__fact_transfers") }}' + tests: + - dbt_utils.recency: + datepart: hour + field: MODIFIED_TIMESTAMP + interval: 3 + severity: error + tags: ['test_recency'] + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: VERSION + description: '{{ doc("version") }}' + - name: SUCCESS + description: '{{ doc("success") }}' + - name: EVENT_INDEX + description: '{{ doc("event_index") }}' + - name: CREATION_NUMBER + description: '{{ doc("creation_number") }}' + - name: TRANSFER_EVENT + description: '{{ doc("transfer_event") }}' + - name: ACCOUNT_ADDRESS + description: '{{ doc("address_event") }}' + - name: AMOUNT + description: '{{ doc("amount") }}' + - name: TOKEN_ADDRESS + description: '{{ doc("token_address") }}' + - name: FACT_TRANSFERS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql new file mode 100644 index 0000000..a688a22 --- /dev/null +++ b/models/silver/core/silver__transfers.sql @@ -0,0 +1,148 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_hash','event_index','block_timestamp::DATE'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], + tags = ['core'] +) }} + +WITH events AS ( + + SELECT + block_number, + version, + success, + block_timestamp, + block_timestamp :: DATE AS block_date, + tx_hash, + event_index, + event_resource, + event_data :amount :: bigint AS amount, + account_address, + creation_number, + modified_timestamp + FROM + {{ ref( + 'core__fact_events' + ) }} + WHERE + event_module = 'coin' + AND event_resource IN ( + 'WithdrawEvent', + 'DepositEvent' + ) + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +changes AS ( + SELECT + block_timestamp :: DATE AS block_date, + tx_hash, + change_index, + change_data, + change_data :deposit_events :guid :id :creation_num :: INT AS creation_number_deposit, + change_data :withdraw_events :guid :id :creation_num :: INT AS creation_number_withdraw, + address, + change_resource AS token_address + FROM + {{ ref( + 'core__fact_changes' + ) }} + WHERE + change_module = 'coin' + AND ( + creation_number_deposit IS NOT NULL + OR creation_number_withdraw IS NOT NULL + ) + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +changes_dep AS ( + SELECT + block_date, + tx_hash, + address, + creation_number_deposit AS creation_number, + token_address + FROM + changes + WHERE + creation_number_deposit IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY tx_hash, creation_number_deposit, address + ORDER BY + change_index DESC) = 1) +), +changes_wth AS ( + SELECT + block_date, + tx_hash, + address, + creation_number_withdraw AS creation_number, + token_address + FROM + changes + WHERE + creation_number_withdraw IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY tx_hash, creation_number_withdraw, address + ORDER BY + change_index DESC) = 1) +) +SELECT + e.block_number, + e.block_timestamp, + e.tx_hash, + e.version, + e.success, + e.event_index, + e.creation_number, + e.event_resource AS transfer_event, + e.account_address, + e.amount, + REPLACE( + REPLACE( + COALESCE( + dep.token_address, + wth.token_address + ), + 'CoinStore<' + ), + '>' + ) AS token_address, + {{ dbt_utils.generate_surrogate_key( + ['e.tx_hash','e.event_index'] + ) }} AS transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + events e + LEFT JOIN changes_dep dep + ON e.block_date = dep.block_date + AND e.tx_hash = dep.tx_hash + AND e.creation_number = dep.creation_number + AND e.account_address = dep.address + AND e.event_resource = 'DepositEvent' + LEFT JOIN changes_wth wth + ON e.block_date = wth.block_date + AND e.tx_hash = wth.tx_hash + AND e.creation_number = wth.creation_number + AND e.account_address = wth.address + AND e.event_resource = 'WithdrawEvent' +WHERE + COALESCE( + dep.token_address, + wth.token_address + ) IS NOT NULL diff --git a/models/silver/core/silver_core.yml b/models/silver/core/silver_core.yml index f897e0e..44afb1e 100644 --- a/models/silver/core/silver_core.yml +++ b/models/silver/core/silver_core.yml @@ -330,5 +330,62 @@ models: data_type: TIMESTAMP_NTZ - name: modified_timestamp data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__transfers + config: + contract: + enforced: true + columns: + - name: block_number + data_type: NUMBER + - name: block_timestamp + data_type: TIMESTAMP_NTZ + - name: tx_hash + data_type: VARCHAR + - name: version + data_type: NUMBER + - name: success + data_type: BOOLEAN + - name: event_index + data_type: NUMBER + tests: + - not_null: + tags: ['test_quality'] + - name: creation_number + data_type: NUMBER + tests: + - not_null: + tags: ['test_quality'] + - name: transfer_event + data_type: VARCHAR + tests: + - not_null: + tags: ['test_quality'] + - name: account_address + data_type: VARCHAR + tests: + - not_null: + tags: ['test_quality'] + - name: amount + data_type: NUMBER + tests: + - not_null: + tags: ['test_quality'] + - dbt_utils.expression_is_true: + expression: ">= 0" + tags: ['test_quality'] + - name: token_address + data_type: VARCHAR + tests: + - not_null: + tags: ['test_quality'] + - name: fact_transfers_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ - name: _invocation_id data_type: VARCHAR \ No newline at end of file