diff --git a/models/descriptions/columns.md b/models/descriptions/columns.md index c563146..7027898 100644 --- a/models/descriptions/columns.md +++ b/models/descriptions/columns.md @@ -78,6 +78,10 @@ Fully qualified Move type identifier for coins/tokens. Format: {package}::{modul Token quantity in the smallest unit (MIST for SUI). Integer value; 1 SUI = 1,000,000,000 MIST. Used for precise financial calculations, balance tracking, and token flow analysis. Example: 1000000000. {% enddocs %} +{% docs amount_raw %} +Token quantity in the smallest unit (MIST for SUI). Integer value; 1 SUI = 1,000,000,000 MIST. Used for precise financial calculations, balance tracking, and token flow analysis. Example: 1000000000. +{% enddocs %} + {% docs owner %} Ownership information for objects or balances. Enum: Address (user), Shared (requires consensus), Immutable (cannot change). Determines accessibility and is critical for wallet analytics, ownership distribution, and access control studies. Example: 'Address'. {% enddocs %} @@ -433,4 +437,13 @@ Lowest price of the recorded hour in USD Closing price of the recorded hour in USD +{% enddocs %} + + +{% docs amount_usd %} + +USD value of the amount at transaction time. + +Example: 1000.50 + {% enddocs %} \ No newline at end of file diff --git a/models/gold/core/core__ez_transfers.sql b/models/gold/core/core__ez_transfers.sql index 3430262..9fed55b 100644 --- a/models/gold/core/core__ez_transfers.sql +++ b/models/gold/core/core__ez_transfers.sql @@ -4,10 +4,9 @@ incremental_strategy = 'merge', incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], merge_exclude_columns = ["inserted_timestamp"], - cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_digest, sender, receiver, coin_type, symbol);", - tags = ['core'], - enabled = false + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_digest, tx_sender, sender, receiver, coin_type, symbol);", + tags = ['core'] ) }} SELECT @@ -16,27 +15,58 @@ SELECT tx_digest, balance_change_index, tx_succeeded, + tx_sender, sender, receiver, ft.coin_type, - symbol, - amount, - ROUND(NULLIFZERO(DIV0NULL(amount, POWER(10, dt.decimals))), 3) as amount_normalized, + COALESCE( + dt.symbol, + eph.symbol + ) AS symbol, + amount_raw, + CASE + WHEN COALESCE( + dt.decimals, + 0 + ) <> 0 THEN amount_raw / power( + 10, + dt.decimals + ) + END AS amount, + ROUND( + amount * eph.price, + 2 + ) AS amount_usd, + COALESCE( + eph.token_is_verified, + FALSE + ) AS token_is_verified, {{ dbt_utils.generate_surrogate_key( ['tx_digest','balance_change_index'] ) }} AS ez_transfers_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp FROM - {{ ref('core__fact_transfers') }} ft -LEFT JOIN - {{ ref('core__dim_tokens') }} dt + {{ ref('silver__transfers') }} + ft + LEFT JOIN {{ ref('core__dim_tokens') }} + dt ON ft.coin_type = dt.coin_type + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + eph + ON ft.coin_type = eph.token_address + AND DATE_TRUNC( + 'HOUR', + ft.block_timestamp + ) = eph.hour +WHERE + amount IS NOT NULL + {% if is_incremental() %} -WHERE ft.modified_timestamp >= ( +AND ft.modified_timestamp >= ( SELECT MAX(modified_timestamp) FROM {{ this }} ) -{% endif %} \ No newline at end of file +{% endif %} diff --git a/models/gold/core/core__fact_transfers.sql b/models/gold/core/core__fact_transfers.sql deleted file mode 100644 index 3353429..0000000 --- a/models/gold/core/core__fact_transfers.sql +++ /dev/null @@ -1,42 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = ['tx_digest','balance_change_index'], - incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], - merge_exclude_columns = ["inserted_timestamp"], - cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_digest, sender, receiver, coin_type);", - tags = ['core'], - enabled = false -) }} - -SELECT - checkpoint_number, - block_timestamp, - tx_digest, - tx_succeeded, - sender, - receiver, - balance_change_index, - coin_type, - amount, - {{ dbt_utils.generate_surrogate_key( - ['tx_digest','balance_change_index'] - ) }} AS fact_transfers_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM - {{ ref( - 'silver__transfers' - ) }} -WHERE - amount is not null - -{% if is_incremental() %} -AND modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} -) -{% endif %} \ No newline at end of file diff --git a/models/gold/core/gold_core.yml b/models/gold/core/gold_core.yml index 20fea31..1099c95 100644 --- a/models/gold/core/gold_core.yml +++ b/models/gold/core/gold_core.yml @@ -690,80 +690,7 @@ models: - name: MODIFIED_TIMESTAMP description: "Timestamp when this record was last modified." - - name: core__fact_transfers - description: "{{ doc('core__fact_transfers') }}" - config: - contract: - enforced: true - columns: - - name: CHECKPOINT_NUMBER - description: "{{ doc('checkpoint_number') }}" - data_type: NUMBER - tests: - - not_null - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - data_type: TIMESTAMP_NTZ - tests: - - not_null - - name: TX_DIGEST - description: "{{ doc('tx_digest') }}" - data_type: VARCHAR - tests: - - not_null - - name: TX_SUCCEEDED - description: "{{ doc('tx_succeeded') }}" - data_type: BOOLEAN - tests: - - not_null - - name: SENDER - description: "{{ doc('sender') }}" - data_type: VARCHAR - tests: - - not_null - - name: RECEIVER - description: "{{ doc('receiver') }}" - data_type: VARCHAR - tests: - - not_null - - name: BALANCE_CHANGE_INDEX - description: "{{ doc('balance_change_index') }}" - data_type: NUMBER - tests: - - not_null - - name: COIN_TYPE - description: "{{ doc('coin_type') }}" - data_type: VARCHAR - tests: - - not_null - - name: AMOUNT - description: "{{ doc('amount') }}" - data_type: NUMBER - tests: - - not_null - - name: FACT_TRANSFERS_ID - description: "{{ doc('fact_transfers_id') }}" - data_type: VARCHAR - tests: - - not_null - - unique - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp') }}" - data_type: TIMESTAMP_NTZ - tests: - - not_null - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp') }}" - data_type: TIMESTAMP_NTZ - tests: - - not_null - tests: - - dbt_utils.recency: - datepart: hour - field: block_timestamp - interval: 12 - severity: error - tags: ['test_recency'] + - name: core__ez_transfers description: "{{ doc('core__ez_transfers') }}" @@ -796,6 +723,11 @@ models: data_type: BOOLEAN tests: - not_null + - name: TX_SENDER + description: "{{ doc('tx_sender') }}" + data_type: VARCHAR + tests: + - not_null - name: SENDER description: "{{ doc('sender') }}" data_type: VARCHAR @@ -814,14 +746,20 @@ models: - name: SYMBOL description: "{{ doc('symbol') }}" data_type: VARCHAR - - name: AMOUNT - description: "{{ doc('amount') }}" + - name: AMOUNT_RAW + description: "{{ doc('amount_raw') }}" data_type: NUMBER tests: - not_null - - name: AMOUNT_NORMALIZED + - name: AMOUNT description: "{{ doc('amount_normalized') }}" data_type: FLOAT + - name: AMOUNT_USD + description: "{{ doc('amount_usd') }}" + data_type: FLOAT + - name: TOKEN_IS_VERIFIED + description: "{{ doc('token_is_verified') }}" + data_type: BOOLEAN - name: EZ_TRANSFERS_ID description: "{{ doc('ez_transfers_id') }}" data_type: VARCHAR diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index 1b6d47b..5b904fe 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -21,6 +21,11 @@ allowed_tx AS ( {% if is_incremental() %} AND modified_timestamp >= (SELECT COALESCE(MAX(modified_timestamp),'1970-01-01') FROM {{ this }}) {% endif %} + qualify ROW_NUMBER() over ( + PARTITION BY tx_digest + ORDER BY + modified_timestamp DESC + ) = 1 ), filtered as ( SELECT @@ -28,6 +33,7 @@ filtered as ( fbc.block_timestamp, fbc.tx_digest, fbc.tx_succeeded, + fbc.tx_sender, case when fbc.amount < 0 and fbc.address_owner IS NOT NULL @@ -51,16 +57,17 @@ filtered as ( AND fbc.modified_timestamp >= (SELECT COALESCE(MAX(modified_timestamp),'1970-01-01') FROM {{ this }}) {% endif %} ) -SELECT DISTINCT +SELECT checkpoint_number, block_timestamp, tx_digest, tx_succeeded, + tx_sender, sender, receiver, balance_change_index, coin_type, - amount, + amount AS amount_raw, {{ dbt_utils.generate_surrogate_key(['tx_digest','balance_change_index']) }} AS transfers_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/core/silver_core.yml b/models/silver/core/silver_core.yml index deb56e7..b8e374c 100644 --- a/models/silver/core/silver_core.yml +++ b/models/silver/core/silver_core.yml @@ -91,6 +91,11 @@ models: data_type: BOOLEAN tests: - not_null + - name: TX_SENDER + description: "{{ doc('tx_sender') }}" + data_type: VARCHAR + tests: + - not_null - name: SENDER description: "{{ doc('sender') }}" data_type: VARCHAR @@ -111,13 +116,16 @@ models: data_type: VARCHAR tests: - not_null - - name: AMOUNT + - name: AMOUNT_RAW description: "{{ doc('amount') }}" data_type: NUMBER tests: - not_null - name: TRANSFERS_ID data_type: VARCHAR + tests: + - not_null + - unique - name: INSERTED_TIMESTAMP description: "{{ doc('inserted_timestamp') }}" data_type: TIMESTAMP_NTZ