Merge pull request #13 from FlipsideCrypto/EZ-transfers

EZ transfers
2026-02-06 11:06:56 +00:00 · 2025-07-31 15:04:28 -04:00 · 2025-07-31 15:04:28 -04:00 · f7005e6326
commit f7005e6326
parent c17a12f5f4 0cfc3393a2
6 changed files with 88 additions and 134 deletions
--- a/models/descriptions/columns.md
+++ b/models/descriptions/columns.md
@ -78,6 +78,10 @@ Fully qualified Move type identifier for coins/tokens. Format: {package}::{modul
 Token quantity in the smallest unit (MIST for SUI). Integer value; 1 SUI = 1,000,000,000 MIST. Used for precise financial calculations, balance tracking, and token flow analysis. Example: 1000000000.
 {% enddocs %}

+{% docs amount_raw %}
+Token quantity in the smallest unit (MIST for SUI). Integer value; 1 SUI = 1,000,000,000 MIST. Used for precise financial calculations, balance tracking, and token flow analysis. Example: 1000000000.
+{% enddocs %}
+
 {% docs owner %}
 Ownership information for objects or balances. Enum: Address (user), Shared (requires consensus), Immutable (cannot change). Determines accessibility and is critical for wallet analytics, ownership distribution, and access control studies. Example: 'Address'.
 {% enddocs %}
@ -433,4 +437,13 @@ Lowest price of the recorded hour in USD

 Closing price of the recorded hour in USD

+{% enddocs %}
+
+
+{% docs amount_usd %}
+
+USD value of the amount at transaction time.
+
+Example: 1000.50
+
 {% enddocs %}
--- a/models/gold/core/core__ez_transfers.sql
+++ b/models/gold/core/core__ez_transfers.sql
@ -4,10 +4,9 @@
    incremental_strategy = 'merge',
    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
    merge_exclude_columns = ["inserted_timestamp"],
-    cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
-    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_digest, sender, receiver, coin_type, symbol);",
-    tags = ['core'],
-    enabled = false
+    cluster_by = ['block_timestamp::DATE'],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_digest, tx_sender, sender, receiver, coin_type, symbol);",
+    tags = ['core']
 ) }}

 SELECT
@ -16,27 +15,58 @@ SELECT
    tx_digest,
    balance_change_index,
    tx_succeeded,
+    tx_sender,
    sender,
    receiver,
    ft.coin_type,
-    symbol,
-    amount,
-    ROUND(NULLIFZERO(DIV0NULL(amount, POWER(10, dt.decimals))), 3) as amount_normalized,
+    COALESCE(
+        dt.symbol,
+        eph.symbol
+    ) AS symbol,
+    amount_raw,
+    CASE
+        WHEN COALESCE(
+            dt.decimals,
+            0
+        ) <> 0 THEN amount_raw / power(
+            10,
+            dt.decimals
+        )
+    END AS amount,
+    ROUND(
+        amount * eph.price,
+        2
+    ) AS amount_usd,
+    COALESCE(
+        eph.token_is_verified,
+        FALSE
+    ) AS token_is_verified,
    {{ dbt_utils.generate_surrogate_key(
        ['tx_digest','balance_change_index']
    ) }} AS ez_transfers_id,
    SYSDATE() AS inserted_timestamp,
    SYSDATE() AS modified_timestamp
 FROM
-    {{ ref('core__fact_transfers') }} ft
-LEFT JOIN
-    {{ ref('core__dim_tokens') }} dt 
+    {{ ref('silver__transfers') }}
+    ft
+    LEFT JOIN {{ ref('core__dim_tokens') }}
+    dt
    ON ft.coin_type = dt.coin_type
+    LEFT JOIN {{ ref('price__ez_prices_hourly') }}
+    eph
+    ON ft.coin_type = eph.token_address
+    AND DATE_TRUNC(
+        'HOUR',
+        ft.block_timestamp
+    ) = eph.hour
+WHERE
+    amount IS NOT NULL
+
 {% if is_incremental() %}
-WHERE ft.modified_timestamp >= (
+AND ft.modified_timestamp >= (
    SELECT
        MAX(modified_timestamp)
    FROM
        {{ this }}
 )
-{% endif %}
+{% endif %}
--- a/models/gold/core/core__fact_transfers.sql
+++ b/models/gold/core/core__fact_transfers.sql
@ -1,42 +0,0 @@
-{{ config(
-    materialized = 'incremental',
-    unique_key = ['tx_digest','balance_change_index'],
-    incremental_strategy = 'merge',
-    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
-    merge_exclude_columns = ["inserted_timestamp"],
-    cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
-    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_digest, sender, receiver, coin_type);",
-    tags = ['core'],
-    enabled = false
-) }}
-
-SELECT
-    checkpoint_number,
-    block_timestamp,
-    tx_digest,
-    tx_succeeded,
-    sender,
-    receiver,
-    balance_change_index,
-    coin_type,
-    amount,
-    {{ dbt_utils.generate_surrogate_key(
-        ['tx_digest','balance_change_index']
-    ) }} AS fact_transfers_id,
-    SYSDATE() AS inserted_timestamp,
-    SYSDATE() AS modified_timestamp
-FROM
-    {{ ref(
-        'silver__transfers'
-    ) }}
-WHERE
-    amount is not null
-
-{% if is_incremental() %}
-AND modified_timestamp >= (
-    SELECT
-        MAX(modified_timestamp)
-    FROM
-        {{ this }}
-)
-{% endif %}
--- a/models/gold/core/gold_core.yml
+++ b/models/gold/core/gold_core.yml
@ -690,80 +690,7 @@ models:
      - name: MODIFIED_TIMESTAMP
        description: "Timestamp when this record was last modified."

-  - name: core__fact_transfers
-    description: "{{ doc('core__fact_transfers') }}"
-    config:
-      contract:
-        enforced: true
-    columns:
-      - name: CHECKPOINT_NUMBER
-        description: "{{ doc('checkpoint_number') }}"
-        data_type: NUMBER
-        tests:
-          - not_null
-      - name: BLOCK_TIMESTAMP
-        description: "{{ doc('block_timestamp') }}"
-        data_type: TIMESTAMP_NTZ
-        tests:
-          - not_null
-      - name: TX_DIGEST
-        description: "{{ doc('tx_digest') }}"
-        data_type: VARCHAR
-        tests:
-          - not_null
-      - name: TX_SUCCEEDED
-        description: "{{ doc('tx_succeeded') }}"
-        data_type: BOOLEAN
-        tests:
-          - not_null
-      - name: SENDER
-        description: "{{ doc('sender') }}"
-        data_type: VARCHAR
-        tests:
-          - not_null
-      - name: RECEIVER
-        description: "{{ doc('receiver') }}"
-        data_type: VARCHAR
-        tests:
-          - not_null
-      - name: BALANCE_CHANGE_INDEX
-        description: "{{ doc('balance_change_index') }}"
-        data_type: NUMBER
-        tests:
-          - not_null
-      - name: COIN_TYPE
-        description: "{{ doc('coin_type') }}"
-        data_type: VARCHAR
-        tests:
-          - not_null
-      - name: AMOUNT
-        description: "{{ doc('amount') }}"
-        data_type: NUMBER
-        tests:
-          - not_null
-      - name: FACT_TRANSFERS_ID
-        description: "{{ doc('fact_transfers_id') }}"
-        data_type: VARCHAR
-        tests:
-          - not_null
-          - unique
-      - name: INSERTED_TIMESTAMP
-        description: "{{ doc('inserted_timestamp') }}"
-        data_type: TIMESTAMP_NTZ
-        tests:
-          - not_null
-      - name: MODIFIED_TIMESTAMP
-        description: "{{ doc('modified_timestamp') }}"
-        data_type: TIMESTAMP_NTZ
-        tests:
-          - not_null
-    tests:
-      - dbt_utils.recency:
-          datepart: hour
-          field: block_timestamp
-          interval: 12
-          severity: error
-          tags: ['test_recency']
+  

  - name: core__ez_transfers
    description: "{{ doc('core__ez_transfers') }}"
@ -796,6 +723,11 @@ models:
        data_type: BOOLEAN
        tests:
          - not_null
+      - name: TX_SENDER
+        description: "{{ doc('tx_sender') }}"
+        data_type: VARCHAR
+        tests:
+          - not_null
      - name: SENDER
        description: "{{ doc('sender') }}"
        data_type: VARCHAR
@ -814,14 +746,20 @@ models:
      - name: SYMBOL
        description: "{{ doc('symbol') }}"
        data_type: VARCHAR
-      - name: AMOUNT
-        description: "{{ doc('amount') }}"
+      - name: AMOUNT_RAW
+        description: "{{ doc('amount_raw') }}"
        data_type: NUMBER
        tests:
          - not_null
-      - name: AMOUNT_NORMALIZED
+      - name: AMOUNT
        description: "{{ doc('amount_normalized') }}"
        data_type: FLOAT
+      - name: AMOUNT_USD
+        description: "{{ doc('amount_usd') }}"
+        data_type: FLOAT
+      - name: TOKEN_IS_VERIFIED
+        description: "{{ doc('token_is_verified') }}"
+        data_type: BOOLEAN  
      - name: EZ_TRANSFERS_ID
        description: "{{ doc('ez_transfers_id') }}"
        data_type: VARCHAR
--- a/models/silver/core/silver__transfers.sql
+++ b/models/silver/core/silver__transfers.sql
@ -21,6 +21,11 @@ allowed_tx AS (
    {% if is_incremental() %}
        AND modified_timestamp >= (SELECT COALESCE(MAX(modified_timestamp),'1970-01-01') FROM {{ this }})
    {% endif %}
+     qualify ROW_NUMBER() over (
+        PARTITION BY tx_digest
+        ORDER BY
+            modified_timestamp DESC
+    ) = 1
 ),
 filtered as (
    SELECT 
@ -28,6 +33,7 @@ filtered as (
        fbc.block_timestamp,
        fbc.tx_digest,
        fbc.tx_succeeded,
+        fbc.tx_sender,
        case 
            when fbc.amount < 0 
            and fbc.address_owner IS NOT NULL 
@ -51,16 +57,17 @@ filtered as (
        AND fbc.modified_timestamp >= (SELECT COALESCE(MAX(modified_timestamp),'1970-01-01') FROM {{ this }})
    {% endif %}
 )
-SELECT DISTINCT
+SELECT  
    checkpoint_number,
    block_timestamp,
    tx_digest,
    tx_succeeded,
+    tx_sender,
    sender,
    receiver,
    balance_change_index,
    coin_type,
-    amount,
+    amount AS amount_raw,
    {{ dbt_utils.generate_surrogate_key(['tx_digest','balance_change_index']) }} AS transfers_id,
    SYSDATE() AS inserted_timestamp,
    SYSDATE() AS modified_timestamp,
--- a/models/silver/core/silver_core.yml
+++ b/models/silver/core/silver_core.yml
@ -91,6 +91,11 @@ models:
        data_type: BOOLEAN
        tests:
          - not_null
+      - name: TX_SENDER
+        description: "{{ doc('tx_sender') }}"
+        data_type: VARCHAR
+        tests:
+          - not_null
      - name: SENDER
        description: "{{ doc('sender') }}"
        data_type: VARCHAR
@ -111,13 +116,16 @@ models:
        data_type: VARCHAR
        tests:
          - not_null
-      - name: AMOUNT
+      - name: AMOUNT_RAW
        description: "{{ doc('amount') }}"
        data_type: NUMBER
        tests:
          - not_null
      - name: TRANSFERS_ID
        data_type: VARCHAR
+        tests:
+          - not_null
+          - unique
      - name: INSERTED_TIMESTAMP
        description: "{{ doc('inserted_timestamp') }}"
        data_type: TIMESTAMP_NTZ