Merge pull request #506 from FlipsideCrypto/DAT2-126-intents-fees

DAT2-126/fix labels parsing, update fee columns

models/descriptions/fee_amount_adj.md

@@ -0,0 +1,5 @@
+{% docs fee_amount_adj %}
+
+Decimal-adjusted fee amount collected from the intent execution. This field provides the fee amount after applying the appropriate decimal precision adjustments based on the fee token's decimal places. For example, if a fee of 1 USDT was collected, the fee_amount_adj would be 1.0 after dividing the raw amount by 10^6 (USDT has 6 decimal places). This field is the most commonly used representation for fee amounts in analytics and reporting as it provides human-readable values. This field is null when no fees were collected or when the fee token's decimal information is unavailable.
+
+{% enddocs %}

models/descriptions/fee_amount_usd.md

@@ -0,0 +1,5 @@
+{% docs fee_amount_usd %}
+
+USD value of the fee collected from the intent execution. This field provides the dollar equivalent value of the fee by multiplying the decimal-adjusted fee amount by the fee token's USD price at the time of the intent execution. This standardized USD representation enables protocol revenue analysis, fee tracking across different tokens, and cost comparisons over time. The field uses ZEROIFNULL to ensure zero values when price data is unavailable rather than null, which simplifies aggregations and revenue calculations. This field may be zero when price data is unavailable for new tokens, tokens with low liquidity, or during periods when price feeds are unavailable.
+
+{% enddocs %}

models/descriptions/fee_token.md

@@ -0,0 +1,5 @@
+{% docs fee_token %}
+
+Token symbol for the fee collected (e.g., 'NEAR', 'USDT'). This field identifies the specific token in which the intent execution fee was charged, extracted and labeled from the fees_collected_raw JSON object. Fee tokens are typically stablecoins (like USDT, USDC) or the native protocol token (NEAR), though any supported token can be used for fees. This field is null when no fees were collected or when the fee token cannot be identified in the token metadata. Understanding fee tokens is important for protocol revenue analysis and cost tracking across different intent executions.
+
+{% enddocs %}

models/descriptions/fees_collected_raw.md

@@ -0,0 +1,5 @@
+{% docs fees_collected_raw %}
+
+Raw JSON object containing fee information collected from the intent execution, as extracted from the DIP4 event log. This field contains the complete on-chain representation of fees charged by the intent protocol, formatted as a JSON object with token addresses as keys and unadjusted fee amounts as values. For example: `{"nep141:wrap.near": "1232145523809524"}` indicates fees collected in wrapped NEAR tokens. This field is null when no fees were collected or when fee information is not available in the event log. The raw format preserves the exact on-chain data structure for precise fee calculations and protocol revenue analysis.
+
+{% enddocs %}

models/descriptions/tables/defi__fact_intents.md

@@ -1,7 +1,7 @@
 {% docs defi__fact_intents %}
 
 ## Description
-This table contains all intent-based transactions on the NEAR Protocol blockchain, capturing user intents for token transfers, swaps, and other DeFi operations through the intents.near protocol. The data includes both NEP-245 and DIP-4 standard intents, providing comprehensive tracking of intent creation, execution, and fulfillment. This table enables analysis of intent-based trading patterns, MEV protection mechanisms, and user behavior in intent-driven DeFi protocols.
+This table contains all intent-based transactions on the NEAR Protocol blockchain, capturing user intents for token transfers, swaps, and other DeFi operations through the intents.near protocol. The data includes both NEP-245 and DIP-4 standard intents, providing comprehensive tracking of intent creation, execution, and fulfillment. This table enables analysis of intent-based trading patterns, MEV protection mechanisms, and user behavior in intent-driven DeFi protocols. This table includes all intent execution within a transaction, which may include several intermediate steps interacting with solvers.
 
 ## Key Use Cases
 - Intent-based trading analysis and pattern recognition

models/gold/defi/defi__ez_intents.sql

@@ -92,6 +92,16 @@ WITH intents AS (
         ) AS asset_identifier,
         referral,
         dip4_version,
+        fees_collected_raw,
+        REGEXP_SUBSTR(
+            object_keys(try_parse_json(fees_collected_raw))[0]::string,
+            'nep(141|171|245):(.*)',
+            1,
+            1,
+            'e',
+            2
+        ) AS fee_asset_identifier,
+        try_parse_json(fees_collected_raw)[object_keys(try_parse_json(fees_collected_raw))[0]]::string as fee_amount_raw,
         gas_burnt,
         receipt_succeeded,
         fact_intents_id,
@@ -223,11 +233,32 @@ FINAL AS (
                 COALESCE(p.price, p2.price)
             )
         ) AS amount_usd,
-        COALESCE(p.is_verified, p2.is_verified, FALSE) AS token_is_verified
+        COALESCE(p.is_verified, p2.is_verified, FALSE) AS token_is_verified,
+        -- fee information
+        fees_collected_raw,
+        l2.symbol AS fee_token,
+        i.fee_asset_identifier,
+        i.fee_amount_raw,
+        l2.decimals AS fee_decimals,
+        i.fee_amount_raw :: NUMBER / pow(
+            10,
+            l2.decimals
+        ) AS fee_amount_adj,
+        ZEROIFNULL(
+            i.fee_amount_raw :: NUMBER / pow(10, l2.decimals) * IFF(
+                l2.symbol ilike 'USD%',
+                COALESCE(p_fee.price, 1),
+                COALESCE(p_fee.price, p2_fee.price)
+            )
+        ) AS fee_amount_usd
     FROM
         intents i
         LEFT JOIN labels l
         ON i.asset_identifier = l.asset_identifier
+        -- label the fee token
+        LEFT JOIN labels l2
+        ON i.fee_asset_identifier = l2.asset_identifier
+        -- price the main token
         ASOF JOIN prices p match_condition (
             i.block_timestamp >= p.hour
         )
@@ -241,6 +272,20 @@ FINAL AS (
             upper(l.symbol) = upper(p2.symbol)
             AND (l.crosschain_token_contract = 'native') = p2.is_native
         )
+        -- price the fee token
+        ASOF JOIN prices p_fee match_condition (
+            i.block_timestamp >= p_fee.hour
+        )
+        ON (
+            l2.crosschain_token_contract = p_fee.contract_address
+        )
+        ASOF JOIN prices_native p2_fee match_condition (
+            i.block_timestamp >= p2_fee.hour
+        )
+        ON (
+            upper(l2.symbol) = upper(p2_fee.symbol)
+            AND (l2.crosschain_token_contract = 'native') = p2_fee.is_native
+        )
 )
 SELECT
     block_timestamp,
@@ -266,6 +311,10 @@ SELECT
     gas_burnt,
     memo,
     referral,
+    fees_collected_raw,
+    fee_token,
+    fee_amount_adj,
+    fee_amount_usd,
     dip4_version,
     log_index,
     log_event_index,

models/gold/defi/defi__ez_intents.yml

@@ -59,6 +59,23 @@ models:
       - name: REFERRAL
         description: "{{ doc('referral') }}"
 
+      - name: FEES_COLLECTED_RAW
+        description: "{{ doc('fees_collected_raw') }}"
+
+      - name: FEE_TOKEN
+        description: "{{ doc('fee_token') }}"
+
+      - name: FEE_AMOUNT_ADJ
+        description: "{{ doc('fee_amount_adj') }}"
+
+      - name: FEE_AMOUNT_USD
+        description: "{{ doc('fee_amount_usd') }}"
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+
       - name: DIP4_VERSION
         description: "{{ doc('dip4_version') }}"
 

models/gold/defi/defi__fact_intents.sql

@@ -74,9 +74,9 @@ logs_base AS(
         predecessor_id,
         signer_id,
         gas_burnt,
-        clean_log,
-        TRY_PARSE_JSON(clean_log) :event :: STRING AS log_event,
-        TRY_PARSE_JSON(clean_log) :data :: ARRAY AS log_data,
+        TRY_PARSE_JSON(clean_log) AS log_json,
+        log_json :event :: STRING AS log_event,
+        log_json :data :: ARRAY AS log_data,
         ARRAY_SIZE(log_data) AS log_data_len,
         receipt_succeeded,
         modified_timestamp
@@ -102,7 +102,7 @@ nep245_logs AS (
     FROM 
         logs_base lb
     WHERE
-        TRY_PARSE_JSON(lb.clean_log) :standard :: STRING = 'nep245'
+        lb.log_json :standard :: STRING = 'nep245'
 
     {% if is_incremental() and not var("MANUAL_FIX") %}
         AND 
@@ -112,12 +112,13 @@ nep245_logs AS (
 dip4_logs AS (
     SELECT 
         lb.*,
-        try_parse_json(lb.clean_log):data[0]:referral::string as referral,
-        try_parse_json(lb.clean_log):version :: string as version
+        lb.log_json:data[0]:referral::string as referral,
+        lb.log_json:data[0]:fees_collected as fees_collected_raw,
+        lb.log_json:version :: string as version
     FROM 
         logs_base lb
     WHERE
-        TRY_PARSE_JSON(lb.clean_log) :standard :: STRING = 'dip4'
+        lb.log_json :standard :: STRING = 'dip4'
     {% if is_incremental() and not var("MANUAL_FIX") %}
         AND 
             COALESCE(lb.modified_timestamp, '1970-01-01') >= '{{max_mod}}'
@@ -197,6 +198,7 @@ SELECT
     final.amount_raw,
     final.token_id,
     dip4.referral,
+    dip4.fees_collected_raw,
     dip4.version AS dip4_version,
     final.gas_burnt,
     final.receipt_succeeded,

models/gold/defi/defi__fact_intents.yml

@@ -60,6 +60,9 @@ models:
       - name: REFERRAL
         description: "{{ doc('referral') }}"
 
+      - name: FEES_COLLECTED_RAW
+        description: "{{ doc('fees_collected_raw') }}"
+
       - name: DIP4_VERSION
         description: "{{ doc('dip4_version') }}"
 

models/silver/labels/external/silver__chainlist_ids.sql

@@ -0,0 +1,38 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = 'chainlist_id',
+    incremental_strategy = 'merge',
+    merge_exclude_columns = ["inserted_timestamp"],
+    tags = ['scheduled_non_core']
+) }}
+
+WITH api_call AS (
+
+    SELECT
+        response
+    FROM
+        {{ ref('streamline__chainlist_ids_realtime') }}
+),
+flattened AS (
+    SELECT
+        VALUE :chain ::STRING AS chain,
+        VALUE :chainId :: INT AS chain_id,
+        VALUE :name :: STRING AS chain_name
+    FROM
+        api_call,
+        LATERAL FLATTEN(
+            input => response :data :: ARRAY
+        )
+)
+SELECT
+    chain,
+    chain_id,
+    chain_name,
+    {{ dbt_utils.generate_surrogate_key(['chain_id']) }} AS chainlist_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    flattened
+
+qualify(row_number() over (partition by chain_id order by inserted_timestamp asc)) = 1

models/silver/labels/external/silver__defuse_ft_metadata.sql

@@ -16,6 +16,8 @@ WITH api_call AS (
 flattened AS (
 SELECT
     VALUE :defuse_asset_identifier :: STRING AS defuse_asset_identifier,
+    VALUE :intents_token_id :: STRING AS intents_token_id,
+    VALUE :standard :: STRING AS standard,
     VALUE :asset_name :: STRING AS asset_name,
     VALUE :decimals :: INT AS decimals,
     VALUE :min_deposit_amount :: STRING AS min_deposit_amount,
@@ -27,15 +29,79 @@ FROM
     LATERAL FLATTEN(
         input => response :data :result :tokens :: ARRAY
     )
+),
+chain_mapping AS (
+    -- Map EVM chain IDs to blockchain names
+    SELECT
+        chain_id :: STRING AS chain_id,
+        LOWER(chain) AS blockchain_name
+    FROM
+        {{ ref('silver__chainlist_ids') }}
+
+),
+parsed AS (
+    SELECT
+        defuse_asset_identifier,
+        intents_token_id,
+        standard,
+        asset_name,
+        decimals,
+        min_deposit_amount,
+        min_withdrawal_amount,
+        near_token_contract,
+        withdrawal_fee,
+        -- Parse the asset_identifier (what ez_intents joins on)
+        CASE
+            WHEN standard = 'nep245' THEN
+                -- For NEP245: extract everything after "nep245:"
+                -- Example: nep245:v2_1.omni.hot.tg:56_11111111111111111111 -> v2_1.omni.hot.tg:56_11111111111111111111
+                REGEXP_SUBSTR(intents_token_id, 'nep245:(.*)', 1, 1, 'e', 1)
+            ELSE
+                -- For NEP141: use near_token_contract as before
+                near_token_contract
+        END AS asset_identifier,
+        -- Parse source_chain
+        CASE
+            WHEN standard = 'nep245' THEN
+                -- For NEP245: parse from defuse_asset_identifier
+                -- Format: blockchain:chainId:contractAddress
+                COALESCE(
+                    cm.blockchain_name,
+                    CASE
+                        WHEN SPLIT_PART(defuse_asset_identifier, ':', 1) = 'ton' THEN 'ton'
+                        WHEN SPLIT_PART(defuse_asset_identifier, ':', 1) = 'sol' THEN 'sol'
+                        WHEN SPLIT_PART(defuse_asset_identifier, ':', 1) = 'stellar' THEN 'stellar'
+                        ELSE 'unknown'
+                    END
+                )
+            WHEN SPLIT_PART(defuse_asset_identifier, ':', 1) = 'near' THEN 'near'
+            WHEN SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) = 'native' THEN
+                SPLIT_PART(near_token_contract, '.', 1) :: STRING
+            ELSE
+                SPLIT_PART(near_token_contract, '-', 1) :: STRING
+        END AS source_chain,
+        -- Parse crosschain_token_contract
+        CASE
+            WHEN standard = 'nep245' THEN
+                -- For NEP245: parse contract address from defuse_asset_identifier
+                CASE
+                    WHEN SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) = 'native' THEN 'native'
+                    ELSE SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':')))
+                END
+            ELSE
+                SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':')))
+        END AS crosschain_token_contract
+    FROM
+        flattened
+    LEFT JOIN chain_mapping cm
+        ON SPLIT_PART(flattened.defuse_asset_identifier, ':', 2) = cm.chain_id
+        AND flattened.standard = 'nep245'
 )
 SELECT
     defuse_asset_identifier,
-    CASE
-        WHEN SPLIT_PART(defuse_asset_identifier, ':', 0) = 'near' THEN 'near'
-        WHEN SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) = 'native' THEN SPLIT_PART(near_token_contract, '.', 0) :: STRING
-        ELSE SPLIT_PART(near_token_contract, '-', 0) :: STRING
-    END AS source_chain,
-    SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) AS crosschain_token_contract,
+    asset_identifier,
+    source_chain,
+    crosschain_token_contract,
     asset_name,
     decimals,
     min_deposit_amount,
@@ -43,12 +109,12 @@ SELECT
     near_token_contract,
     withdrawal_fee,
     {{ dbt_utils.generate_surrogate_key(
-        ['defuse_asset_identifier']
+        ['asset_identifier']
     ) }} AS defuse_ft_metadata_id,
     SYSDATE() AS inserted_timestamp,
     SYSDATE() AS modified_timestamp,
     '{{ invocation_id }}' AS _invocation_id
 FROM
-    flattened
+    parsed
 
-qualify(row_number() over (partition by defuse_asset_identifier order by inserted_timestamp asc)) = 1
+qualify(row_number() over (partition by asset_identifier order by inserted_timestamp asc)) = 1

models/silver/labels/silver__ft_contract_metadata.sql

@@ -92,10 +92,10 @@ omni_unmapped AS (
 ),
 defuse AS (
     SELECT
-        d.near_token_contract AS asset_identifier,
+        d.asset_identifier,
         d.source_chain,
         d.crosschain_token_contract,
-        d.near_token_contract,   
+        d.near_token_contract,
         d.decimals,
         NULL AS name,
         asset_name AS symbol,

models/streamline/external/token_metadata/chainlist/streamline__chainlist_ids_realtime.sql

@@ -0,0 +1,30 @@
+{{ config (
+    materialized = "view",
+    tags = ['streamline_non_core']
+) }}
+
+WITH api_call AS (
+
+    SELECT
+        {{ target.database }}.live.udf_api(
+            'GET',
+            'https://chainlist.org/rpcs.json',
+            OBJECT_CONSTRUCT(
+                'Content-Type',
+                'application/json',
+                'fsc-quantum-state',
+                'livequery'
+            ),
+            {}
+        ) :: variant AS response
+)
+SELECT
+    response
+FROM
+    api_call
+WHERE
+    response IS NOT NULL
+    AND response :status_code :: INT IN (
+        200,
+        201
+    )