diff --git a/models/descriptions/fee_amount_adj.md b/models/descriptions/fee_amount_adj.md new file mode 100644 index 0000000..8d02d3b --- /dev/null +++ b/models/descriptions/fee_amount_adj.md @@ -0,0 +1,5 @@ +{% docs fee_amount_adj %} + +Decimal-adjusted fee amount collected from the intent execution. This field provides the fee amount after applying the appropriate decimal precision adjustments based on the fee token's decimal places. For example, if a fee of 1 USDT was collected, the fee_amount_adj would be 1.0 after dividing the raw amount by 10^6 (USDT has 6 decimal places). This field is the most commonly used representation for fee amounts in analytics and reporting as it provides human-readable values. This field is null when no fees were collected or when the fee token's decimal information is unavailable. + +{% enddocs %} diff --git a/models/descriptions/fee_amount_usd.md b/models/descriptions/fee_amount_usd.md new file mode 100644 index 0000000..e6e9f03 --- /dev/null +++ b/models/descriptions/fee_amount_usd.md @@ -0,0 +1,5 @@ +{% docs fee_amount_usd %} + +USD value of the fee collected from the intent execution. This field provides the dollar equivalent value of the fee by multiplying the decimal-adjusted fee amount by the fee token's USD price at the time of the intent execution. This standardized USD representation enables protocol revenue analysis, fee tracking across different tokens, and cost comparisons over time. The field uses ZEROIFNULL to ensure zero values when price data is unavailable rather than null, which simplifies aggregations and revenue calculations. This field may be zero when price data is unavailable for new tokens, tokens with low liquidity, or during periods when price feeds are unavailable. + +{% enddocs %} diff --git a/models/descriptions/fee_token.md b/models/descriptions/fee_token.md new file mode 100644 index 0000000..8661af6 --- /dev/null +++ b/models/descriptions/fee_token.md @@ -0,0 +1,5 @@ +{% docs fee_token %} + +Token symbol for the fee collected (e.g., 'NEAR', 'USDT'). This field identifies the specific token in which the intent execution fee was charged, extracted and labeled from the fees_collected_raw JSON object. Fee tokens are typically stablecoins (like USDT, USDC) or the native protocol token (NEAR), though any supported token can be used for fees. This field is null when no fees were collected or when the fee token cannot be identified in the token metadata. Understanding fee tokens is important for protocol revenue analysis and cost tracking across different intent executions. + +{% enddocs %} diff --git a/models/descriptions/fees_collected_raw.md b/models/descriptions/fees_collected_raw.md new file mode 100644 index 0000000..a7095e6 --- /dev/null +++ b/models/descriptions/fees_collected_raw.md @@ -0,0 +1,5 @@ +{% docs fees_collected_raw %} + +Raw JSON object containing fee information collected from the intent execution, as extracted from the DIP4 event log. This field contains the complete on-chain representation of fees charged by the intent protocol, formatted as a JSON object with token addresses as keys and unadjusted fee amounts as values. For example: `{"nep141:wrap.near": "1232145523809524"}` indicates fees collected in wrapped NEAR tokens. This field is null when no fees were collected or when fee information is not available in the event log. The raw format preserves the exact on-chain data structure for precise fee calculations and protocol revenue analysis. + +{% enddocs %} diff --git a/models/descriptions/tables/defi__fact_intents.md b/models/descriptions/tables/defi__fact_intents.md index d7b8154..4398c9b 100644 --- a/models/descriptions/tables/defi__fact_intents.md +++ b/models/descriptions/tables/defi__fact_intents.md @@ -1,7 +1,7 @@ {% docs defi__fact_intents %} ## Description -This table contains all intent-based transactions on the NEAR Protocol blockchain, capturing user intents for token transfers, swaps, and other DeFi operations through the intents.near protocol. The data includes both NEP-245 and DIP-4 standard intents, providing comprehensive tracking of intent creation, execution, and fulfillment. This table enables analysis of intent-based trading patterns, MEV protection mechanisms, and user behavior in intent-driven DeFi protocols. +This table contains all intent-based transactions on the NEAR Protocol blockchain, capturing user intents for token transfers, swaps, and other DeFi operations through the intents.near protocol. The data includes both NEP-245 and DIP-4 standard intents, providing comprehensive tracking of intent creation, execution, and fulfillment. This table enables analysis of intent-based trading patterns, MEV protection mechanisms, and user behavior in intent-driven DeFi protocols. This table includes all intent execution within a transaction, which may include several intermediate steps interacting with solvers. ## Key Use Cases - Intent-based trading analysis and pattern recognition diff --git a/models/gold/defi/defi__ez_intents.sql b/models/gold/defi/defi__ez_intents.sql index 9049fcb..0ed3160 100644 --- a/models/gold/defi/defi__ez_intents.sql +++ b/models/gold/defi/defi__ez_intents.sql @@ -92,6 +92,16 @@ WITH intents AS ( ) AS asset_identifier, referral, dip4_version, + fees_collected_raw, + REGEXP_SUBSTR( + object_keys(try_parse_json(fees_collected_raw))[0]::string, + 'nep(141|171|245):(.*)', + 1, + 1, + 'e', + 2 + ) AS fee_asset_identifier, + try_parse_json(fees_collected_raw)[object_keys(try_parse_json(fees_collected_raw))[0]]::string as fee_amount_raw, gas_burnt, receipt_succeeded, fact_intents_id, @@ -223,11 +233,32 @@ FINAL AS ( COALESCE(p.price, p2.price) ) ) AS amount_usd, - COALESCE(p.is_verified, p2.is_verified, FALSE) AS token_is_verified + COALESCE(p.is_verified, p2.is_verified, FALSE) AS token_is_verified, + -- fee information + fees_collected_raw, + l2.symbol AS fee_token, + i.fee_asset_identifier, + i.fee_amount_raw, + l2.decimals AS fee_decimals, + i.fee_amount_raw :: NUMBER / pow( + 10, + l2.decimals + ) AS fee_amount_adj, + ZEROIFNULL( + i.fee_amount_raw :: NUMBER / pow(10, l2.decimals) * IFF( + l2.symbol ilike 'USD%', + COALESCE(p_fee.price, 1), + COALESCE(p_fee.price, p2_fee.price) + ) + ) AS fee_amount_usd FROM intents i LEFT JOIN labels l ON i.asset_identifier = l.asset_identifier + -- label the fee token + LEFT JOIN labels l2 + ON i.fee_asset_identifier = l2.asset_identifier + -- price the main token ASOF JOIN prices p match_condition ( i.block_timestamp >= p.hour ) @@ -241,6 +272,20 @@ FINAL AS ( upper(l.symbol) = upper(p2.symbol) AND (l.crosschain_token_contract = 'native') = p2.is_native ) + -- price the fee token + ASOF JOIN prices p_fee match_condition ( + i.block_timestamp >= p_fee.hour + ) + ON ( + l2.crosschain_token_contract = p_fee.contract_address + ) + ASOF JOIN prices_native p2_fee match_condition ( + i.block_timestamp >= p2_fee.hour + ) + ON ( + upper(l2.symbol) = upper(p2_fee.symbol) + AND (l2.crosschain_token_contract = 'native') = p2_fee.is_native + ) ) SELECT block_timestamp, @@ -266,6 +311,10 @@ SELECT gas_burnt, memo, referral, + fees_collected_raw, + fee_token, + fee_amount_adj, + fee_amount_usd, dip4_version, log_index, log_event_index, diff --git a/models/gold/defi/defi__ez_intents.yml b/models/gold/defi/defi__ez_intents.yml index 190f6b1..464cb7e 100644 --- a/models/gold/defi/defi__ez_intents.yml +++ b/models/gold/defi/defi__ez_intents.yml @@ -59,6 +59,23 @@ models: - name: REFERRAL description: "{{ doc('referral') }}" + - name: FEES_COLLECTED_RAW + description: "{{ doc('fees_collected_raw') }}" + + - name: FEE_TOKEN + description: "{{ doc('fee_token') }}" + + - name: FEE_AMOUNT_ADJ + description: "{{ doc('fee_amount_adj') }}" + + - name: FEE_AMOUNT_USD + description: "{{ doc('fee_amount_usd') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: DIP4_VERSION description: "{{ doc('dip4_version') }}" diff --git a/models/gold/defi/defi__fact_intents.sql b/models/gold/defi/defi__fact_intents.sql index 5100bc4..84ec2ae 100644 --- a/models/gold/defi/defi__fact_intents.sql +++ b/models/gold/defi/defi__fact_intents.sql @@ -74,9 +74,9 @@ logs_base AS( predecessor_id, signer_id, gas_burnt, - clean_log, - TRY_PARSE_JSON(clean_log) :event :: STRING AS log_event, - TRY_PARSE_JSON(clean_log) :data :: ARRAY AS log_data, + TRY_PARSE_JSON(clean_log) AS log_json, + log_json :event :: STRING AS log_event, + log_json :data :: ARRAY AS log_data, ARRAY_SIZE(log_data) AS log_data_len, receipt_succeeded, modified_timestamp @@ -102,7 +102,7 @@ nep245_logs AS ( FROM logs_base lb WHERE - TRY_PARSE_JSON(lb.clean_log) :standard :: STRING = 'nep245' + lb.log_json :standard :: STRING = 'nep245' {% if is_incremental() and not var("MANUAL_FIX") %} AND @@ -112,12 +112,13 @@ nep245_logs AS ( dip4_logs AS ( SELECT lb.*, - try_parse_json(lb.clean_log):data[0]:referral::string as referral, - try_parse_json(lb.clean_log):version :: string as version + lb.log_json:data[0]:referral::string as referral, + lb.log_json:data[0]:fees_collected as fees_collected_raw, + lb.log_json:version :: string as version FROM logs_base lb WHERE - TRY_PARSE_JSON(lb.clean_log) :standard :: STRING = 'dip4' + lb.log_json :standard :: STRING = 'dip4' {% if is_incremental() and not var("MANUAL_FIX") %} AND COALESCE(lb.modified_timestamp, '1970-01-01') >= '{{max_mod}}' @@ -197,6 +198,7 @@ SELECT final.amount_raw, final.token_id, dip4.referral, + dip4.fees_collected_raw, dip4.version AS dip4_version, final.gas_burnt, final.receipt_succeeded, diff --git a/models/gold/defi/defi__fact_intents.yml b/models/gold/defi/defi__fact_intents.yml index 71150bf..5e1fda3 100644 --- a/models/gold/defi/defi__fact_intents.yml +++ b/models/gold/defi/defi__fact_intents.yml @@ -60,6 +60,9 @@ models: - name: REFERRAL description: "{{ doc('referral') }}" + - name: FEES_COLLECTED_RAW + description: "{{ doc('fees_collected_raw') }}" + - name: DIP4_VERSION description: "{{ doc('dip4_version') }}" diff --git a/models/silver/labels/external/silver__chainlist_ids.sql b/models/silver/labels/external/silver__chainlist_ids.sql new file mode 100644 index 0000000..2f062bc --- /dev/null +++ b/models/silver/labels/external/silver__chainlist_ids.sql @@ -0,0 +1,38 @@ +{{ config( + materialized = 'incremental', + unique_key = 'chainlist_id', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + tags = ['scheduled_non_core'] +) }} + +WITH api_call AS ( + + SELECT + response + FROM + {{ ref('streamline__chainlist_ids_realtime') }} +), +flattened AS ( + SELECT + VALUE :chain ::STRING AS chain, + VALUE :chainId :: INT AS chain_id, + VALUE :name :: STRING AS chain_name + FROM + api_call, + LATERAL FLATTEN( + input => response :data :: ARRAY + ) +) +SELECT + chain, + chain_id, + chain_name, + {{ dbt_utils.generate_surrogate_key(['chain_id']) }} AS chainlist_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + flattened + +qualify(row_number() over (partition by chain_id order by inserted_timestamp asc)) = 1 diff --git a/models/silver/labels/external/silver__defuse_ft_metadata.sql b/models/silver/labels/external/silver__defuse_ft_metadata.sql index c05e0ef..c60797e 100644 --- a/models/silver/labels/external/silver__defuse_ft_metadata.sql +++ b/models/silver/labels/external/silver__defuse_ft_metadata.sql @@ -16,6 +16,8 @@ WITH api_call AS ( flattened AS ( SELECT VALUE :defuse_asset_identifier :: STRING AS defuse_asset_identifier, + VALUE :intents_token_id :: STRING AS intents_token_id, + VALUE :standard :: STRING AS standard, VALUE :asset_name :: STRING AS asset_name, VALUE :decimals :: INT AS decimals, VALUE :min_deposit_amount :: STRING AS min_deposit_amount, @@ -27,15 +29,79 @@ FROM LATERAL FLATTEN( input => response :data :result :tokens :: ARRAY ) +), +chain_mapping AS ( + -- Map EVM chain IDs to blockchain names + SELECT + chain_id :: STRING AS chain_id, + LOWER(chain) AS blockchain_name + FROM + {{ ref('silver__chainlist_ids') }} + +), +parsed AS ( + SELECT + defuse_asset_identifier, + intents_token_id, + standard, + asset_name, + decimals, + min_deposit_amount, + min_withdrawal_amount, + near_token_contract, + withdrawal_fee, + -- Parse the asset_identifier (what ez_intents joins on) + CASE + WHEN standard = 'nep245' THEN + -- For NEP245: extract everything after "nep245:" + -- Example: nep245:v2_1.omni.hot.tg:56_11111111111111111111 -> v2_1.omni.hot.tg:56_11111111111111111111 + REGEXP_SUBSTR(intents_token_id, 'nep245:(.*)', 1, 1, 'e', 1) + ELSE + -- For NEP141: use near_token_contract as before + near_token_contract + END AS asset_identifier, + -- Parse source_chain + CASE + WHEN standard = 'nep245' THEN + -- For NEP245: parse from defuse_asset_identifier + -- Format: blockchain:chainId:contractAddress + COALESCE( + cm.blockchain_name, + CASE + WHEN SPLIT_PART(defuse_asset_identifier, ':', 1) = 'ton' THEN 'ton' + WHEN SPLIT_PART(defuse_asset_identifier, ':', 1) = 'sol' THEN 'sol' + WHEN SPLIT_PART(defuse_asset_identifier, ':', 1) = 'stellar' THEN 'stellar' + ELSE 'unknown' + END + ) + WHEN SPLIT_PART(defuse_asset_identifier, ':', 1) = 'near' THEN 'near' + WHEN SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) = 'native' THEN + SPLIT_PART(near_token_contract, '.', 1) :: STRING + ELSE + SPLIT_PART(near_token_contract, '-', 1) :: STRING + END AS source_chain, + -- Parse crosschain_token_contract + CASE + WHEN standard = 'nep245' THEN + -- For NEP245: parse contract address from defuse_asset_identifier + CASE + WHEN SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) = 'native' THEN 'native' + ELSE SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) + END + ELSE + SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) + END AS crosschain_token_contract + FROM + flattened + LEFT JOIN chain_mapping cm + ON SPLIT_PART(flattened.defuse_asset_identifier, ':', 2) = cm.chain_id + AND flattened.standard = 'nep245' ) SELECT defuse_asset_identifier, - CASE - WHEN SPLIT_PART(defuse_asset_identifier, ':', 0) = 'near' THEN 'near' - WHEN SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) = 'native' THEN SPLIT_PART(near_token_contract, '.', 0) :: STRING - ELSE SPLIT_PART(near_token_contract, '-', 0) :: STRING - END AS source_chain, - SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) AS crosschain_token_contract, + asset_identifier, + source_chain, + crosschain_token_contract, asset_name, decimals, min_deposit_amount, @@ -43,12 +109,12 @@ SELECT near_token_contract, withdrawal_fee, {{ dbt_utils.generate_surrogate_key( - ['defuse_asset_identifier'] + ['asset_identifier'] ) }} AS defuse_ft_metadata_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - flattened + parsed -qualify(row_number() over (partition by defuse_asset_identifier order by inserted_timestamp asc)) = 1 +qualify(row_number() over (partition by asset_identifier order by inserted_timestamp asc)) = 1 diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index 7376ed2..df713ef 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -92,10 +92,10 @@ omni_unmapped AS ( ), defuse AS ( SELECT - d.near_token_contract AS asset_identifier, + d.asset_identifier, d.source_chain, d.crosschain_token_contract, - d.near_token_contract, + d.near_token_contract, d.decimals, NULL AS name, asset_name AS symbol, diff --git a/models/streamline/external/token_metadata/chainlist/streamline__chainlist_ids_realtime.sql b/models/streamline/external/token_metadata/chainlist/streamline__chainlist_ids_realtime.sql new file mode 100644 index 0000000..36b67f5 --- /dev/null +++ b/models/streamline/external/token_metadata/chainlist/streamline__chainlist_ids_realtime.sql @@ -0,0 +1,30 @@ +{{ config ( + materialized = "view", + tags = ['streamline_non_core'] +) }} + +WITH api_call AS ( + + SELECT + {{ target.database }}.live.udf_api( + 'GET', + 'https://chainlist.org/rpcs.json', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'fsc-quantum-state', + 'livequery' + ), + {} + ) :: variant AS response +) +SELECT + response +FROM + api_call +WHERE + response IS NOT NULL + AND response :status_code :: INT IN ( + 200, + 201 + )