upd ft model

models/descriptions/asset_identifier.md

@@ -0,0 +1,5 @@
+{% docs asset_identifier %}
+
+The onchain representation of a token id, which may include source chain metadata if involved in a crosschain bridge event. Unique per token in the FT contract metadata table.
+
+{% enddocs %} 

models/descriptions/crosschain_token_contract.md

@@ -0,0 +1,5 @@
+{% docs crosschain_token_contract %}
+
+The contract address of the token on the source chain, where applicable.
+
+{% enddocs %} 

models/descriptions/near_token_contract.md

@@ -0,0 +1,5 @@
+{% docs near_token_contract %}
+
+The contract address of the token deployed to NEAR.
+
+{% enddocs %} 

models/gold/core/core__dim_ft_contract_metadata.sql

@@ -3,23 +3,18 @@
     tags = ['scheduled_non_core']
 ) }}
 
-WITH ft_contract_metadata AS (
-
-    SELECT
-        *
-    FROM
-        {{ ref('silver__ft_contract_metadata') }}
-)
 SELECT
-    contract_address,
-    raw_token_id,
-    token_id,
+    asset_identifier,
+    source_chain,
+    crosschain_token_contract,
+    near_token_contract AS contract_address,
     NAME,
     symbol,
     decimals,
-    DATA,
     ft_contract_metadata_id AS dim_ft_contract_metadata_id,
     inserted_timestamp,
     modified_timestamp
 FROM
-    ft_contract_metadata
+    {{ ref('silver__ft_contract_metadata') }}
+WHERE
+    name is not null OR symbol is not null

models/gold/core/core__dim_ft_contract_metadata.yml

@@ -6,46 +6,34 @@ models:
       Fungible Token contract metadata provided by the Nearblocks NFT endpoint.
 
     columns:
-      - name: CONTRACT_ADDRESS
-        description: "{{ doc('contract_address')}}"
+      - name: ASSET_IDENTIFIER
+        description: "{{ doc('asset_identifier') }}"
         tests:
           - not_null
-
+          - unique
+      - name: SOURCE_CHAIN
+        description: "{{ doc('source_chain') }}"
+      - name: CROSSCHAIN_TOKEN_CONTRACT
+        description: "{{ doc('crosschain_token_contract') }}"
+      - name: CONTRACT_ADDRESS
+        description: "{{ doc('contract_address') }}"
       - name: NAME
-        description: "{{ doc('name')}}"
-        tests: 
+        description: "{{ doc('name') }}"
+        tests:
           - not_null
-
       - name: SYMBOL
-        description: "{{ doc('symbol')}}"
-        tests: 
+        description: "{{ doc('symbol') }}"
+        tests:
           - not_null
-
-      - name: DATA
-        description: "{{ doc('data')}}"
-        tests: 
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - VARIANT
-                - OBJECT
-                - ARRAY
-
       - name: DECIMALS
-        description: "{{ doc('decimals')}}"
-
-      - name: DATA
-        description: "{{ doc('data')}}"
-
+        description: "{{ doc('decimals') }}"
       - name: DIM_FT_CONTRACT_METADATA_ID
         description: "{{doc('id')}}"
         tests:
           - not_null
           - unique:
               where: inserted_timestamp >= SYSDATE() - INTERVAL '{{ var('DBT_TEST_LOOKBACK_DAYS', 14) }} days'
-
       - name: INSERTED_TIMESTAMP
         description: "{{doc('inserted_timestamp')}}"
-
       - name: MODIFIED_TIMESTAMP
         description: "{{doc('modified_timestamp')}}"

models/silver/labels/silver__ft_contract_metadata.sql

@@ -33,11 +33,10 @@ WITH nearblocks AS (
 ),
 nearblocks_metadata AS (
     SELECT
-        VALUE :contract :: STRING AS contract_address,
+        VALUE :contract :: STRING AS near_token_contract,
         VALUE :decimals :: INT AS decimals,
         VALUE :name :: STRING AS NAME,
-        VALUE :symbol :: STRING AS symbol,
-        VALUE AS DATA
+        VALUE :symbol :: STRING AS symbol
     FROM
         nearblocks,
         LATERAL FLATTEN(
@@ -46,14 +45,36 @@ nearblocks_metadata AS (
 ),
 omni AS (
     SELECT
-        omni_address,
-        contract_address
+        omni_asset_identifier AS asset_identifier,
+        SPLIT_PART(omni_asset_identifier, ':', 1) :: STRING AS source_chain,
+        SPLIT_PART(omni_asset_identifier, ':', 2) :: STRING AS crosschain_token_contract,
+        contract_address AS near_token_contract
     FROM
         {{ ref('silver__omni_metadata')}}
 
     {% if is_incremental() %}
     WHERE
-        inserted_timestamp >= (
+        modified_timestamp >= (
+            SELECT
+                MAX(modified_timestamp)
+            FROM
+                {{ this }}
+        )
+    {% endif %}
+),
+omni_unmapped AS (
+    SELECT
+        contract_address AS asset_identifier,
+        SPLIT_PART(contract_address, ':', 1) :: STRING AS source_chain,
+        SPLIT_PART(contract_address, ':', 2) :: STRING AS crosschain_token_contract
+    FROM
+        {{ ref('streamline__omni_tokenlist')}}
+    WHERE
+        source_chain_id IN ('near', 'sol')
+
+    {% if is_incremental() %}
+    WHERE
+        modified_timestamp >= (
             SELECT
                 MAX(modified_timestamp)
             FROM
@@ -63,8 +84,14 @@ omni AS (
 ),
 defuse AS (
     SELECT
-        defuse_asset_identifier AS defuse_address,
-        near_token_id AS contract_address,   
+        defuse_asset_identifier AS asset_identifier,
+        CASE
+            WHEN SPLIT_PART(defuse_asset_identifier, ':', 0) = 'near' THEN 'near'
+            WHEN SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) = 'native' THEN SPLIT_PART(near_token_id, '.', 0) :: STRING
+            ELSE SPLIT_PART(near_token_id, '-', 0) :: STRING
+        END AS source_chain,
+        SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) AS crosschain_token_contract,
+        near_token_id AS near_token_contract,   
         asset_name AS symbol,
         decimals
     FROM
@@ -72,7 +99,7 @@ defuse AS (
 
     {% if is_incremental() %}
     WHERE
-        inserted_timestamp >= (
+        modified_timestamp >= (
             SELECT
                 MAX(modified_timestamp)
             FROM
@@ -83,9 +110,10 @@ defuse AS (
 final AS (
     -- Omni
     SELECT
-        o.omni_address :: STRING AS raw_token_id,
-        SPLIT_PART(o.omni_address, ':', ARRAY_SIZE(SPLIT(o.omni_address, ':'))) AS token_id,
-        o.contract_address :: STRING AS contract_address,
+        o.asset_identifier,
+        o.source_chain,
+        o.crosschain_token_contract,
+        o.near_token_contract,
         n.decimals :: INT AS decimals,
         n.name :: STRING AS name,
         n.symbol :: STRING AS symbol,
@@ -93,15 +121,38 @@ final AS (
     FROM
         omni o
     LEFT JOIN nearblocks_metadata n
-        ON o.contract_address = n.contract_address
+        ON o.near_token_contract = n.near_token_contract
     
     UNION ALL
 
+    -- Omni unmapped
+    SELECT
+        o.asset_identifier,
+        o.source_chain,
+        o.crosschain_token_contract,
+        n.near_token_contract,
+        COALESCE(n.decimals, c.decimals) :: INT AS decimals,
+        COALESCE(n.name, c.name) :: STRING AS name,
+        COALESCE(n.symbol, c.symbol) :: STRING AS symbol,
+        'omni_unmapped' AS source
+    FROM
+        omni_unmapped o
+    LEFT JOIN nearblocks_metadata n
+        ON o.crosschain_token_contract = n.near_token_contract
+    LEFT JOIN {{ source('crosschain_silver', 'complete_token_asset_metadata')}} c
+        ON o.crosschain_token_contract = c.token_address
+        AND c.blockchain = 'solana'
+        -- note, this does not give use the Near token contract.
+        -- could join on symbol, but some symbols have multiple contract records as symbol is not unique
+
+    UNION ALL
+
     -- Nearblocks
     SELECT
-        n.contract_address :: STRING AS token_id,
-        n.contract_address :: STRING AS raw_token_id,
-        n.contract_address :: STRING AS contract_address,
+        n.near_token_contract :: STRING AS asset_identifier,
+        'near' AS source_chain,
+        n.near_token_contract :: STRING AS crosschain_token_contract,
+        n.near_token_contract,
         n.decimals :: INT AS decimals,
         n.name :: STRING AS name,
         n.symbol :: STRING AS symbol,
@@ -113,31 +164,35 @@ final AS (
 
     -- Defuse
     SELECT
-        d.defuse_address :: STRING AS token_id,
-        SPLIT_PART(d.defuse_address, ':', ARRAY_SIZE(SPLIT(d.defuse_address, ':'))) AS token_id,
-        d.contract_address :: STRING AS contract_address,
+        d.asset_identifier,
+        d.source_chain,
+        d.crosschain_token_contract,
+        d.near_token_contract,
         d.decimals :: INT AS decimals,
-        NULL :: STRING AS name,
+        n.name :: STRING AS name,
         d.symbol :: STRING AS symbol,
         'defuse' AS source
     FROM 
         defuse d
+    LEFT JOIN nearblocks_metadata n
+        ON d.near_token_contract = n.near_token_contract
 )
 SELECT
-    contract_address,
-    raw_token_id,
-    token_id,
+    asset_identifier,
+    source_chain,
+    crosschain_token_contract,
+    near_token_contract,
     decimals,
     name,
     symbol,
     source as metadata_source,
-    {{ dbt_utils.generate_surrogate_key(['token_id', 'contract_address']) }} AS ft_contract_metadata_id,
+    {{ dbt_utils.generate_surrogate_key(['asset_identifier']) }} AS ft_contract_metadata_id,
     SYSDATE() AS inserted_timestamp,
     SYSDATE() AS modified_timestamp,
     '{{ invocation_id }}' AS _invocation_id
 FROM final
 
 qualify ROW_NUMBER() over (
-    PARTITION BY token_id
+    PARTITION BY asset_identifier
     ORDER BY modified_timestamp DESC
 ) = 1

models/silver/labels/silver__ft_contract_metadata.yml

@@ -11,44 +11,33 @@ models:
           interval: 8
 
     columns:
-      - name: CONTRACT_ADDRESS
-        description: "{{ doc('contract_address')}}"
+      - name: ASSET_IDENTIFIER
+        description: "{{ doc('asset_identifier') }}"
         tests:
           - not_null
           - unique
-          - dbt_expectations.expect_column_values_to_be_of_type:
-              column_type: VARCHAR
-
+      - name: SOURCE_CHAIN
+        description: "{{ doc('source_chain') }}"
+      - name: CROSSCHAIN_TOKEN_CONTRACT
+        description: "{{ doc('crosschain_token_contract') }}"
+      - name: NEAR_TOKEN_CONTRACT
+        description: "{{ doc('near_token_contract') }}"
       - name: DECIMALS
-        description: "{{ doc('decimals')}}"
-        tests:
-          - not_null
-
+        description: "{{ doc('decimals') }}"
       - name: NAME
-        description: "{{ doc('name')}}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_of_type:
-              column_type: VARCHAR
-
+        description: "{{ doc('name') }}"
       - name: SYMBOL
-        description: "{{ doc('symbol')}}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_of_type:
-              column_type: VARCHAR
-
-      - name: DATA
-        description: "{{ doc('data')}}"
-
+        description: "{{ doc('symbol') }}"
+      - name: METADATA_SOURCE
+        description: "Source of the metadata record (e.g., omni, nearblocks, defuse)."
       - name: FT_CONTRACT_METADATA_ID
         description: "{{doc('id')}}"
-
+        tests:
+          - not_null
+          - unique
       - name: INSERTED_TIMESTAMP
         description: "{{doc('inserted_timestamp')}}"
-
       - name: MODIFIED_TIMESTAMP
         description: "{{doc('modified_timestamp')}}"
-
       - name: _INVOCATION_ID
         description: "{{doc('invocation_id')}}"

models/silver/labels/silver__omni_metadata.sql

@@ -1,23 +1,20 @@
 {{ config(
     materialized = 'incremental',
     merge_exclude_columns = ["inserted_timestamp"],
-    unique_key = 'omni_address',
+    unique_key = 'omni_asset_identifier',
     tags = ['scheduled_non_core']
 ) }}
 
 WITH omni AS (
 
     SELECT
-        VALUE:CONTRACT_ADDRESS::STRING AS omni_address,
+        VALUE:CONTRACT_ADDRESS::STRING AS omni_asset_identifier,
         VALUE:data:result:result::ARRAY AS result_array,
         DATA
     FROM
-        near_dev.bronze.omni_metadata
-    WHERE
-        typeof(DATA) != 'NULL_VALUE'
-
+        {{ ref('bronze__omni_metadata') }}
     {% if is_incremental() %}
-    AND
+    WHERE
         _inserted_timestamp >= (
             SELECT
                 MAX(modified_timestamp)
@@ -28,11 +25,11 @@ WITH omni AS (
 ),
 try_decode_hex AS (
     SELECT 
-        omni_address,
+        omni_asset_identifier,
         b.value AS raw,
         b.index,
         LPAD(TRIM(to_char(b.value::INT, 'XXXXXXX'))::STRING, 2, '0') AS hex,
-        ROW_NUMBER() OVER (PARTITION BY omni_address, raw, index, hex ORDER BY omni_address) AS rn
+        ROW_NUMBER() OVER (PARTITION BY omni_asset_identifier, raw, index, hex ORDER BY omni_asset_identifier) AS rn
     FROM omni o,
     TABLE(FLATTEN(o.result_array, recursive => TRUE)) b
     WHERE IS_ARRAY(o.result_array) = TRUE
@@ -40,27 +37,27 @@ try_decode_hex AS (
 ),
 decoded_response AS (
     SELECT 
-        omni_address,
+        omni_asset_identifier,
         ARRAY_TO_STRING(ARRAY_AGG(hex) WITHIN GROUP (ORDER BY index ASC), '') AS decoded_response
     FROM try_decode_hex
-    GROUP BY omni_address, rn
+    GROUP BY omni_asset_identifier, rn
     HAVING rn = 1
 ),
 conversion AS (
     SELECT
-        omni_address,
+        omni_asset_identifier,
         TRIM(livequery.utils.udf_hex_to_string(decoded_response), '"') AS decoded_result
     FROM decoded_response
 )
 SELECT
-    omni_address,
+    omni_asset_identifier,
     decoded_result AS contract_address,
     {{ dbt_utils.generate_surrogate_key(
-        ['omni_address']
+        ['omni_asset_identifier']
     ) }} AS omni_metadata_id,
     SYSDATE() AS inserted_timestamp,
     SYSDATE() AS modified_timestamp,
     '{{ invocation_id }}' AS _invocation_id
 FROM
     conversion
-qualify(row_number() over (partition by omni_address order by inserted_timestamp asc)) = 1
+qualify(row_number() over (partition by omni_asset_identifier order by inserted_timestamp asc)) = 1

models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql

@@ -9,7 +9,7 @@
 
 WITH omni_token AS (
     SELECT
-        DISTINCT lower(raw_token_id) AS contract_address
+        DISTINCT raw_token_id AS contract_address
     FROM 
         {{ ref('silver__bridge_omni') }}
     
@@ -23,6 +23,7 @@ WHERE modified_timestamp >= (
 )
 SELECT
     contract_address,
+    SPLIT_PART(contract_address, ':', 0) :: STRING AS source_chain_id,
     {{ dbt_utils.generate_surrogate_key(
         ['contract_address']
     ) }} AS omni_tokenlist_id,