diff --git a/models/descriptions/asset_identifier.md b/models/descriptions/asset_identifier.md new file mode 100644 index 0000000..2f074f8 --- /dev/null +++ b/models/descriptions/asset_identifier.md @@ -0,0 +1,5 @@ +{% docs asset_identifier %} + +The onchain representation of a token id, which may include source chain metadata if involved in a crosschain bridge event. Unique per token in the FT contract metadata table. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/crosschain_token_contract.md b/models/descriptions/crosschain_token_contract.md new file mode 100644 index 0000000..2dfa559 --- /dev/null +++ b/models/descriptions/crosschain_token_contract.md @@ -0,0 +1,5 @@ +{% docs crosschain_token_contract %} + +The contract address of the token on the source chain, where applicable. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/near_token_contract.md b/models/descriptions/near_token_contract.md new file mode 100644 index 0000000..039a08b --- /dev/null +++ b/models/descriptions/near_token_contract.md @@ -0,0 +1,5 @@ +{% docs near_token_contract %} + +The contract address of the token deployed to NEAR. + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/core/core__dim_ft_contract_metadata.sql b/models/gold/core/core__dim_ft_contract_metadata.sql index b4963ea..7ec50b3 100644 --- a/models/gold/core/core__dim_ft_contract_metadata.sql +++ b/models/gold/core/core__dim_ft_contract_metadata.sql @@ -3,23 +3,18 @@ tags = ['scheduled_non_core'] ) }} -WITH ft_contract_metadata AS ( - - SELECT - * - FROM - {{ ref('silver__ft_contract_metadata') }} -) SELECT - contract_address, - raw_token_id, - token_id, + asset_identifier, + source_chain, + crosschain_token_contract, + near_token_contract AS contract_address, NAME, symbol, decimals, - DATA, ft_contract_metadata_id AS dim_ft_contract_metadata_id, inserted_timestamp, modified_timestamp FROM - ft_contract_metadata + {{ ref('silver__ft_contract_metadata') }} +WHERE + name is not null OR symbol is not null diff --git a/models/gold/core/core__dim_ft_contract_metadata.yml b/models/gold/core/core__dim_ft_contract_metadata.yml index d193f77..d738e73 100644 --- a/models/gold/core/core__dim_ft_contract_metadata.yml +++ b/models/gold/core/core__dim_ft_contract_metadata.yml @@ -6,46 +6,34 @@ models: Fungible Token contract metadata provided by the Nearblocks NFT endpoint. columns: - - name: CONTRACT_ADDRESS - description: "{{ doc('contract_address')}}" + - name: ASSET_IDENTIFIER + description: "{{ doc('asset_identifier') }}" tests: - not_null - + - unique + - name: SOURCE_CHAIN + description: "{{ doc('source_chain') }}" + - name: CROSSCHAIN_TOKEN_CONTRACT + description: "{{ doc('crosschain_token_contract') }}" + - name: CONTRACT_ADDRESS + description: "{{ doc('contract_address') }}" - name: NAME - description: "{{ doc('name')}}" - tests: + description: "{{ doc('name') }}" + tests: - not_null - - name: SYMBOL - description: "{{ doc('symbol')}}" - tests: + description: "{{ doc('symbol') }}" + tests: - not_null - - - name: DATA - description: "{{ doc('data')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - - OBJECT - - ARRAY - - name: DECIMALS - description: "{{ doc('decimals')}}" - - - name: DATA - description: "{{ doc('data')}}" - + description: "{{ doc('decimals') }}" - name: DIM_FT_CONTRACT_METADATA_ID description: "{{doc('id')}}" tests: - not_null - unique: where: inserted_timestamp >= SYSDATE() - INTERVAL '{{ var('DBT_TEST_LOOKBACK_DAYS', 14) }} days' - - name: INSERTED_TIMESTAMP description: "{{doc('inserted_timestamp')}}" - - name: MODIFIED_TIMESTAMP description: "{{doc('modified_timestamp')}}" diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index e7735dd..4547a30 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -33,11 +33,10 @@ WITH nearblocks AS ( ), nearblocks_metadata AS ( SELECT - VALUE :contract :: STRING AS contract_address, + VALUE :contract :: STRING AS near_token_contract, VALUE :decimals :: INT AS decimals, VALUE :name :: STRING AS NAME, - VALUE :symbol :: STRING AS symbol, - VALUE AS DATA + VALUE :symbol :: STRING AS symbol FROM nearblocks, LATERAL FLATTEN( @@ -46,14 +45,36 @@ nearblocks_metadata AS ( ), omni AS ( SELECT - omni_address, - contract_address + omni_asset_identifier AS asset_identifier, + SPLIT_PART(omni_asset_identifier, ':', 1) :: STRING AS source_chain, + SPLIT_PART(omni_asset_identifier, ':', 2) :: STRING AS crosschain_token_contract, + contract_address AS near_token_contract FROM {{ ref('silver__omni_metadata')}} {% if is_incremental() %} WHERE - inserted_timestamp >= ( + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} +), +omni_unmapped AS ( + SELECT + contract_address AS asset_identifier, + SPLIT_PART(contract_address, ':', 1) :: STRING AS source_chain, + SPLIT_PART(contract_address, ':', 2) :: STRING AS crosschain_token_contract + FROM + {{ ref('streamline__omni_tokenlist')}} + WHERE + source_chain_id IN ('near', 'sol') + + {% if is_incremental() %} + WHERE + modified_timestamp >= ( SELECT MAX(modified_timestamp) FROM @@ -63,8 +84,14 @@ omni AS ( ), defuse AS ( SELECT - defuse_asset_identifier AS defuse_address, - near_token_id AS contract_address, + defuse_asset_identifier AS asset_identifier, + CASE + WHEN SPLIT_PART(defuse_asset_identifier, ':', 0) = 'near' THEN 'near' + WHEN SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) = 'native' THEN SPLIT_PART(near_token_id, '.', 0) :: STRING + ELSE SPLIT_PART(near_token_id, '-', 0) :: STRING + END AS source_chain, + SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) AS crosschain_token_contract, + near_token_id AS near_token_contract, asset_name AS symbol, decimals FROM @@ -72,7 +99,7 @@ defuse AS ( {% if is_incremental() %} WHERE - inserted_timestamp >= ( + modified_timestamp >= ( SELECT MAX(modified_timestamp) FROM @@ -83,9 +110,10 @@ defuse AS ( final AS ( -- Omni SELECT - o.omni_address :: STRING AS raw_token_id, - SPLIT_PART(o.omni_address, ':', ARRAY_SIZE(SPLIT(o.omni_address, ':'))) AS token_id, - o.contract_address :: STRING AS contract_address, + o.asset_identifier, + o.source_chain, + o.crosschain_token_contract, + o.near_token_contract, n.decimals :: INT AS decimals, n.name :: STRING AS name, n.symbol :: STRING AS symbol, @@ -93,15 +121,38 @@ final AS ( FROM omni o LEFT JOIN nearblocks_metadata n - ON o.contract_address = n.contract_address + ON o.near_token_contract = n.near_token_contract UNION ALL + -- Omni unmapped + SELECT + o.asset_identifier, + o.source_chain, + o.crosschain_token_contract, + n.near_token_contract, + COALESCE(n.decimals, c.decimals) :: INT AS decimals, + COALESCE(n.name, c.name) :: STRING AS name, + COALESCE(n.symbol, c.symbol) :: STRING AS symbol, + 'omni_unmapped' AS source + FROM + omni_unmapped o + LEFT JOIN nearblocks_metadata n + ON o.crosschain_token_contract = n.near_token_contract + LEFT JOIN {{ source('crosschain_silver', 'complete_token_asset_metadata')}} c + ON o.crosschain_token_contract = c.token_address + AND c.blockchain = 'solana' + -- note, this does not give use the Near token contract. + -- could join on symbol, but some symbols have multiple contract records as symbol is not unique + + UNION ALL + -- Nearblocks SELECT - n.contract_address :: STRING AS token_id, - n.contract_address :: STRING AS raw_token_id, - n.contract_address :: STRING AS contract_address, + n.near_token_contract :: STRING AS asset_identifier, + 'near' AS source_chain, + n.near_token_contract :: STRING AS crosschain_token_contract, + n.near_token_contract, n.decimals :: INT AS decimals, n.name :: STRING AS name, n.symbol :: STRING AS symbol, @@ -113,31 +164,35 @@ final AS ( -- Defuse SELECT - d.defuse_address :: STRING AS token_id, - SPLIT_PART(d.defuse_address, ':', ARRAY_SIZE(SPLIT(d.defuse_address, ':'))) AS token_id, - d.contract_address :: STRING AS contract_address, + d.asset_identifier, + d.source_chain, + d.crosschain_token_contract, + d.near_token_contract, d.decimals :: INT AS decimals, - NULL :: STRING AS name, + n.name :: STRING AS name, d.symbol :: STRING AS symbol, 'defuse' AS source FROM defuse d + LEFT JOIN nearblocks_metadata n + ON d.near_token_contract = n.near_token_contract ) SELECT - contract_address, - raw_token_id, - token_id, + asset_identifier, + source_chain, + crosschain_token_contract, + near_token_contract, decimals, name, symbol, source as metadata_source, - {{ dbt_utils.generate_surrogate_key(['token_id', 'contract_address']) }} AS ft_contract_metadata_id, + {{ dbt_utils.generate_surrogate_key(['asset_identifier']) }} AS ft_contract_metadata_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM final qualify ROW_NUMBER() over ( - PARTITION BY token_id + PARTITION BY asset_identifier ORDER BY modified_timestamp DESC ) = 1 diff --git a/models/silver/labels/silver__ft_contract_metadata.yml b/models/silver/labels/silver__ft_contract_metadata.yml index 22ef9e3..2584a1b 100644 --- a/models/silver/labels/silver__ft_contract_metadata.yml +++ b/models/silver/labels/silver__ft_contract_metadata.yml @@ -11,44 +11,33 @@ models: interval: 8 columns: - - name: CONTRACT_ADDRESS - description: "{{ doc('contract_address')}}" + - name: ASSET_IDENTIFIER + description: "{{ doc('asset_identifier') }}" tests: - not_null - unique - - dbt_expectations.expect_column_values_to_be_of_type: - column_type: VARCHAR - + - name: SOURCE_CHAIN + description: "{{ doc('source_chain') }}" + - name: CROSSCHAIN_TOKEN_CONTRACT + description: "{{ doc('crosschain_token_contract') }}" + - name: NEAR_TOKEN_CONTRACT + description: "{{ doc('near_token_contract') }}" - name: DECIMALS - description: "{{ doc('decimals')}}" - tests: - - not_null - + description: "{{ doc('decimals') }}" - name: NAME - description: "{{ doc('name')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_of_type: - column_type: VARCHAR - + description: "{{ doc('name') }}" - name: SYMBOL - description: "{{ doc('symbol')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_of_type: - column_type: VARCHAR - - - name: DATA - description: "{{ doc('data')}}" - + description: "{{ doc('symbol') }}" + - name: METADATA_SOURCE + description: "Source of the metadata record (e.g., omni, nearblocks, defuse)." - name: FT_CONTRACT_METADATA_ID description: "{{doc('id')}}" - + tests: + - not_null + - unique - name: INSERTED_TIMESTAMP description: "{{doc('inserted_timestamp')}}" - - name: MODIFIED_TIMESTAMP description: "{{doc('modified_timestamp')}}" - - name: _INVOCATION_ID description: "{{doc('invocation_id')}}" diff --git a/models/silver/labels/silver__omni_metadata.sql b/models/silver/labels/silver__omni_metadata.sql index 197eb88..22f1b8a 100644 --- a/models/silver/labels/silver__omni_metadata.sql +++ b/models/silver/labels/silver__omni_metadata.sql @@ -1,23 +1,20 @@ {{ config( materialized = 'incremental', merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'omni_address', + unique_key = 'omni_asset_identifier', tags = ['scheduled_non_core'] ) }} WITH omni AS ( SELECT - VALUE:CONTRACT_ADDRESS::STRING AS omni_address, + VALUE:CONTRACT_ADDRESS::STRING AS omni_asset_identifier, VALUE:data:result:result::ARRAY AS result_array, DATA FROM - near_dev.bronze.omni_metadata - WHERE - typeof(DATA) != 'NULL_VALUE' - + {{ ref('bronze__omni_metadata') }} {% if is_incremental() %} - AND + WHERE _inserted_timestamp >= ( SELECT MAX(modified_timestamp) @@ -28,11 +25,11 @@ WITH omni AS ( ), try_decode_hex AS ( SELECT - omni_address, + omni_asset_identifier, b.value AS raw, b.index, LPAD(TRIM(to_char(b.value::INT, 'XXXXXXX'))::STRING, 2, '0') AS hex, - ROW_NUMBER() OVER (PARTITION BY omni_address, raw, index, hex ORDER BY omni_address) AS rn + ROW_NUMBER() OVER (PARTITION BY omni_asset_identifier, raw, index, hex ORDER BY omni_asset_identifier) AS rn FROM omni o, TABLE(FLATTEN(o.result_array, recursive => TRUE)) b WHERE IS_ARRAY(o.result_array) = TRUE @@ -40,27 +37,27 @@ try_decode_hex AS ( ), decoded_response AS ( SELECT - omni_address, + omni_asset_identifier, ARRAY_TO_STRING(ARRAY_AGG(hex) WITHIN GROUP (ORDER BY index ASC), '') AS decoded_response FROM try_decode_hex - GROUP BY omni_address, rn + GROUP BY omni_asset_identifier, rn HAVING rn = 1 ), conversion AS ( SELECT - omni_address, + omni_asset_identifier, TRIM(livequery.utils.udf_hex_to_string(decoded_response), '"') AS decoded_result FROM decoded_response ) SELECT - omni_address, + omni_asset_identifier, decoded_result AS contract_address, {{ dbt_utils.generate_surrogate_key( - ['omni_address'] + ['omni_asset_identifier'] ) }} AS omni_metadata_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM conversion -qualify(row_number() over (partition by omni_address order by inserted_timestamp asc)) = 1 +qualify(row_number() over (partition by omni_asset_identifier order by inserted_timestamp asc)) = 1 diff --git a/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql index 69318d3..00e8123 100644 --- a/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql +++ b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql @@ -9,7 +9,7 @@ WITH omni_token AS ( SELECT - DISTINCT lower(raw_token_id) AS contract_address + DISTINCT raw_token_id AS contract_address FROM {{ ref('silver__bridge_omni') }} @@ -23,6 +23,7 @@ WHERE modified_timestamp >= ( ) SELECT contract_address, + SPLIT_PART(contract_address, ':', 0) :: STRING AS source_chain_id, {{ dbt_utils.generate_surrogate_key( ['contract_address'] ) }} AS omni_tokenlist_id,