From fafc130bf562c4a588b640b60d29ca75c552a45c Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Thu, 22 May 2025 00:45:25 +0700 Subject: [PATCH 01/24] streamline (partial) + bronze --- models/bronze/bronze__omni_metadata.sql | 9 ++ .../complete/streamline__omni_complete.sql | 11 +++ .../realtime/streamline__omni_realtime.sql | 87 +++++++++++++++++++ .../omni/streamline__omni_tokenlist.sql | 39 +++++++++ 4 files changed, 146 insertions(+) create mode 100644 models/bronze/bronze__omni_metadata.sql create mode 100644 models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql create mode 100644 models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql create mode 100644 models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql diff --git a/models/bronze/bronze__omni_metadata.sql b/models/bronze/bronze__omni_metadata.sql new file mode 100644 index 0000000..f2023db --- /dev/null +++ b/models/bronze/bronze__omni_metadata.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['streamline_helper'] +) }} + +{{ streamline_external_table_FR_query_v2( + model = "omni_metadata", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )" +) }} diff --git a/models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql b/models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql new file mode 100644 index 0000000..632e176 --- /dev/null +++ b/models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql @@ -0,0 +1,11 @@ +-- depends_on: {{ ref('bronze__omni_metadata') }} +{{ config( + materialized = "incremental", + unique_key = "contract_address", + merge_exclude_columns = ["inserted_timestamp"], + tags = ['streamline_non_core'] +) }} + +SELECT + + diff --git a/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql b/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql new file mode 100644 index 0000000..222ae36 --- /dev/null +++ b/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql @@ -0,0 +1,87 @@ +-- depends_on: {{ ref('streamline__omni_tokenlist') }} +{{ config( + materialized = 'view', + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params = { + "external_table": "omni_metadata", + "sql_limit": "30", + "producer_batch_size": "5", + "worker_batch_size": "5", + "sql_source": "{{this.identifier}}" + } + ), + tags = ['streamline_non_core'] +) }} + +WITH omni_token AS ( + SELECT + contract_address + FROM + {{ ref('streamline__omni_tokenlist') }} +), +api_call AS ( + SELECT + contract_address, + response:data:result:result AS res_array + FROM ( + SELECT + contract_address, + NEAR.LIVE.UDF_API( + 'POST', + 'https://rpc.mainnet.near.org', + OBJECT_CONSTRUCT('Content-Type', 'application/json'), + OBJECT_CONSTRUCT( + 'jsonrpc', '2.0', + 'id', 'dontcare', + 'method', 'query', + 'params', OBJECT_CONSTRUCT( + 'request_type', 'call_function', + 'finality', 'final', + 'account_id', 'omni.bridge.near', + 'method_name', 'get_token_id', + 'args_base64', BASE64_ENCODE(OBJECT_CONSTRUCT('address', contract_address)::STRING) + ) + ) + ) AS response + FROM omni_token + ) +), +flat AS ( + SELECT + contract_address, + VALUE::INT AS byte_val, + LPAD(TO_CHAR(VALUE::INT, 'XX'), 2, '0') AS hex_val, + INDEX AS idx + FROM + api_call, + LATERAL FLATTEN(input => res_array) +), +agg AS ( + SELECT + contract_address, + LISTAGG(hex_val, '') WITHIN GROUP (ORDER BY idx) AS hex_string + FROM + flat + GROUP BY 1 +), +final AS ( + SELECT + contract_address, + hex_string, + TRY_PARSE_JSON( + livequery.utils.udf_hex_to_string(hex_string) + ) AS decoded_result + FROM + agg +) +SELECT + contract_address, + DATE_PART('EPOCH', SYSDATE()) :: INTEGER AS partition_key, + OBJECT_CONSTRUCT( + 'contract_address', contract_address, + 'decoded_result', decoded_result + ) AS request +FROM + final diff --git a/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql new file mode 100644 index 0000000..260fb20 --- /dev/null +++ b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql @@ -0,0 +1,39 @@ +{{ config ( + materialized = "incremental", + incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate","modified_timestamp::date"], + unique_key = "contract_address", + cluster_by = ['modified_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(contract_address)", + tags = ['streamline_non_core'] +) }} + +WITH omni_token AS ( + SELECT + DISTINCT lower(raw_token_id) AS contract_address + FROM + {{ ref('silver__bridge_omni') }} + WHERE + source_chain_id NOT IN ('near', 'sol') + +{% if is_incremental() %} +WHERE modified_timestamp >= ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01') + FROM + {{ this }}) +{% endif %} +) +SELECT + contract_address, + {{ dbt_utils.generate_surrogate_key( + ['contract_address'] + ) }} AS omni_tokenlist_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + omni_token +QUALIFY (ROW_NUMBER() OVER (PARTITION BY contract_address + ORDER BY + modified_timestamp ASC) = 1) From 4a6a90220fb0422103b03a00824c1123d521709a Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Fri, 23 May 2025 15:38:38 +0700 Subject: [PATCH 02/24] SL models --- models/sources.yml | 1 + .../complete/streamline__omni_complete.sql | 26 +++++- .../realtime/streamline__omni_realtime.sql | 83 ++++++------------- 3 files changed, 50 insertions(+), 60 deletions(-) diff --git a/models/sources.yml b/models/sources.yml index e9e4cd7..12dc903 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -43,3 +43,4 @@ sources: - name: chunks_v2 - name: transactions_v2 - name: nearblocks_ft_metadata + - name: omni_metadata diff --git a/models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql b/models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql index 632e176..799fca4 100644 --- a/models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql +++ b/models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql @@ -7,5 +7,29 @@ ) }} SELECT - + VALUE:CONTRACT_ADDRESS :: STRING AS contract_address, + partition_key, + _inserted_timestamp, + contract_address AS omni_complete_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('bronze__omni_metadata') }} +WHERE + typeof(DATA) != 'NULL_VALUE' +{% if is_incremental() %} +AND + _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01' :: timestamp_ntz + ) +{% endif %} + +qualify(row_number() over (partition by contract_address order by _inserted_timestamp desc)) = 1 \ No newline at end of file diff --git a/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql b/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql index 222ae36..6c9e8d4 100644 --- a/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql +++ b/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql @@ -16,72 +16,37 @@ ) }} WITH omni_token AS ( + SELECT contract_address FROM {{ ref('streamline__omni_tokenlist') }} -), -api_call AS ( + EXCEPT SELECT - contract_address, - response:data:result:result AS res_array - FROM ( - SELECT - contract_address, - NEAR.LIVE.UDF_API( - 'POST', - 'https://rpc.mainnet.near.org', - OBJECT_CONSTRUCT('Content-Type', 'application/json'), - OBJECT_CONSTRUCT( - 'jsonrpc', '2.0', - 'id', 'dontcare', - 'method', 'query', - 'params', OBJECT_CONSTRUCT( - 'request_type', 'call_function', - 'finality', 'final', - 'account_id', 'omni.bridge.near', - 'method_name', 'get_token_id', - 'args_base64', BASE64_ENCODE(OBJECT_CONSTRUCT('address', contract_address)::STRING) - ) - ) - ) AS response - FROM omni_token - ) -), -flat AS ( - SELECT - contract_address, - VALUE::INT AS byte_val, - LPAD(TO_CHAR(VALUE::INT, 'XX'), 2, '0') AS hex_val, - INDEX AS idx - FROM - api_call, - LATERAL FLATTEN(input => res_array) -), -agg AS ( - SELECT - contract_address, - LISTAGG(hex_val, '') WITHIN GROUP (ORDER BY idx) AS hex_string - FROM - flat - GROUP BY 1 -), -final AS ( - SELECT - contract_address, - hex_string, - TRY_PARSE_JSON( - livequery.utils.udf_hex_to_string(hex_string) - ) AS decoded_result - FROM - agg + contract_address + FROM + {{ ref('streamline__omni_complete')}} ) -SELECT +SELECT contract_address, DATE_PART('EPOCH', SYSDATE()) :: INTEGER AS partition_key, - OBJECT_CONSTRUCT( - 'contract_address', contract_address, - 'decoded_result', decoded_result + {{ target.database }}.LIVE.UDF_API( + 'POST', + '{Service}', + OBJECT_CONSTRUCT('Content-Type', 'application/json'), + OBJECT_CONSTRUCT( + 'jsonrpc', '2.0', + 'id', 'dontcare', + 'method', 'query', + 'params', OBJECT_CONSTRUCT( + 'request_type', 'call_function', + 'finality', 'final', + 'account_id', 'omni.bridge.near', + 'method_name', 'get_token_id', + 'args_base64', BASE64_ENCODE(OBJECT_CONSTRUCT('address', contract_address) :: STRING) + ) + ), + 'Vault/prod/near/quicknode/mainnet' ) AS request FROM - final + omni_token \ No newline at end of file From b516ebac3b9f79d90ff3188a254d17e932e64fa1 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Fri, 23 May 2025 18:16:42 +0700 Subject: [PATCH 03/24] omni integration (temp) --- .../labels/silver__ft_contract_metadata.sql | 73 +++++++++++++++---- .../silver/labels/silver__omni_metadata.sql | 66 +++++++++++++++++ 2 files changed, 126 insertions(+), 13 deletions(-) create mode 100644 models/silver/labels/silver__omni_metadata.sql diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index 6d363ae..23a77cb 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -1,4 +1,5 @@ -- depends on: {{ ref('bronze__nearblocks_ft_metadata')}} +-- depends on: {{ ref('bronze__omni_metadata')}} {{ config( materialized = 'incremental', @@ -9,7 +10,7 @@ ) }} -WITH bronze AS ( +WITH nearblocks AS ( SELECT VALUE :CONTRACT_ADDRESS :: STRING AS contract_address, @@ -29,7 +30,7 @@ WITH bronze AS ( ) {% endif %} ), -flatten_results AS ( +nearblocks_metadata AS ( SELECT VALUE :contract :: STRING AS contract_address, VALUE :decimals :: INT AS decimals, @@ -37,28 +38,74 @@ flatten_results AS ( VALUE :symbol :: STRING AS symbol, VALUE AS DATA FROM - bronze, + nearblocks, LATERAL FLATTEN( input => DATA :contracts ) +), +omni AS ( + SELECT + omni_address, + contract_address, + FROM + {{ ref('silver__omni_metadata')}} + + {% if is_incremental() %} + WHERE + _inserted_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} +), +final AS ( + -- Omni + SELECT + o.omni_address, + o.contract_address, + n.decimals, + n.name, + n.symbol, + n.data, + 'omni' AS source + FROM + omni o + LEFT JOIN nearblocks_metadata n + ON o.contract_address = n.contract_address + + UNION ALL + + -- Nearblocks + + SELECT + NULL AS omni_address, + n.contract_address, + n.decimals, + n.name, + n.symbol, + n.data, + 'nearblocks' AS source + FROM + nearblocks_metadata n + WHERE n.contract_address NOT IN (SELECT contract_address FROM omni) ) SELECT contract_address, + omni_address, decimals, - NAME, + name, symbol, - DATA, - {{ dbt_utils.generate_surrogate_key( - ['contract_address'] - ) }} AS ft_contract_metadata_id, + data, + source, + {{ dbt_utils.generate_surrogate_key(['omni_address', 'contract_address']) }} AS ft_contract_metadata_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id -FROM - flatten_results +FROM final qualify ROW_NUMBER() over ( - PARTITION BY contract_address - ORDER BY - modified_timestamp DESC + PARTITION BY omni_address, contract_address + ORDER BY modified_timestamp DESC ) = 1 diff --git a/models/silver/labels/silver__omni_metadata.sql b/models/silver/labels/silver__omni_metadata.sql new file mode 100644 index 0000000..f46fee7 --- /dev/null +++ b/models/silver/labels/silver__omni_metadata.sql @@ -0,0 +1,66 @@ +{{ config( + materialized = 'incremental', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'omni_address', + tags = ['scheduled_non_core'] +) }} + +WITH omni AS ( + + SELECT + VALUE:CONTRACT_ADDRESS::STRING AS omni_address, + DATA + FROM + {{ ref('bronze__omni_metadata')}} + WHERE + typeof(DATA) != 'NULL_VALUE' + + {% if is_incremental() %} + AND + _inserted_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} +), +flattened AS ( + SELECT + omni_address, + value::INT AS byte_val, + index + FROM + omni, + LATERAL FLATTEN(input => DATA:result) +), +hex_strings AS ( + SELECT + omni_address, + LISTAGG(LPAD(TO_CHAR(byte_val, 'XX'), 2, '0'), '') WITHIN GROUP (ORDER BY index) AS hex_string + FROM + flattened + GROUP BY + omni_address +), +decode AS ( + SELECT + omni_address, + TRY_PARSE_JSON( + livequery.utils.udf_hex_to_string(hex_string) + ) AS decoded_result + FROM + hex_strings +) +SELECT + omni_address, + decoded_result AS contract_address, + {{ dbt_utils.generate_surrogate_key( + ['omni_address'] + ) }} AS omni_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + decode +qualify(row_number() over (partition by omni_address order by inserted_timestamp asc)) = 1 From cd781b357fefde974f7220d2d0cae0ddbabf84fa Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Sat, 24 May 2025 11:44:40 +0700 Subject: [PATCH 04/24] omni integration --- .../labels/silver__ft_contract_metadata.sql | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index 23a77cb..2a0c6d1 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -3,7 +3,7 @@ {{ config( materialized = 'incremental', - unique_key = 'contract_address', + unique_key = ['omni_address', 'contract_address'], incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], tags = ['scheduled_non_core'] @@ -46,13 +46,12 @@ nearblocks_metadata AS ( omni AS ( SELECT omni_address, - contract_address, + contract_address FROM {{ ref('silver__omni_metadata')}} - {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + inserted_timestamp >= ( SELECT MAX(modified_timestamp) FROM @@ -63,13 +62,12 @@ omni AS ( final AS ( -- Omni SELECT - o.omni_address, - o.contract_address, - n.decimals, - n.name, - n.symbol, - n.data, - 'omni' AS source + o.omni_address::STRING AS omni_address, + o.contract_address::STRING AS contract_address, + n.decimals::INT AS decimals, + n.name::STRING AS name, + n.symbol::STRING AS symbol, + 'omni'::STRING AS source FROM omni o LEFT JOIN nearblocks_metadata n @@ -78,15 +76,13 @@ final AS ( UNION ALL -- Nearblocks - SELECT - NULL AS omni_address, - n.contract_address, - n.decimals, - n.name, - n.symbol, - n.data, - 'nearblocks' AS source + NULL::STRING AS omni_address, + n.contract_address::STRING AS contract_address, + n.decimals::INT AS decimals, + n.name::STRING AS name, + n.symbol::STRING AS symbol, + 'nearblocks'::STRING AS source FROM nearblocks_metadata n WHERE n.contract_address NOT IN (SELECT contract_address FROM omni) @@ -97,7 +93,6 @@ SELECT decimals, name, symbol, - data, source, {{ dbt_utils.generate_surrogate_key(['omni_address', 'contract_address']) }} AS ft_contract_metadata_id, SYSDATE() AS inserted_timestamp, From 2edf3fce4381b4277d2d127a04212dd8b8d90e80 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Mon, 26 May 2025 18:02:39 +0700 Subject: [PATCH 05/24] defuse integration --- .../labels/silver__ft_contract_metadata.sql | 58 +++++++++++++++---- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index 2a0c6d1..74489fa 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -1,5 +1,6 @@ -- depends on: {{ ref('bronze__nearblocks_ft_metadata')}} -- depends on: {{ ref('bronze__omni_metadata')}} +-- depends on: {{ ref('silver__defuse_tokens_metadata')}} {{ config( materialized = 'incremental', @@ -49,6 +50,26 @@ omni AS ( contract_address FROM {{ ref('silver__omni_metadata')}} + + {% if is_incremental() %} + WHERE + inserted_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} +), +defuse AS ( + SELECT + defuse_asset_identifier AS defuse_address, + near_token_id AS contract_address, + asset_name AS symbol, + decimals + FROM + {{ ref('silver__defuse_tokens_metadata')}} + {% if is_incremental() %} WHERE inserted_timestamp >= ( @@ -62,12 +83,12 @@ omni AS ( final AS ( -- Omni SELECT - o.omni_address::STRING AS omni_address, - o.contract_address::STRING AS contract_address, - n.decimals::INT AS decimals, - n.name::STRING AS name, - n.symbol::STRING AS symbol, - 'omni'::STRING AS source + o.omni_address :: STRING AS omni_address, + o.contract_address :: STRING AS contract_address, + n.decimals :: INT AS decimals, + n.name :: STRING AS name, + n.symbol :: STRING AS symbol, + 'omni' AS source FROM omni o LEFT JOIN nearblocks_metadata n @@ -77,15 +98,28 @@ final AS ( -- Nearblocks SELECT - NULL::STRING AS omni_address, - n.contract_address::STRING AS contract_address, - n.decimals::INT AS decimals, - n.name::STRING AS name, - n.symbol::STRING AS symbol, - 'nearblocks'::STRING AS source + NULL :: STRING AS omni_address, + n.contract_address :: STRING AS contract_address, + n.decimals :: INT AS decimals, + n.name :: STRING AS name, + n.symbol :: STRING AS symbol, + 'nearblocks' AS source FROM nearblocks_metadata n WHERE n.contract_address NOT IN (SELECT contract_address FROM omni) + + UNION ALL + + -- Defuse + SELECT + d.defuse_address :: STRING AS omni_address, + d.contract_address :: STRING AS contract_address, + d.decimals :: INT AS decimals, + NULL :: STRING AS name, + d.symbol :: STRING AS symbol, + 'defuse' AS source + FROM + defuse d ) SELECT contract_address, From fbf2e89cb29df8eca6874dc82700d59b865a12a4 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Mon, 26 May 2025 18:53:26 +0700 Subject: [PATCH 06/24] downstream model --- models/gold/defi/defi__ez_intents.sql | 52 ++++++++++++--------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/models/gold/defi/defi__ez_intents.sql b/models/gold/defi/defi__ez_intents.sql index d530519..f9a7a53 100644 --- a/models/gold/defi/defi__ez_intents.sql +++ b/models/gold/defi/defi__ez_intents.sql @@ -123,38 +123,32 @@ WITH intents AS ( ), labels AS ( SELECT - near_token_id AS contract_address_raw, - SPLIT( - defuse_asset_identifier, - ':' - ) [0] :: STRING AS ecosystem, - SPLIT( - defuse_asset_identifier, - ':' - ) [1] :: STRING AS chain_id, - SPLIT( - defuse_asset_identifier, - ':' - ) [2] :: STRING AS contract_address, + contract_address AS contract_address_raw, + CASE + WHEN source = 'defuse' THEN SPLIT( + omni_address, + ':' + ) [0] :: STRING + ELSE 'near' + END AS ecosystem, + CASE + WHEN source = 'defuse' THEN SPLIT( + omni_address, + ':' + ) [1] :: STRING + ELSE '397' + END AS chain_id, + CASE + WHEN source = 'defuse' THEN SPLIT( + omni_address, + ':' + ) [2] :: STRING + ELSE contract_address + END AS contract_address, asset_name AS symbol, decimals - FROM - {{ ref('silver__defuse_tokens_metadata') }} - UNION ALL - SELECT - contract_address AS contract_address_raw, - 'near' AS ecosystem, - '397' AS chain_id, - contract_address, - symbol, - decimals - FROM + FROM {{ ref('silver__ft_contract_metadata') }} - WHERE - contract_address not in ( - select distinct near_token_id - from {{ ref('silver__defuse_tokens_metadata') }} - ) ), prices AS ( SELECT From 9e982a4370d1dbacd154ab7a5834fef31e515f8c Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Wed, 28 May 2025 23:23:24 +0700 Subject: [PATCH 07/24] couple tweaks --- .../labels/silver__ft_contract_metadata.sql | 17 ++++++++--------- .../omni/realtime/streamline__omni_realtime.sql | 2 ++ .../omni/streamline__omni_tokenlist.sql | 3 --- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index 74489fa..df1a338 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -4,7 +4,7 @@ {{ config( materialized = 'incremental', - unique_key = ['omni_address', 'contract_address'], + unique_key = 'ft_contract_metadata_id', incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], tags = ['scheduled_non_core'] @@ -83,7 +83,7 @@ defuse AS ( final AS ( -- Omni SELECT - o.omni_address :: STRING AS omni_address, + o.omni_address :: STRING AS token_id, o.contract_address :: STRING AS contract_address, n.decimals :: INT AS decimals, n.name :: STRING AS name, @@ -98,7 +98,7 @@ final AS ( -- Nearblocks SELECT - NULL :: STRING AS omni_address, + n.contract_address :: STRING AS token_id, n.contract_address :: STRING AS contract_address, n.decimals :: INT AS decimals, n.name :: STRING AS name, @@ -106,13 +106,12 @@ final AS ( 'nearblocks' AS source FROM nearblocks_metadata n - WHERE n.contract_address NOT IN (SELECT contract_address FROM omni) UNION ALL -- Defuse SELECT - d.defuse_address :: STRING AS omni_address, + d.defuse_address :: STRING AS token_id, d.contract_address :: STRING AS contract_address, d.decimals :: INT AS decimals, NULL :: STRING AS name, @@ -123,18 +122,18 @@ final AS ( ) SELECT contract_address, - omni_address, + token_id, decimals, name, symbol, - source, - {{ dbt_utils.generate_surrogate_key(['omni_address', 'contract_address']) }} AS ft_contract_metadata_id, + source as metadata_source, + {{ dbt_utils.generate_surrogate_key(['token_id', 'contract_address']) }} AS ft_contract_metadata_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM final qualify ROW_NUMBER() over ( - PARTITION BY omni_address, contract_address + PARTITION BY token_id ORDER BY modified_timestamp DESC ) = 1 diff --git a/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql b/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql index 6c9e8d4..9b15656 100644 --- a/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql +++ b/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql @@ -21,6 +21,8 @@ WITH omni_token AS ( contract_address FROM {{ ref('streamline__omni_tokenlist') }} + WHERE + source_chain_id NOT IN ('near', 'sol') EXCEPT SELECT contract_address diff --git a/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql index 260fb20..69318d3 100644 --- a/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql +++ b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql @@ -4,7 +4,6 @@ incremental_predicates = ["dynamic_range_predicate","modified_timestamp::date"], unique_key = "contract_address", cluster_by = ['modified_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(contract_address)", tags = ['streamline_non_core'] ) }} @@ -13,8 +12,6 @@ WITH omni_token AS ( DISTINCT lower(raw_token_id) AS contract_address FROM {{ ref('silver__bridge_omni') }} - WHERE - source_chain_id NOT IN ('near', 'sol') {% if is_incremental() %} WHERE modified_timestamp >= ( From c63d8d839bdb62df0b699bf9546d3d859fb64c11 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Wed, 28 May 2025 23:29:12 +0700 Subject: [PATCH 08/24] rm downstream changes --- models/gold/defi/defi__ez_intents.sql | 54 +++++++++++++++------------ 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/models/gold/defi/defi__ez_intents.sql b/models/gold/defi/defi__ez_intents.sql index f9a7a53..31fb2dc 100644 --- a/models/gold/defi/defi__ez_intents.sql +++ b/models/gold/defi/defi__ez_intents.sql @@ -123,32 +123,38 @@ WITH intents AS ( ), labels AS ( SELECT - contract_address AS contract_address_raw, - CASE - WHEN source = 'defuse' THEN SPLIT( - omni_address, - ':' - ) [0] :: STRING - ELSE 'near' - END AS ecosystem, - CASE - WHEN source = 'defuse' THEN SPLIT( - omni_address, - ':' - ) [1] :: STRING - ELSE '397' - END AS chain_id, - CASE - WHEN source = 'defuse' THEN SPLIT( - omni_address, - ':' - ) [2] :: STRING - ELSE contract_address - END AS contract_address, + near_token_id AS contract_address_raw, + SPLIT( + defuse_asset_identifier, + ':' + ) [0] :: STRING AS ecosystem, + SPLIT( + defuse_asset_identifier, + ':' + ) [1] :: STRING AS chain_id, + SPLIT( + defuse_asset_identifier, + ':' + ) [2] :: STRING AS contract_address, asset_name AS symbol, decimals - FROM + FROM + {{ ref('silver__defuse_tokens_metadata') }} + UNION ALL + SELECT + contract_address AS contract_address_raw, + 'near' AS ecosystem, + '397' AS chain_id, + contract_address, + symbol, + decimals + FROM {{ ref('silver__ft_contract_metadata') }} + WHERE + contract_address not in ( + select distinct near_token_id + from {{ ref('silver__defuse_tokens_metadata') }} + ) ), prices AS ( SELECT @@ -271,4 +277,4 @@ FROM qualify(ROW_NUMBER() over (PARTITION BY ez_intents_id ORDER BY - price IS NOT NULL DESC, is_native DESC) = 1) + price IS NOT NULL DESC, is_native DESC) = 1) \ No newline at end of file From e7098f0f4e928132ecf42d1615ba25c270aa822c Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Thu, 29 May 2025 14:40:14 +0700 Subject: [PATCH 09/24] rework decoding logic --- .../silver/labels/silver__omni_metadata.sql | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/models/silver/labels/silver__omni_metadata.sql b/models/silver/labels/silver__omni_metadata.sql index f46fee7..197eb88 100644 --- a/models/silver/labels/silver__omni_metadata.sql +++ b/models/silver/labels/silver__omni_metadata.sql @@ -9,9 +9,10 @@ WITH omni AS ( SELECT VALUE:CONTRACT_ADDRESS::STRING AS omni_address, + VALUE:data:result:result::ARRAY AS result_array, DATA FROM - {{ ref('bronze__omni_metadata')}} + near_dev.bronze.omni_metadata WHERE typeof(DATA) != 'NULL_VALUE' @@ -25,32 +26,31 @@ WITH omni AS ( ) {% endif %} ), -flattened AS ( - SELECT +try_decode_hex AS ( + SELECT omni_address, - value::INT AS byte_val, - index - FROM - omni, - LATERAL FLATTEN(input => DATA:result) + b.value AS raw, + b.index, + LPAD(TRIM(to_char(b.value::INT, 'XXXXXXX'))::STRING, 2, '0') AS hex, + ROW_NUMBER() OVER (PARTITION BY omni_address, raw, index, hex ORDER BY omni_address) AS rn + FROM omni o, + TABLE(FLATTEN(o.result_array, recursive => TRUE)) b + WHERE IS_ARRAY(o.result_array) = TRUE + ORDER BY 1, 3 ), -hex_strings AS ( - SELECT +decoded_response AS ( + SELECT omni_address, - LISTAGG(LPAD(TO_CHAR(byte_val, 'XX'), 2, '0'), '') WITHIN GROUP (ORDER BY index) AS hex_string - FROM - flattened - GROUP BY - omni_address + ARRAY_TO_STRING(ARRAY_AGG(hex) WITHIN GROUP (ORDER BY index ASC), '') AS decoded_response + FROM try_decode_hex + GROUP BY omni_address, rn + HAVING rn = 1 ), -decode AS ( +conversion AS ( SELECT omni_address, - TRY_PARSE_JSON( - livequery.utils.udf_hex_to_string(hex_string) - ) AS decoded_result - FROM - hex_strings + TRIM(livequery.utils.udf_hex_to_string(decoded_response), '"') AS decoded_result + FROM decoded_response ) SELECT omni_address, @@ -62,5 +62,5 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - decode + conversion qualify(row_number() over (partition by omni_address order by inserted_timestamp asc)) = 1 From 487f72fa0aaf25f7f226a80b74f9a817f4559ae4 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Thu, 29 May 2025 15:32:28 +0700 Subject: [PATCH 10/24] add token_id downstream --- models/gold/core/core__dim_ft_contract_metadata.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/gold/core/core__dim_ft_contract_metadata.sql b/models/gold/core/core__dim_ft_contract_metadata.sql index d226fba..04ddfe9 100644 --- a/models/gold/core/core__dim_ft_contract_metadata.sql +++ b/models/gold/core/core__dim_ft_contract_metadata.sql @@ -12,6 +12,7 @@ WITH ft_contract_metadata AS ( ) SELECT contract_address, + token_id NAME, symbol, decimals, From 237835fe1a4b024bc7ee82b56f9db8d084cb08d3 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Thu, 29 May 2025 16:50:40 +0700 Subject: [PATCH 11/24] add raw_token_id col --- models/silver/labels/silver__ft_contract_metadata.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index df1a338..e7735dd 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -83,7 +83,8 @@ defuse AS ( final AS ( -- Omni SELECT - o.omni_address :: STRING AS token_id, + o.omni_address :: STRING AS raw_token_id, + SPLIT_PART(o.omni_address, ':', ARRAY_SIZE(SPLIT(o.omni_address, ':'))) AS token_id, o.contract_address :: STRING AS contract_address, n.decimals :: INT AS decimals, n.name :: STRING AS name, @@ -99,6 +100,7 @@ final AS ( -- Nearblocks SELECT n.contract_address :: STRING AS token_id, + n.contract_address :: STRING AS raw_token_id, n.contract_address :: STRING AS contract_address, n.decimals :: INT AS decimals, n.name :: STRING AS name, @@ -112,6 +114,7 @@ final AS ( -- Defuse SELECT d.defuse_address :: STRING AS token_id, + SPLIT_PART(d.defuse_address, ':', ARRAY_SIZE(SPLIT(d.defuse_address, ':'))) AS token_id, d.contract_address :: STRING AS contract_address, d.decimals :: INT AS decimals, NULL :: STRING AS name, @@ -122,6 +125,7 @@ final AS ( ) SELECT contract_address, + raw_token_id, token_id, decimals, name, From e7ba77e0ddf8c1aee6b4f523e009d87644c0a10b Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Thu, 29 May 2025 18:46:53 +0700 Subject: [PATCH 12/24] downstream dependencies --- models/gold/core/core__dim_ft_contract_metadata.sql | 3 ++- models/silver/defi/lending/silver__burrow_lending.sql | 2 +- models/silver/prices/silver__complete_token_prices.sql | 2 +- models/silver/transfers/bridging/silver__bridge_allbridge.sql | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/models/gold/core/core__dim_ft_contract_metadata.sql b/models/gold/core/core__dim_ft_contract_metadata.sql index 04ddfe9..b4963ea 100644 --- a/models/gold/core/core__dim_ft_contract_metadata.sql +++ b/models/gold/core/core__dim_ft_contract_metadata.sql @@ -12,7 +12,8 @@ WITH ft_contract_metadata AS ( ) SELECT contract_address, - token_id + raw_token_id, + token_id, NAME, symbol, decimals, diff --git a/models/silver/defi/lending/silver__burrow_lending.sql b/models/silver/defi/lending/silver__burrow_lending.sql index 720049d..08546d6 100644 --- a/models/silver/defi/lending/silver__burrow_lending.sql +++ b/models/silver/defi/lending/silver__burrow_lending.sql @@ -6,7 +6,7 @@ WITH metadata AS ( SELECT - contract_address, + token_id as contract_address, NAME, symbol, decimals diff --git a/models/silver/prices/silver__complete_token_prices.sql b/models/silver/prices/silver__complete_token_prices.sql index 8e71778..760c2ec 100644 --- a/models/silver/prices/silver__complete_token_prices.sql +++ b/models/silver/prices/silver__complete_token_prices.sql @@ -86,4 +86,4 @@ FROM complete_token_prices p LEFT JOIN {{ ref('silver__ft_contract_metadata') }} ft - ON p.token_address = ft.contract_address + ON p.token_address = ft.token_id diff --git a/models/silver/transfers/bridging/silver__bridge_allbridge.sql b/models/silver/transfers/bridging/silver__bridge_allbridge.sql index 9214ca2..d5a13be 100644 --- a/models/silver/transfers/bridging/silver__bridge_allbridge.sql +++ b/models/silver/transfers/bridging/silver__bridge_allbridge.sql @@ -41,7 +41,7 @@ WITH functioncall AS ( ), metadata AS ( SELECT - contract_address, + token_id as contract_address, NAME, symbol, decimals From d02597962ad2bee515268ef0028e16c09c4e1e15 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Fri, 30 May 2025 01:09:36 +0700 Subject: [PATCH 13/24] gold dependencies --- models/gold/core/core__ez_token_transfers.sql | 2 +- models/gold/defi/defi__ez_bridge_activity.sql | 2 +- models/gold/defi/defi__ez_dex_swaps.sql | 2 +- models/gold/defi/defi__ez_intents.sql | 4 ++-- models/gold/defi/defi__ez_lending.sql | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index 24168a8..c95dbc6 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -100,7 +100,7 @@ FROM ASOF JOIN hourly_prices p MATCH_CONDITION (t.block_timestamp >= p.HOUR) ON (t.contract_address = p.token_address) - LEFT JOIN {{ ref('silver__ft_contract_metadata') }} C USING (contract_address) + LEFT JOIN {{ ref('silver__ft_contract_metadata') }} C ON (t.contract_address = C.token_id) {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} diff --git a/models/gold/defi/defi__ez_bridge_activity.sql b/models/gold/defi/defi__ez_bridge_activity.sql index 12e4423..700f03b 100644 --- a/models/gold/defi/defi__ez_bridge_activity.sql +++ b/models/gold/defi/defi__ez_bridge_activity.sql @@ -31,7 +31,7 @@ WITH fact_bridging AS ( ), labels AS ( SELECT - contract_address, + token_id AS contract_address, name, symbol, decimals diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql index 9301b4b..119decb 100644 --- a/models/gold/defi/defi__ez_dex_swaps.sql +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -29,7 +29,7 @@ WITH dex_swaps AS ( ), labels AS ( SELECT - contract_address, + token_id AS contract_address, NAME, symbol, decimals diff --git a/models/gold/defi/defi__ez_intents.sql b/models/gold/defi/defi__ez_intents.sql index 31fb2dc..5fe0e8b 100644 --- a/models/gold/defi/defi__ez_intents.sql +++ b/models/gold/defi/defi__ez_intents.sql @@ -142,10 +142,10 @@ labels AS ( {{ ref('silver__defuse_tokens_metadata') }} UNION ALL SELECT - contract_address AS contract_address_raw, + token_id AS contract_address_raw, 'near' AS ecosystem, '397' AS chain_id, - contract_address, + token_id AS contract_address, symbol, decimals FROM diff --git a/models/gold/defi/defi__ez_lending.sql b/models/gold/defi/defi__ez_lending.sql index 6a71f36..9e9c7c0 100644 --- a/models/gold/defi/defi__ez_lending.sql +++ b/models/gold/defi/defi__ez_lending.sql @@ -26,7 +26,7 @@ WITH lending AS ( ), labels AS ( SELECT - contract_address, + token_id AS contract_address, NAME, symbol, IFF( From 5a2ebc96ccfc3382cd4e8526361605915ab054d8 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 2 Jun 2025 11:02:26 -0600 Subject: [PATCH 14/24] upd ft model --- models/descriptions/asset_identifier.md | 5 + .../descriptions/crosschain_token_contract.md | 5 + models/descriptions/near_token_contract.md | 5 + .../core/core__dim_ft_contract_metadata.sql | 19 ++-- .../core/core__dim_ft_contract_metadata.yml | 40 +++---- .../labels/silver__ft_contract_metadata.sql | 105 +++++++++++++----- .../labels/silver__ft_contract_metadata.yml | 43 +++---- .../silver/labels/silver__omni_metadata.sql | 27 ++--- .../omni/streamline__omni_tokenlist.sql | 3 +- 9 files changed, 146 insertions(+), 106 deletions(-) create mode 100644 models/descriptions/asset_identifier.md create mode 100644 models/descriptions/crosschain_token_contract.md create mode 100644 models/descriptions/near_token_contract.md diff --git a/models/descriptions/asset_identifier.md b/models/descriptions/asset_identifier.md new file mode 100644 index 0000000..2f074f8 --- /dev/null +++ b/models/descriptions/asset_identifier.md @@ -0,0 +1,5 @@ +{% docs asset_identifier %} + +The onchain representation of a token id, which may include source chain metadata if involved in a crosschain bridge event. Unique per token in the FT contract metadata table. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/crosschain_token_contract.md b/models/descriptions/crosschain_token_contract.md new file mode 100644 index 0000000..2dfa559 --- /dev/null +++ b/models/descriptions/crosschain_token_contract.md @@ -0,0 +1,5 @@ +{% docs crosschain_token_contract %} + +The contract address of the token on the source chain, where applicable. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/near_token_contract.md b/models/descriptions/near_token_contract.md new file mode 100644 index 0000000..039a08b --- /dev/null +++ b/models/descriptions/near_token_contract.md @@ -0,0 +1,5 @@ +{% docs near_token_contract %} + +The contract address of the token deployed to NEAR. + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/core/core__dim_ft_contract_metadata.sql b/models/gold/core/core__dim_ft_contract_metadata.sql index b4963ea..7ec50b3 100644 --- a/models/gold/core/core__dim_ft_contract_metadata.sql +++ b/models/gold/core/core__dim_ft_contract_metadata.sql @@ -3,23 +3,18 @@ tags = ['scheduled_non_core'] ) }} -WITH ft_contract_metadata AS ( - - SELECT - * - FROM - {{ ref('silver__ft_contract_metadata') }} -) SELECT - contract_address, - raw_token_id, - token_id, + asset_identifier, + source_chain, + crosschain_token_contract, + near_token_contract AS contract_address, NAME, symbol, decimals, - DATA, ft_contract_metadata_id AS dim_ft_contract_metadata_id, inserted_timestamp, modified_timestamp FROM - ft_contract_metadata + {{ ref('silver__ft_contract_metadata') }} +WHERE + name is not null OR symbol is not null diff --git a/models/gold/core/core__dim_ft_contract_metadata.yml b/models/gold/core/core__dim_ft_contract_metadata.yml index d193f77..d738e73 100644 --- a/models/gold/core/core__dim_ft_contract_metadata.yml +++ b/models/gold/core/core__dim_ft_contract_metadata.yml @@ -6,46 +6,34 @@ models: Fungible Token contract metadata provided by the Nearblocks NFT endpoint. columns: - - name: CONTRACT_ADDRESS - description: "{{ doc('contract_address')}}" + - name: ASSET_IDENTIFIER + description: "{{ doc('asset_identifier') }}" tests: - not_null - + - unique + - name: SOURCE_CHAIN + description: "{{ doc('source_chain') }}" + - name: CROSSCHAIN_TOKEN_CONTRACT + description: "{{ doc('crosschain_token_contract') }}" + - name: CONTRACT_ADDRESS + description: "{{ doc('contract_address') }}" - name: NAME - description: "{{ doc('name')}}" - tests: + description: "{{ doc('name') }}" + tests: - not_null - - name: SYMBOL - description: "{{ doc('symbol')}}" - tests: + description: "{{ doc('symbol') }}" + tests: - not_null - - - name: DATA - description: "{{ doc('data')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - - OBJECT - - ARRAY - - name: DECIMALS - description: "{{ doc('decimals')}}" - - - name: DATA - description: "{{ doc('data')}}" - + description: "{{ doc('decimals') }}" - name: DIM_FT_CONTRACT_METADATA_ID description: "{{doc('id')}}" tests: - not_null - unique: where: inserted_timestamp >= SYSDATE() - INTERVAL '{{ var('DBT_TEST_LOOKBACK_DAYS', 14) }} days' - - name: INSERTED_TIMESTAMP description: "{{doc('inserted_timestamp')}}" - - name: MODIFIED_TIMESTAMP description: "{{doc('modified_timestamp')}}" diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index e7735dd..4547a30 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -33,11 +33,10 @@ WITH nearblocks AS ( ), nearblocks_metadata AS ( SELECT - VALUE :contract :: STRING AS contract_address, + VALUE :contract :: STRING AS near_token_contract, VALUE :decimals :: INT AS decimals, VALUE :name :: STRING AS NAME, - VALUE :symbol :: STRING AS symbol, - VALUE AS DATA + VALUE :symbol :: STRING AS symbol FROM nearblocks, LATERAL FLATTEN( @@ -46,14 +45,36 @@ nearblocks_metadata AS ( ), omni AS ( SELECT - omni_address, - contract_address + omni_asset_identifier AS asset_identifier, + SPLIT_PART(omni_asset_identifier, ':', 1) :: STRING AS source_chain, + SPLIT_PART(omni_asset_identifier, ':', 2) :: STRING AS crosschain_token_contract, + contract_address AS near_token_contract FROM {{ ref('silver__omni_metadata')}} {% if is_incremental() %} WHERE - inserted_timestamp >= ( + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} +), +omni_unmapped AS ( + SELECT + contract_address AS asset_identifier, + SPLIT_PART(contract_address, ':', 1) :: STRING AS source_chain, + SPLIT_PART(contract_address, ':', 2) :: STRING AS crosschain_token_contract + FROM + {{ ref('streamline__omni_tokenlist')}} + WHERE + source_chain_id IN ('near', 'sol') + + {% if is_incremental() %} + WHERE + modified_timestamp >= ( SELECT MAX(modified_timestamp) FROM @@ -63,8 +84,14 @@ omni AS ( ), defuse AS ( SELECT - defuse_asset_identifier AS defuse_address, - near_token_id AS contract_address, + defuse_asset_identifier AS asset_identifier, + CASE + WHEN SPLIT_PART(defuse_asset_identifier, ':', 0) = 'near' THEN 'near' + WHEN SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) = 'native' THEN SPLIT_PART(near_token_id, '.', 0) :: STRING + ELSE SPLIT_PART(near_token_id, '-', 0) :: STRING + END AS source_chain, + SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) AS crosschain_token_contract, + near_token_id AS near_token_contract, asset_name AS symbol, decimals FROM @@ -72,7 +99,7 @@ defuse AS ( {% if is_incremental() %} WHERE - inserted_timestamp >= ( + modified_timestamp >= ( SELECT MAX(modified_timestamp) FROM @@ -83,9 +110,10 @@ defuse AS ( final AS ( -- Omni SELECT - o.omni_address :: STRING AS raw_token_id, - SPLIT_PART(o.omni_address, ':', ARRAY_SIZE(SPLIT(o.omni_address, ':'))) AS token_id, - o.contract_address :: STRING AS contract_address, + o.asset_identifier, + o.source_chain, + o.crosschain_token_contract, + o.near_token_contract, n.decimals :: INT AS decimals, n.name :: STRING AS name, n.symbol :: STRING AS symbol, @@ -93,15 +121,38 @@ final AS ( FROM omni o LEFT JOIN nearblocks_metadata n - ON o.contract_address = n.contract_address + ON o.near_token_contract = n.near_token_contract UNION ALL + -- Omni unmapped + SELECT + o.asset_identifier, + o.source_chain, + o.crosschain_token_contract, + n.near_token_contract, + COALESCE(n.decimals, c.decimals) :: INT AS decimals, + COALESCE(n.name, c.name) :: STRING AS name, + COALESCE(n.symbol, c.symbol) :: STRING AS symbol, + 'omni_unmapped' AS source + FROM + omni_unmapped o + LEFT JOIN nearblocks_metadata n + ON o.crosschain_token_contract = n.near_token_contract + LEFT JOIN {{ source('crosschain_silver', 'complete_token_asset_metadata')}} c + ON o.crosschain_token_contract = c.token_address + AND c.blockchain = 'solana' + -- note, this does not give use the Near token contract. + -- could join on symbol, but some symbols have multiple contract records as symbol is not unique + + UNION ALL + -- Nearblocks SELECT - n.contract_address :: STRING AS token_id, - n.contract_address :: STRING AS raw_token_id, - n.contract_address :: STRING AS contract_address, + n.near_token_contract :: STRING AS asset_identifier, + 'near' AS source_chain, + n.near_token_contract :: STRING AS crosschain_token_contract, + n.near_token_contract, n.decimals :: INT AS decimals, n.name :: STRING AS name, n.symbol :: STRING AS symbol, @@ -113,31 +164,35 @@ final AS ( -- Defuse SELECT - d.defuse_address :: STRING AS token_id, - SPLIT_PART(d.defuse_address, ':', ARRAY_SIZE(SPLIT(d.defuse_address, ':'))) AS token_id, - d.contract_address :: STRING AS contract_address, + d.asset_identifier, + d.source_chain, + d.crosschain_token_contract, + d.near_token_contract, d.decimals :: INT AS decimals, - NULL :: STRING AS name, + n.name :: STRING AS name, d.symbol :: STRING AS symbol, 'defuse' AS source FROM defuse d + LEFT JOIN nearblocks_metadata n + ON d.near_token_contract = n.near_token_contract ) SELECT - contract_address, - raw_token_id, - token_id, + asset_identifier, + source_chain, + crosschain_token_contract, + near_token_contract, decimals, name, symbol, source as metadata_source, - {{ dbt_utils.generate_surrogate_key(['token_id', 'contract_address']) }} AS ft_contract_metadata_id, + {{ dbt_utils.generate_surrogate_key(['asset_identifier']) }} AS ft_contract_metadata_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM final qualify ROW_NUMBER() over ( - PARTITION BY token_id + PARTITION BY asset_identifier ORDER BY modified_timestamp DESC ) = 1 diff --git a/models/silver/labels/silver__ft_contract_metadata.yml b/models/silver/labels/silver__ft_contract_metadata.yml index 22ef9e3..2584a1b 100644 --- a/models/silver/labels/silver__ft_contract_metadata.yml +++ b/models/silver/labels/silver__ft_contract_metadata.yml @@ -11,44 +11,33 @@ models: interval: 8 columns: - - name: CONTRACT_ADDRESS - description: "{{ doc('contract_address')}}" + - name: ASSET_IDENTIFIER + description: "{{ doc('asset_identifier') }}" tests: - not_null - unique - - dbt_expectations.expect_column_values_to_be_of_type: - column_type: VARCHAR - + - name: SOURCE_CHAIN + description: "{{ doc('source_chain') }}" + - name: CROSSCHAIN_TOKEN_CONTRACT + description: "{{ doc('crosschain_token_contract') }}" + - name: NEAR_TOKEN_CONTRACT + description: "{{ doc('near_token_contract') }}" - name: DECIMALS - description: "{{ doc('decimals')}}" - tests: - - not_null - + description: "{{ doc('decimals') }}" - name: NAME - description: "{{ doc('name')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_of_type: - column_type: VARCHAR - + description: "{{ doc('name') }}" - name: SYMBOL - description: "{{ doc('symbol')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_of_type: - column_type: VARCHAR - - - name: DATA - description: "{{ doc('data')}}" - + description: "{{ doc('symbol') }}" + - name: METADATA_SOURCE + description: "Source of the metadata record (e.g., omni, nearblocks, defuse)." - name: FT_CONTRACT_METADATA_ID description: "{{doc('id')}}" - + tests: + - not_null + - unique - name: INSERTED_TIMESTAMP description: "{{doc('inserted_timestamp')}}" - - name: MODIFIED_TIMESTAMP description: "{{doc('modified_timestamp')}}" - - name: _INVOCATION_ID description: "{{doc('invocation_id')}}" diff --git a/models/silver/labels/silver__omni_metadata.sql b/models/silver/labels/silver__omni_metadata.sql index 197eb88..22f1b8a 100644 --- a/models/silver/labels/silver__omni_metadata.sql +++ b/models/silver/labels/silver__omni_metadata.sql @@ -1,23 +1,20 @@ {{ config( materialized = 'incremental', merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'omni_address', + unique_key = 'omni_asset_identifier', tags = ['scheduled_non_core'] ) }} WITH omni AS ( SELECT - VALUE:CONTRACT_ADDRESS::STRING AS omni_address, + VALUE:CONTRACT_ADDRESS::STRING AS omni_asset_identifier, VALUE:data:result:result::ARRAY AS result_array, DATA FROM - near_dev.bronze.omni_metadata - WHERE - typeof(DATA) != 'NULL_VALUE' - + {{ ref('bronze__omni_metadata') }} {% if is_incremental() %} - AND + WHERE _inserted_timestamp >= ( SELECT MAX(modified_timestamp) @@ -28,11 +25,11 @@ WITH omni AS ( ), try_decode_hex AS ( SELECT - omni_address, + omni_asset_identifier, b.value AS raw, b.index, LPAD(TRIM(to_char(b.value::INT, 'XXXXXXX'))::STRING, 2, '0') AS hex, - ROW_NUMBER() OVER (PARTITION BY omni_address, raw, index, hex ORDER BY omni_address) AS rn + ROW_NUMBER() OVER (PARTITION BY omni_asset_identifier, raw, index, hex ORDER BY omni_asset_identifier) AS rn FROM omni o, TABLE(FLATTEN(o.result_array, recursive => TRUE)) b WHERE IS_ARRAY(o.result_array) = TRUE @@ -40,27 +37,27 @@ try_decode_hex AS ( ), decoded_response AS ( SELECT - omni_address, + omni_asset_identifier, ARRAY_TO_STRING(ARRAY_AGG(hex) WITHIN GROUP (ORDER BY index ASC), '') AS decoded_response FROM try_decode_hex - GROUP BY omni_address, rn + GROUP BY omni_asset_identifier, rn HAVING rn = 1 ), conversion AS ( SELECT - omni_address, + omni_asset_identifier, TRIM(livequery.utils.udf_hex_to_string(decoded_response), '"') AS decoded_result FROM decoded_response ) SELECT - omni_address, + omni_asset_identifier, decoded_result AS contract_address, {{ dbt_utils.generate_surrogate_key( - ['omni_address'] + ['omni_asset_identifier'] ) }} AS omni_metadata_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM conversion -qualify(row_number() over (partition by omni_address order by inserted_timestamp asc)) = 1 +qualify(row_number() over (partition by omni_asset_identifier order by inserted_timestamp asc)) = 1 diff --git a/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql index 69318d3..00e8123 100644 --- a/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql +++ b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql @@ -9,7 +9,7 @@ WITH omni_token AS ( SELECT - DISTINCT lower(raw_token_id) AS contract_address + DISTINCT raw_token_id AS contract_address FROM {{ ref('silver__bridge_omni') }} @@ -23,6 +23,7 @@ WHERE modified_timestamp >= ( ) SELECT contract_address, + SPLIT_PART(contract_address, ':', 0) :: STRING AS source_chain_id, {{ dbt_utils.generate_surrogate_key( ['contract_address'] ) }} AS omni_tokenlist_id, From f8a9482a957b11aa698449fa3b90a23ca517e16e Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 2 Jun 2025 11:06:59 -0600 Subject: [PATCH 15/24] upd refs to silver__ft_contract_metadata --- models/gold/core/core__ez_token_transfers.sql | 2 +- models/gold/defi/defi__ez_bridge_activity.sql | 2 +- models/gold/defi/defi__ez_dex_swaps.sql | 2 +- models/gold/defi/defi__ez_intents.sql | 4 ++-- models/gold/defi/defi__ez_lending.sql | 2 +- models/silver/defi/lending/silver__burrow_lending.sql | 2 +- models/silver/prices/silver__complete_token_prices.sql | 2 +- models/silver/transfers/bridging/silver__bridge_allbridge.sql | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index 092f767..5394d6b 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -102,7 +102,7 @@ FROM ASOF JOIN hourly_prices p MATCH_CONDITION (t.block_timestamp >= p.HOUR) ON (t.contract_address = p.token_address) - LEFT JOIN {{ ref('silver__ft_contract_metadata') }} C ON (t.contract_address = C.token_id) + LEFT JOIN {{ ref('silver__ft_contract_metadata') }} C ON (t.contract_address = C.asset_identifier) {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} diff --git a/models/gold/defi/defi__ez_bridge_activity.sql b/models/gold/defi/defi__ez_bridge_activity.sql index 700f03b..51b2384 100644 --- a/models/gold/defi/defi__ez_bridge_activity.sql +++ b/models/gold/defi/defi__ez_bridge_activity.sql @@ -31,7 +31,7 @@ WITH fact_bridging AS ( ), labels AS ( SELECT - token_id AS contract_address, + asset_identifier AS contract_address, name, symbol, decimals diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql index 119decb..55d23dc 100644 --- a/models/gold/defi/defi__ez_dex_swaps.sql +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -29,7 +29,7 @@ WITH dex_swaps AS ( ), labels AS ( SELECT - token_id AS contract_address, + asset_identifier AS contract_address, NAME, symbol, decimals diff --git a/models/gold/defi/defi__ez_intents.sql b/models/gold/defi/defi__ez_intents.sql index 5fe0e8b..d689767 100644 --- a/models/gold/defi/defi__ez_intents.sql +++ b/models/gold/defi/defi__ez_intents.sql @@ -142,10 +142,10 @@ labels AS ( {{ ref('silver__defuse_tokens_metadata') }} UNION ALL SELECT - token_id AS contract_address_raw, + asset_identifier AS contract_address_raw, 'near' AS ecosystem, '397' AS chain_id, - token_id AS contract_address, + asset_identifier AS contract_address, symbol, decimals FROM diff --git a/models/gold/defi/defi__ez_lending.sql b/models/gold/defi/defi__ez_lending.sql index 9e9c7c0..e56154e 100644 --- a/models/gold/defi/defi__ez_lending.sql +++ b/models/gold/defi/defi__ez_lending.sql @@ -26,7 +26,7 @@ WITH lending AS ( ), labels AS ( SELECT - token_id AS contract_address, + asset_identifier AS contract_address, NAME, symbol, IFF( diff --git a/models/silver/defi/lending/silver__burrow_lending.sql b/models/silver/defi/lending/silver__burrow_lending.sql index 3d887be..dfa27f8 100644 --- a/models/silver/defi/lending/silver__burrow_lending.sql +++ b/models/silver/defi/lending/silver__burrow_lending.sql @@ -6,7 +6,7 @@ WITH metadata AS ( SELECT - token_id as contract_address, + asset_identifier as contract_address, NAME, symbol, decimals diff --git a/models/silver/prices/silver__complete_token_prices.sql b/models/silver/prices/silver__complete_token_prices.sql index 760c2ec..b845710 100644 --- a/models/silver/prices/silver__complete_token_prices.sql +++ b/models/silver/prices/silver__complete_token_prices.sql @@ -86,4 +86,4 @@ FROM complete_token_prices p LEFT JOIN {{ ref('silver__ft_contract_metadata') }} ft - ON p.token_address = ft.token_id + ON p.token_address = ft.asset_identifier diff --git a/models/silver/transfers/bridging/silver__bridge_allbridge.sql b/models/silver/transfers/bridging/silver__bridge_allbridge.sql index d5a13be..34afb26 100644 --- a/models/silver/transfers/bridging/silver__bridge_allbridge.sql +++ b/models/silver/transfers/bridging/silver__bridge_allbridge.sql @@ -41,7 +41,7 @@ WITH functioncall AS ( ), metadata AS ( SELECT - token_id as contract_address, + asset_identifier as contract_address, NAME, symbol, decimals From 7475bec6e2f7b960f547ac64929d54cddbb56903 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 2 Jun 2025 13:10:13 -0600 Subject: [PATCH 16/24] upd usage of metadata in ez intents --- models/gold/defi/defi__ez_intents.sql | 85 +++++++++---------- .../labels/silver__ft_contract_metadata.sql | 11 ++- 2 files changed, 51 insertions(+), 45 deletions(-) diff --git a/models/gold/defi/defi__ez_intents.sql b/models/gold/defi/defi__ez_intents.sql index d689767..cf7b2b4 100644 --- a/models/gold/defi/defi__ez_intents.sql +++ b/models/gold/defi/defi__ez_intents.sql @@ -9,7 +9,6 @@ tags = ['scheduled_non_core'] ) }} -- depends on {{ ref('defi__fact_intents') }} --- depends on {{ ref('silver__defuse_tokens_metadata') }} -- depends on {{ ref('silver__ft_contract_metadata') }} -- depends on {{ ref('price__ez_prices_hourly') }} @@ -85,12 +84,12 @@ WITH intents AS ( token_id, REGEXP_SUBSTR( token_id, - 'nep(141|245):(.*)', + 'nep(141|171|245):(.*)', 1, 1, 'e', 2 - ) AS contract_address_raw, + ) AS asset_identifier, referral, dip4_version, gas_burnt, @@ -123,42 +122,19 @@ WITH intents AS ( ), labels AS ( SELECT - near_token_id AS contract_address_raw, - SPLIT( - defuse_asset_identifier, - ':' - ) [0] :: STRING AS ecosystem, - SPLIT( - defuse_asset_identifier, - ':' - ) [1] :: STRING AS chain_id, - SPLIT( - defuse_asset_identifier, - ':' - ) [2] :: STRING AS contract_address, - asset_name AS symbol, - decimals - FROM - {{ ref('silver__defuse_tokens_metadata') }} - UNION ALL - SELECT - asset_identifier AS contract_address_raw, - 'near' AS ecosystem, - '397' AS chain_id, - asset_identifier AS contract_address, + asset_identifier, + source_chain, + crosschain_token_contract, + near_token_contract, symbol, decimals FROM {{ ref('silver__ft_contract_metadata') }} - WHERE - contract_address not in ( - select distinct near_token_id - from {{ ref('silver__defuse_tokens_metadata') }} - ) ), prices AS ( SELECT token_address AS contract_address, + blockchain, symbol, price, is_native, @@ -227,12 +203,9 @@ FINAL AS ( gas_burnt, receipt_succeeded, fact_intents_id AS ez_intents_id, - COALESCE( - dl.short_name, - l.ecosystem - ) AS blockchain, - l.contract_address, - l.contract_address = 'native' AS is_native, + l.source_chain AS blockchain, + l.crosschain_token_contract AS contract_address, + l.crosschain_token_contract = 'native' AS is_native, l.symbol, l.decimals, amount_raw / pow( @@ -251,25 +224,51 @@ FINAL AS ( FROM intents i LEFT JOIN labels l - ON i.contract_address_raw = l.contract_address_raw - LEFT JOIN EXTERNAL.defillama.dim_chains dl - ON l.chain_id = dl.chain_id + ON i.asset_identifier = l.asset_identifier ASOF JOIN prices p match_condition ( i.block_timestamp >= p.hour ) ON ( - l.contract_address = p.contract_address + l.crosschain_token_contract = p.contract_address ) ASOF JOIN prices_native p2 match_condition ( i.block_timestamp >= p2.hour ) ON ( upper(l.symbol) = upper(p2.symbol) - AND (l.contract_address = 'native') = p2.is_native + AND (l.crosschain_token_contract = 'native') = p2.is_native ) ) SELECT - *, + block_timestamp, + block_id, + tx_hash, + receipt_id, + receiver_id, + predecessor_id, + log_event, + token_id, + symbol, + amount_adj, + amount_usd, + owner_id, + old_owner_id, + new_owner_id, + amount_raw, + blockchain, + contract_address, + is_native, + price, + decimals, + gas_burnt, + memo, + referral, + dip4_version, + log_index, + log_event_index, + amount_index, + receipt_succeeded, + ez_intents_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp FROM diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index 4547a30..e31adb2 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -150,8 +150,15 @@ final AS ( -- Nearblocks SELECT n.near_token_contract :: STRING AS asset_identifier, - 'near' AS source_chain, - n.near_token_contract :: STRING AS crosschain_token_contract, + IFF( + n.near_token_contract ilike '%-0x%', + SPLIT_PART(n.near_token_contract, '-', 1) :: STRING, + 'near' + ) AS source_chain, + COALESCE( + REGEXP_SUBSTR(n.near_token_contract, '0x[a-fA-F0-9]{40}') :: STRING, + n.near_token_contract + ) AS crosschain_token_contract, n.near_token_contract, n.decimals :: INT AS decimals, n.name :: STRING AS name, From 1398646c66972d8df3af267733725312ce908691 Mon Sep 17 00:00:00 2001 From: jacksan Date: Mon, 2 Jun 2025 14:53:00 -0600 Subject: [PATCH 17/24] external ft contract metadata --- .../silver__defuse_ft_metadata.sql} | 41 +++- .../silver__nearblocks_ft_metadata.sql | 71 ++++++ .../silver__omni_ft_metadata.sql} | 11 +- .../labels/silver__ft_contract_metadata.sql | 212 +++++++++--------- .../complete/streamline__omni_complete.sql | 8 +- .../realtime/streamline__omni_realtime.sql | 12 +- .../omni/streamline__omni_tokenlist.sql | 13 +- 7 files changed, 228 insertions(+), 140 deletions(-) rename models/silver/labels/{silver__defuse_tokens_metadata.sql => external/silver__defuse_ft_metadata.sql} (51%) create mode 100644 models/silver/labels/external/silver__nearblocks_ft_metadata.sql rename models/silver/labels/{silver__omni_metadata.sql => external/silver__omni_ft_metadata.sql} (81%) diff --git a/models/silver/labels/silver__defuse_tokens_metadata.sql b/models/silver/labels/external/silver__defuse_ft_metadata.sql similarity index 51% rename from models/silver/labels/silver__defuse_tokens_metadata.sql rename to models/silver/labels/external/silver__defuse_ft_metadata.sql index c52767d..c05e0ef 100644 --- a/models/silver/labels/silver__defuse_tokens_metadata.sql +++ b/models/silver/labels/external/silver__defuse_ft_metadata.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', + unique_key = 'defuse_ft_metadata_id', + incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'defuse_asset_identifier', tags = ['scheduled_non_core'] ) }} @@ -11,25 +12,43 @@ WITH api_call AS ( response FROM {{ ref('streamline__defuse_token_ids_realtime') }} -) +), +flattened AS ( SELECT + VALUE :defuse_asset_identifier :: STRING AS defuse_asset_identifier, VALUE :asset_name :: STRING AS asset_name, VALUE :decimals :: INT AS decimals, - VALUE :defuse_asset_identifier :: STRING AS defuse_asset_identifier, VALUE :min_deposit_amount :: STRING AS min_deposit_amount, VALUE :min_withdrawal_amount :: STRING AS min_withdrawal_amount, - VALUE :near_token_id :: STRING AS near_token_id, - VALUE :withdrawal_fee :: STRING AS withdrawal_fee, - {{ dbt_utils.generate_surrogate_key( - ['defuse_asset_identifier'] - ) }} AS defuse_token_ids_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id + VALUE :near_token_id :: STRING AS near_token_contract, + VALUE :withdrawal_fee :: STRING AS withdrawal_fee FROM api_call, LATERAL FLATTEN( input => response :data :result :tokens :: ARRAY ) +) +SELECT + defuse_asset_identifier, + CASE + WHEN SPLIT_PART(defuse_asset_identifier, ':', 0) = 'near' THEN 'near' + WHEN SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) = 'native' THEN SPLIT_PART(near_token_contract, '.', 0) :: STRING + ELSE SPLIT_PART(near_token_contract, '-', 0) :: STRING + END AS source_chain, + SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) AS crosschain_token_contract, + asset_name, + decimals, + min_deposit_amount, + min_withdrawal_amount, + near_token_contract, + withdrawal_fee, + {{ dbt_utils.generate_surrogate_key( + ['defuse_asset_identifier'] + ) }} AS defuse_ft_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + flattened qualify(row_number() over (partition by defuse_asset_identifier order by inserted_timestamp asc)) = 1 diff --git a/models/silver/labels/external/silver__nearblocks_ft_metadata.sql b/models/silver/labels/external/silver__nearblocks_ft_metadata.sql new file mode 100644 index 0000000..f024901 --- /dev/null +++ b/models/silver/labels/external/silver__nearblocks_ft_metadata.sql @@ -0,0 +1,71 @@ +{{ config( + materialized = 'incremental', + unique_key = 'nearblocks_ft_metadata_id', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + tags = ['scheduled_non_core'] +) }} + + + +WITH nearblocks AS ( + + SELECT + VALUE :CONTRACT_ADDRESS :: STRING AS contract_address, + DATA + FROM + {{ ref('bronze__nearblocks_ft_metadata')}} + WHERE + typeof(DATA) != 'NULL_VALUE' + + {% if is_incremental() %} + AND + _inserted_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} +), +nearblocks_metadata AS ( + SELECT + VALUE :contract :: STRING AS near_token_contract, + VALUE :decimals :: INT AS decimals, + VALUE :name :: STRING AS NAME, + VALUE :symbol :: STRING AS symbol + FROM + nearblocks, + LATERAL FLATTEN( + input => DATA :contracts + ) +) +SELECT + near_token_contract, + decimals, + name, + symbol, + CASE + WHEN near_token_contract ilike 'sol-%.omft.near' THEN 'sol' + WHEN near_token_contract rlike '.*-0x[0-9a-fA-F]+\\.(duse|omft)\\.near' THEN SPLIT_PART(near_token_contract, '-', 1) :: STRING + WHEN near_token_contract ilike '%-native.duse.near' THEN SPLIT_PART(near_token_contract, '-', 1) :: STRING + ELSE 'near' + END AS source_chain, + IFF( + source_chain = 'near', + near_token_contract, + COALESCE( + REGEXP_SUBSTR(near_token_contract, '0x[a-fA-F0-9]{40}') :: STRING, + REGEXP_SUBSTR(near_token_contract, 'native') :: STRING, + REGEXP_SUBSTR(near_token_contract, 'sol-([^.]+)\\.omft\\.near', 1, 1, 'e', 1) + ) + ) AS crosschain_token_contract, + {{ dbt_utils.generate_surrogate_key( + ['near_token_contract'] + ) }} AS nearblocks_ft_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM nearblocks_metadata + +qualify(row_number() over (partition by near_token_contract order by modified_timestamp desc)) = 1 diff --git a/models/silver/labels/silver__omni_metadata.sql b/models/silver/labels/external/silver__omni_ft_metadata.sql similarity index 81% rename from models/silver/labels/silver__omni_metadata.sql rename to models/silver/labels/external/silver__omni_ft_metadata.sql index 22f1b8a..83669f3 100644 --- a/models/silver/labels/silver__omni_metadata.sql +++ b/models/silver/labels/external/silver__omni_ft_metadata.sql @@ -1,14 +1,15 @@ {{ config( materialized = 'incremental', + unique_key = 'omni_ft_metadata_id', + incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'omni_asset_identifier', tags = ['scheduled_non_core'] ) }} WITH omni AS ( SELECT - VALUE:CONTRACT_ADDRESS::STRING AS omni_asset_identifier, + VALUE:OMNI_ASSET_IDENTIFIER::STRING AS omni_asset_identifier, VALUE:data:result:result::ARRAY AS result_array, DATA FROM @@ -51,10 +52,12 @@ conversion AS ( ) SELECT omni_asset_identifier, - decoded_result AS contract_address, + SPLIT_PART(omni_asset_identifier, ':', 1) :: STRING AS source_chain, + SPLIT_PART(omni_asset_identifier, ':', 2) :: STRING AS crosschain_token_contract, + decoded_result AS near_token_contract, {{ dbt_utils.generate_surrogate_key( ['omni_asset_identifier'] - ) }} AS omni_metadata_id, + ) }} AS omni_ft_metadata_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index e31adb2..37c85ed 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -1,6 +1,6 @@ --- depends on: {{ ref('bronze__nearblocks_ft_metadata')}} --- depends on: {{ ref('bronze__omni_metadata')}} --- depends on: {{ ref('silver__defuse_tokens_metadata')}} +-- depends on: {{ ref('silver__nearblocks_ft_metadata')}} +-- depends on: {{ ref('silver__omni_ft_metadata')}} +-- depends on: {{ ref('silver__defuse_ft_metadata')}} {{ config( materialized = 'incremental', @@ -14,16 +14,19 @@ WITH nearblocks AS ( SELECT - VALUE :CONTRACT_ADDRESS :: STRING AS contract_address, - DATA + near_token_contract, + decimals, + name, + symbol, + source_chain, + crosschain_token_contract, + 'nearblocks' AS source FROM - {{ ref('bronze__nearblocks_ft_metadata')}} - WHERE - typeof(DATA) != 'NULL_VALUE' + {{ ref('silver__nearblocks_ft_metadata')}} {% if is_incremental() %} - AND - _inserted_timestamp >= ( + WHERE + modified_timestamp >= ( SELECT MAX(modified_timestamp) FROM @@ -31,32 +34,26 @@ WITH nearblocks AS ( ) {% endif %} ), -nearblocks_metadata AS ( - SELECT - VALUE :contract :: STRING AS near_token_contract, - VALUE :decimals :: INT AS decimals, - VALUE :name :: STRING AS NAME, - VALUE :symbol :: STRING AS symbol - FROM - nearblocks, - LATERAL FLATTEN( - input => DATA :contracts - ) -), omni AS ( SELECT omni_asset_identifier AS asset_identifier, - SPLIT_PART(omni_asset_identifier, ':', 1) :: STRING AS source_chain, - SPLIT_PART(omni_asset_identifier, ':', 2) :: STRING AS crosschain_token_contract, - contract_address AS near_token_contract + o.source_chain, + o.crosschain_token_contract, + o.near_token_contract, + n.decimals, + n.name, + n.symbol, + 'omni' AS source FROM - {{ ref('silver__omni_metadata')}} - + {{ ref('silver__omni_ft_metadata')}} o + LEFT JOIN {{ ref('silver__nearblocks_ft_metadata') }} n + ON o.near_token_contract = n.near_token_contract + {% if is_incremental() %} WHERE - modified_timestamp >= ( + o.modified_timestamp >= ( SELECT - MAX(modified_timestamp) + MAX(o.modified_timestamp) FROM {{ this }} ) @@ -64,19 +61,32 @@ omni AS ( ), omni_unmapped AS ( SELECT - contract_address AS asset_identifier, - SPLIT_PART(contract_address, ':', 1) :: STRING AS source_chain, - SPLIT_PART(contract_address, ':', 2) :: STRING AS crosschain_token_contract + omni_asset_identifier AS asset_identifier, + o.source_chain, + o.crosschain_token_contract, + n.near_token_contract, + COALESCE(n.decimals, c.decimals) :: INT AS decimals, + COALESCE(n.name, c.name) :: STRING AS name, + COALESCE(n.symbol, c.symbol) :: STRING AS symbol, + 'omni_unmapped' AS source FROM - {{ ref('streamline__omni_tokenlist')}} + {{ ref('streamline__omni_tokenlist')}} o + LEFT JOIN {{ ref('silver__nearblocks_ft_metadata') }} n + ON o.crosschain_token_contract = n.near_token_contract + AND o.source_chain = 'near' + LEFT JOIN {{ source('crosschain_silver', 'complete_token_asset_metadata')}} c + ON o.crosschain_token_contract = c.token_address + AND c.blockchain = 'solana' + -- note, this does not give use the Near token contract. + -- could join on symbol, but some symbols have multiple contract records as symbol is not unique WHERE - source_chain_id IN ('near', 'sol') + o.source_chain IN ('near', 'sol') {% if is_incremental() %} - WHERE - modified_timestamp >= ( + AND + o.modified_timestamp >= ( SELECT - MAX(modified_timestamp) + MAX(o.modified_timestamp) FROM {{ this }} ) @@ -84,18 +94,18 @@ omni_unmapped AS ( ), defuse AS ( SELECT - defuse_asset_identifier AS asset_identifier, - CASE - WHEN SPLIT_PART(defuse_asset_identifier, ':', 0) = 'near' THEN 'near' - WHEN SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) = 'native' THEN SPLIT_PART(near_token_id, '.', 0) :: STRING - ELSE SPLIT_PART(near_token_id, '-', 0) :: STRING - END AS source_chain, - SPLIT_PART(defuse_asset_identifier, ':', ARRAY_SIZE(SPLIT(defuse_asset_identifier, ':'))) AS crosschain_token_contract, - near_token_id AS near_token_contract, + d.near_token_contract AS asset_identifier, + d.source_chain, + d.crosschain_token_contract, + d.near_token_contract, + d.decimals, + n.name, asset_name AS symbol, - decimals + 'defuse' AS source FROM - {{ ref('silver__defuse_tokens_metadata')}} + {{ ref('silver__defuse_ft_metadata')}} d + LEFT JOIN {{ ref('silver__nearblocks_ft_metadata') }} n + ON d.near_token_contract = n.near_token_contract {% if is_incremental() %} WHERE @@ -108,81 +118,63 @@ defuse AS ( {% endif %} ), final AS ( + -- Nearblocks + SELECT + near_token_contract AS asset_identifier, + source_chain, + crosschain_token_contract, + near_token_contract, + decimals, + name, + symbol, + source + FROM + nearblocks + + UNION ALL + -- Omni SELECT - o.asset_identifier, - o.source_chain, - o.crosschain_token_contract, - o.near_token_contract, - n.decimals :: INT AS decimals, - n.name :: STRING AS name, - n.symbol :: STRING AS symbol, - 'omni' AS source + asset_identifier, + source_chain, + crosschain_token_contract, + near_token_contract, + decimals, + name, + symbol, + source FROM - omni o - LEFT JOIN nearblocks_metadata n - ON o.near_token_contract = n.near_token_contract + omni UNION ALL -- Omni unmapped SELECT - o.asset_identifier, - o.source_chain, - o.crosschain_token_contract, - n.near_token_contract, - COALESCE(n.decimals, c.decimals) :: INT AS decimals, - COALESCE(n.name, c.name) :: STRING AS name, - COALESCE(n.symbol, c.symbol) :: STRING AS symbol, - 'omni_unmapped' AS source + asset_identifier, + source_chain, + crosschain_token_contract, + near_token_contract, + decimals, + name, + symbol, + source FROM - omni_unmapped o - LEFT JOIN nearblocks_metadata n - ON o.crosschain_token_contract = n.near_token_contract - LEFT JOIN {{ source('crosschain_silver', 'complete_token_asset_metadata')}} c - ON o.crosschain_token_contract = c.token_address - AND c.blockchain = 'solana' - -- note, this does not give use the Near token contract. - -- could join on symbol, but some symbols have multiple contract records as symbol is not unique - - UNION ALL - - -- Nearblocks - SELECT - n.near_token_contract :: STRING AS asset_identifier, - IFF( - n.near_token_contract ilike '%-0x%', - SPLIT_PART(n.near_token_contract, '-', 1) :: STRING, - 'near' - ) AS source_chain, - COALESCE( - REGEXP_SUBSTR(n.near_token_contract, '0x[a-fA-F0-9]{40}') :: STRING, - n.near_token_contract - ) AS crosschain_token_contract, - n.near_token_contract, - n.decimals :: INT AS decimals, - n.name :: STRING AS name, - n.symbol :: STRING AS symbol, - 'nearblocks' AS source - FROM - nearblocks_metadata n + omni_unmapped UNION ALL -- Defuse SELECT - d.asset_identifier, - d.source_chain, - d.crosschain_token_contract, - d.near_token_contract, - d.decimals :: INT AS decimals, - n.name :: STRING AS name, - d.symbol :: STRING AS symbol, - 'defuse' AS source + asset_identifier, + source_chain, + crosschain_token_contract, + near_token_contract, + decimals, + name, + symbol, + source FROM - defuse d - LEFT JOIN nearblocks_metadata n - ON d.near_token_contract = n.near_token_contract + defuse ) SELECT asset_identifier, @@ -192,7 +184,7 @@ SELECT decimals, name, symbol, - source as metadata_source, + source as metadata_provider, {{ dbt_utils.generate_surrogate_key(['asset_identifier']) }} AS ft_contract_metadata_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, @@ -201,5 +193,7 @@ FROM final qualify ROW_NUMBER() over ( PARTITION BY asset_identifier - ORDER BY modified_timestamp DESC + ORDER BY + metadata_provider = 'defuse' DESC, -- prioritize defuse over nearblocks for those tokens + modified_timestamp DESC ) = 1 diff --git a/models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql b/models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql index 799fca4..7da8ad8 100644 --- a/models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql +++ b/models/streamline/external/token_metadata/omni/complete/streamline__omni_complete.sql @@ -1,16 +1,16 @@ -- depends_on: {{ ref('bronze__omni_metadata') }} {{ config( materialized = "incremental", - unique_key = "contract_address", + unique_key = "omni_complete_id", merge_exclude_columns = ["inserted_timestamp"], tags = ['streamline_non_core'] ) }} SELECT - VALUE:CONTRACT_ADDRESS :: STRING AS contract_address, + VALUE:OMNI_ASSET_IDENTIFIER :: STRING AS omni_asset_identifier, partition_key, _inserted_timestamp, - contract_address AS omni_complete_id, + VALUE:OMNI_ASSET_IDENTIFIER :: STRING AS omni_complete_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id @@ -32,4 +32,4 @@ AND ) {% endif %} -qualify(row_number() over (partition by contract_address order by _inserted_timestamp desc)) = 1 \ No newline at end of file +qualify(row_number() over (partition by omni_asset_identifier order by _inserted_timestamp desc)) = 1 diff --git a/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql b/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql index 9b15656..8407203 100644 --- a/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql +++ b/models/streamline/external/token_metadata/omni/realtime/streamline__omni_realtime.sql @@ -18,19 +18,19 @@ WITH omni_token AS ( SELECT - contract_address + omni_asset_identifier FROM {{ ref('streamline__omni_tokenlist') }} WHERE - source_chain_id NOT IN ('near', 'sol') + source_chain NOT IN ('near', 'sol') EXCEPT SELECT - contract_address + omni_asset_identifier FROM {{ ref('streamline__omni_complete')}} ) SELECT - contract_address, + omni_asset_identifier, DATE_PART('EPOCH', SYSDATE()) :: INTEGER AS partition_key, {{ target.database }}.LIVE.UDF_API( 'POST', @@ -45,10 +45,10 @@ SELECT 'finality', 'final', 'account_id', 'omni.bridge.near', 'method_name', 'get_token_id', - 'args_base64', BASE64_ENCODE(OBJECT_CONSTRUCT('address', contract_address) :: STRING) + 'args_base64', BASE64_ENCODE(OBJECT_CONSTRUCT('address', omni_asset_identifier) :: STRING) ) ), 'Vault/prod/near/quicknode/mainnet' ) AS request FROM - omni_token \ No newline at end of file + omni_token diff --git a/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql index 00e8123..a0f950c 100644 --- a/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql +++ b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql @@ -2,14 +2,14 @@ materialized = "incremental", incremental_strategy = 'merge', incremental_predicates = ["dynamic_range_predicate","modified_timestamp::date"], - unique_key = "contract_address", + unique_key = "omni_tokenlist_id", cluster_by = ['modified_timestamp::DATE'], tags = ['streamline_non_core'] ) }} WITH omni_token AS ( SELECT - DISTINCT raw_token_id AS contract_address + DISTINCT raw_token_id AS omni_asset_identifier FROM {{ ref('silver__bridge_omni') }} @@ -22,16 +22,17 @@ WHERE modified_timestamp >= ( {% endif %} ) SELECT - contract_address, - SPLIT_PART(contract_address, ':', 0) :: STRING AS source_chain_id, + omni_asset_identifier, + SPLIT_PART(omni_asset_identifier, ':', 1) :: STRING AS source_chain, + SPLIT_PART(omni_asset_identifier, ':', 2) :: STRING AS crosschain_token_contract, {{ dbt_utils.generate_surrogate_key( - ['contract_address'] + ['omni_asset_identifier'] ) }} AS omni_tokenlist_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM omni_token -QUALIFY (ROW_NUMBER() OVER (PARTITION BY contract_address +QUALIFY (ROW_NUMBER() OVER (PARTITION BY omni_asset_identifier ORDER BY modified_timestamp ASC) = 1) From 1705b425c8df712a013c9712ffe58997a69bfe2c Mon Sep 17 00:00:00 2001 From: jacksan Date: Mon, 2 Jun 2025 15:08:50 -0600 Subject: [PATCH 18/24] upd col ref --- models/gold/defi/defi__ez_intents.sql | 2 +- models/silver/labels/silver__ft_contract_metadata.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/gold/defi/defi__ez_intents.sql b/models/gold/defi/defi__ez_intents.sql index cf7b2b4..0ac349d 100644 --- a/models/gold/defi/defi__ez_intents.sql +++ b/models/gold/defi/defi__ez_intents.sql @@ -276,4 +276,4 @@ FROM qualify(ROW_NUMBER() over (PARTITION BY ez_intents_id ORDER BY - price IS NOT NULL DESC, is_native DESC) = 1) \ No newline at end of file + price IS NOT NULL DESC, is_native DESC) = 1) diff --git a/models/silver/labels/silver__ft_contract_metadata.yml b/models/silver/labels/silver__ft_contract_metadata.yml index 2584a1b..a964a0a 100644 --- a/models/silver/labels/silver__ft_contract_metadata.yml +++ b/models/silver/labels/silver__ft_contract_metadata.yml @@ -28,7 +28,7 @@ models: description: "{{ doc('name') }}" - name: SYMBOL description: "{{ doc('symbol') }}" - - name: METADATA_SOURCE + - name: METADATA_PROVIDER description: "Source of the metadata record (e.g., omni, nearblocks, defuse)." - name: FT_CONTRACT_METADATA_ID description: "{{doc('id')}}" From 40159dfeff7fa83071a49b34139a93297c673831 Mon Sep 17 00:00:00 2001 From: jacksan Date: Mon, 2 Jun 2025 15:11:42 -0600 Subject: [PATCH 19/24] re-update token transfers after merge from main --- models/gold/core/core__ez_token_transfers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index cbc21f7..072c8c5 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -109,7 +109,7 @@ FROM ON (t.contract_address = p.token_address) AND date_trunc('hour', t.block_timestamp) = p.HOUR {% endif %} - LEFT JOIN {{ ref('silver__ft_contract_metadata') }} C USING (contract_address) + LEFT JOIN {{ ref('silver__ft_contract_metadata') }} C on (t.contract_address = C.asset_identifier) {% if var("MANUAL_FIX") %} WHERE From cf833e044a0282bbae4b8cf5d0eeab097546c497 Mon Sep 17 00:00:00 2001 From: jacksan Date: Mon, 2 Jun 2025 15:31:03 -0600 Subject: [PATCH 20/24] rm dynamic predicate causing dupes --- models/gold/defi/defi__ez_intents.sql | 2 +- models/silver/labels/silver__ft_contract_metadata.sql | 2 +- .../token_metadata/nearblocks/streamline__ft_tokenlist.sql | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/models/gold/defi/defi__ez_intents.sql b/models/gold/defi/defi__ez_intents.sql index 0ac349d..7bd9f82 100644 --- a/models/gold/defi/defi__ez_intents.sql +++ b/models/gold/defi/defi__ez_intents.sql @@ -224,7 +224,7 @@ FINAL AS ( FROM intents i LEFT JOIN labels l - ON i.asset_identifier = l.asset_identifier + ON i.token_id = l.asset_identifier ASOF JOIN prices p match_condition ( i.block_timestamp >= p.hour ) diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index 37c85ed..3527f40 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -94,7 +94,7 @@ omni_unmapped AS ( ), defuse AS ( SELECT - d.near_token_contract AS asset_identifier, + CONCAT_WS(':', 'nep141', d.near_token_contract) AS asset_identifier, d.source_chain, d.crosschain_token_contract, d.near_token_contract, diff --git a/models/streamline/external/token_metadata/nearblocks/streamline__ft_tokenlist.sql b/models/streamline/external/token_metadata/nearblocks/streamline__ft_tokenlist.sql index f9f798a..936467f 100644 --- a/models/streamline/external/token_metadata/nearblocks/streamline__ft_tokenlist.sql +++ b/models/streamline/external/token_metadata/nearblocks/streamline__ft_tokenlist.sql @@ -1,7 +1,6 @@ {{ config ( materialized = "incremental", incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate","modified_timestamp::date"], unique_key = "contract_address", cluster_by = ['modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(contract_address)", From 6ed3da1a6fab5785b1ccdf1910089ae9a47f35b0 Mon Sep 17 00:00:00 2001 From: jacksan Date: Mon, 2 Jun 2025 15:35:25 -0600 Subject: [PATCH 21/24] rm nep141: concat --- models/gold/defi/defi__ez_intents.sql | 2 +- models/silver/labels/silver__ft_contract_metadata.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/gold/defi/defi__ez_intents.sql b/models/gold/defi/defi__ez_intents.sql index 7bd9f82..0ac349d 100644 --- a/models/gold/defi/defi__ez_intents.sql +++ b/models/gold/defi/defi__ez_intents.sql @@ -224,7 +224,7 @@ FINAL AS ( FROM intents i LEFT JOIN labels l - ON i.token_id = l.asset_identifier + ON i.asset_identifier = l.asset_identifier ASOF JOIN prices p match_condition ( i.block_timestamp >= p.hour ) diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index 3527f40..37c85ed 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -94,7 +94,7 @@ omni_unmapped AS ( ), defuse AS ( SELECT - CONCAT_WS(':', 'nep141', d.near_token_contract) AS asset_identifier, + d.near_token_contract AS asset_identifier, d.source_chain, d.crosschain_token_contract, d.near_token_contract, From 8a61959edc5eb03ee6f118c083e9f8a8ef7b215f Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Mon, 9 Jun 2025 19:27:13 +0700 Subject: [PATCH 22/24] changes --- models/gold/core/core__dim_ft_contract_metadata.sql | 2 +- models/gold/core/core__dim_ft_contract_metadata.yml | 4 ---- models/silver/labels/silver__ft_contract_metadata.sql | 6 +++--- .../token_metadata/omni/streamline__omni_tokenlist.sql | 4 ++-- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/models/gold/core/core__dim_ft_contract_metadata.sql b/models/gold/core/core__dim_ft_contract_metadata.sql index 7ec50b3..e230edc 100644 --- a/models/gold/core/core__dim_ft_contract_metadata.sql +++ b/models/gold/core/core__dim_ft_contract_metadata.sql @@ -17,4 +17,4 @@ SELECT FROM {{ ref('silver__ft_contract_metadata') }} WHERE - name is not null OR symbol is not null + name is not null OR symbol is not null OR decimals is not null diff --git a/models/gold/core/core__dim_ft_contract_metadata.yml b/models/gold/core/core__dim_ft_contract_metadata.yml index d738e73..16bf9f8 100644 --- a/models/gold/core/core__dim_ft_contract_metadata.yml +++ b/models/gold/core/core__dim_ft_contract_metadata.yml @@ -19,12 +19,8 @@ models: description: "{{ doc('contract_address') }}" - name: NAME description: "{{ doc('name') }}" - tests: - - not_null - name: SYMBOL description: "{{ doc('symbol') }}" - tests: - - not_null - name: DECIMALS description: "{{ doc('decimals') }}" - name: DIM_FT_CONTRACT_METADATA_ID diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index 37c85ed..d8edb58 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -53,7 +53,7 @@ omni AS ( WHERE o.modified_timestamp >= ( SELECT - MAX(o.modified_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -86,7 +86,7 @@ omni_unmapped AS ( AND o.modified_timestamp >= ( SELECT - MAX(o.modified_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -109,7 +109,7 @@ defuse AS ( {% if is_incremental() %} WHERE - modified_timestamp >= ( + d.modified_timestamp >= ( SELECT MAX(modified_timestamp) FROM diff --git a/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql index a0f950c..6b9150d 100644 --- a/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql +++ b/models/streamline/external/token_metadata/omni/streamline__omni_tokenlist.sql @@ -1,9 +1,9 @@ {{ config ( materialized = "incremental", incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate","modified_timestamp::date"], - unique_key = "omni_tokenlist_id", + unique_key = "omni_asset_identifier", cluster_by = ['modified_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(omni_asset_identifier);", tags = ['streamline_non_core'] ) }} From f71a9422a81b6513fad64bf502058b6c03f7c094 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Wed, 11 Jun 2025 01:15:06 +0700 Subject: [PATCH 23/24] final joins --- .../labels/silver__ft_contract_metadata.sql | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index d8edb58..f491c79 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -46,8 +46,6 @@ omni AS ( 'omni' AS source FROM {{ ref('silver__omni_ft_metadata')}} o - LEFT JOIN {{ ref('silver__nearblocks_ft_metadata') }} n - ON o.near_token_contract = n.near_token_contract {% if is_incremental() %} WHERE @@ -71,9 +69,6 @@ omni_unmapped AS ( 'omni_unmapped' AS source FROM {{ ref('streamline__omni_tokenlist')}} o - LEFT JOIN {{ ref('silver__nearblocks_ft_metadata') }} n - ON o.crosschain_token_contract = n.near_token_contract - AND o.source_chain = 'near' LEFT JOIN {{ source('crosschain_silver', 'complete_token_asset_metadata')}} c ON o.crosschain_token_contract = c.token_address AND c.blockchain = 'solana' @@ -104,8 +99,6 @@ defuse AS ( 'defuse' AS source FROM {{ ref('silver__defuse_ft_metadata')}} d - LEFT JOIN {{ ref('silver__nearblocks_ft_metadata') }} n - ON d.near_token_contract = n.near_token_contract {% if is_incremental() %} WHERE @@ -175,6 +168,30 @@ final AS ( source FROM defuse +), +final_joined AS ( + SELECT + f.asset_identifier, + f.source_chain, + f.crosschain_token_contract, + f.near_token_contract, + CASE + WHEN f.source = 'omni_unmapped' AND f.source_chain != 'near' THEN f.decimals + ELSE COALESCE(f.decimals, n.decimals) + END AS decimals, + CASE + WHEN f.source = 'omni_unmapped' AND f.source_chain != 'near' THEN f.name + ELSE COALESCE(f.name, n.name) + END AS name, + CASE + WHEN f.source = 'omni_unmapped' AND f.source_chain != 'near' THEN f.symbol + ELSE COALESCE(f.symbol, n.symbol) + END AS symbol, + f.source AS metadata_provider + FROM final f + LEFT JOIN {{ ref('silver__nearblocks_ft_metadata') }} n + ON f.near_token_contract = n.near_token_contract + AND NOT (f.source = 'omni_unmapped' AND f.source_chain != 'near') ) SELECT asset_identifier, From 813f2b15c47fc889acf0abcd07784db4a0b6a8c6 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Wed, 11 Jun 2025 16:18:02 +0700 Subject: [PATCH 24/24] final fix --- .../labels/silver__ft_contract_metadata.sql | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index f491c79..a08f376 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -1,6 +1,7 @@ -- depends on: {{ ref('silver__nearblocks_ft_metadata')}} -- depends on: {{ ref('silver__omni_ft_metadata')}} -- depends on: {{ ref('silver__defuse_ft_metadata')}} +-- depends on: {{ ref('streamline__omni_tokenlist')}} {{ config( materialized = 'incremental', @@ -40,9 +41,9 @@ omni AS ( o.source_chain, o.crosschain_token_contract, o.near_token_contract, - n.decimals, - n.name, - n.symbol, + NULL AS decimals, + NULL AS name, + NULL AS symbol, 'omni' AS source FROM {{ ref('silver__omni_ft_metadata')}} o @@ -69,6 +70,9 @@ omni_unmapped AS ( 'omni_unmapped' AS source FROM {{ ref('streamline__omni_tokenlist')}} o + LEFT JOIN {{ ref('silver__nearblocks_ft_metadata') }} n + ON o.crosschain_token_contract = n.near_token_contract + AND o.source_chain = 'near' LEFT JOIN {{ source('crosschain_silver', 'complete_token_asset_metadata')}} c ON o.crosschain_token_contract = c.token_address AND c.blockchain = 'solana' @@ -94,7 +98,7 @@ defuse AS ( d.crosschain_token_contract, d.near_token_contract, d.decimals, - n.name, + NULL AS name, asset_name AS symbol, 'defuse' AS source FROM @@ -170,27 +174,19 @@ final AS ( defuse ), final_joined AS ( - SELECT + SELECT f.asset_identifier, f.source_chain, f.crosschain_token_contract, f.near_token_contract, - CASE - WHEN f.source = 'omni_unmapped' AND f.source_chain != 'near' THEN f.decimals - ELSE COALESCE(f.decimals, n.decimals) - END AS decimals, - CASE - WHEN f.source = 'omni_unmapped' AND f.source_chain != 'near' THEN f.name - ELSE COALESCE(f.name, n.name) - END AS name, - CASE - WHEN f.source = 'omni_unmapped' AND f.source_chain != 'near' THEN f.symbol - ELSE COALESCE(f.symbol, n.symbol) - END AS symbol, + COALESCE(f.decimals, n.decimals) AS decimals, + COALESCE(f.name, n.name) AS name, + COALESCE(f.symbol, n.symbol) AS symbol, f.source AS metadata_provider FROM final f LEFT JOIN {{ ref('silver__nearblocks_ft_metadata') }} n ON f.near_token_contract = n.near_token_contract + AND f.source IN ('omni', 'defuse') AND NOT (f.source = 'omni_unmapped' AND f.source_chain != 'near') ) SELECT @@ -201,12 +197,13 @@ SELECT decimals, name, symbol, - source as metadata_provider, + metadata_provider, {{ dbt_utils.generate_surrogate_key(['asset_identifier']) }} AS ft_contract_metadata_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id -FROM final +FROM + final_joined qualify ROW_NUMBER() over ( PARTITION BY asset_identifier