seed initial ft metadata pull with local job

data/seeds__intent_token_ids.csv

@@ -1,81 +0,0 @@
-BLOCKCHAIN,CONTRACT_ADDRESS,DECIMALS,DEFUSE_ASSET_ID
-"near","17208628f84f5d6ad33f0da3bbbeb27ffcb398eac501a31bd6ad2011e36133a1","6","nep141:17208628f84f5d6ad33f0da3bbbeb27ffcb398eac501a31bd6ad2011e36133a1"
-"near","2260fac5e5542a773aa44fbcfedf7c193bc2c599.factory.bridge.near","8","nep141:2260fac5e5542a773aa44fbcfedf7c193bc2c599.factory.bridge.near"
-"near","853d955acef822db058eb8505911ed77f175b99e.factory.bridge.near","18","nep141:853d955acef822db058eb8505911ed77f175b99e.factory.bridge.near"
-"near","a35923162c49cf95e6bf26623385eb431ad920d3.factory.bridge.near","18","nep141:a35923162c49cf95e6bf26623385eb431ad920d3.factory.bridge.near"
-"near","aaaaaa20d9e0e2461697782ef11675f668207961.factory.bridge.near","18","nep141:aaaaaa20d9e0e2461697782ef11675f668207961.factory.bridge.near"
-"near","abg-966.meme-cooking.near","18","nep141:abg-966.meme-cooking.near"
-"arb","0x912ce59144191c1204e64559fe8253a0e49e6548","18","nep141:arb-0x912ce59144191c1204e64559fe8253a0e49e6548.omft.near"
-"arb","0xaf88d065e77c8cc2239327c5edb3a432268e5831","6","nep141:arb-0xaf88d065e77c8cc2239327c5edb3a432268e5831.omft.near"
-"arb","0xfc5a1a6eb076a2c7ad06ed22c90d7e710e35ad0a","18","nep141:arb-0xfc5a1a6eb076a2c7ad06ed22c90d7e710e35ad0a.omft.near"
-"arb","0xfd086bc7cd5c481dcc9c85ebe478a1c0b69fcbb9","6","nep141:arb-0xfd086bc7cd5c481dcc9c85ebe478a1c0b69fcbb9.omft.near"
-"arb","","18","nep141:arb.omft.near"
-"near","aurora","18","nep141:aurora"
-"base","0x532f27101965dd16442e59d40670faf5ebb142e4","18","nep141:base-0x532f27101965dd16442e59d40670faf5ebb142e4.omft.near"
-"base","0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","6","nep141:base-0x833589fcd6edb6e08f4c7c32d4f71b54bda02913.omft.near"
-"base","0x98d0baa52b2d063e780de12f615f963fe8537553","18","nep141:base-0x98d0baa52b2d063e780de12f615f963fe8537553.omft.near"
-"base","0xa5c67d8d37b88c2d88647814da5578128e2c93b2","18","nep141:base-0xa5c67d8d37b88c2d88647814da5578128e2c93b2.omft.near"
-"base","","18","nep141:base.omft.near"
-"near","bera.omft.near","18","nep141:bera.omft.near"
-"bera","","18","nep141:bera.omft.near"
-"near","blackdragon.tkn.near","24","nep141:blackdragon.tkn.near"
-"near","bsc-0x2170ed0880ac9a755fd29b2688956bd959f933f8.omft.near","18","nep141:bsc-0x2170ed0880ac9a755fd29b2688956bd959f933f8.omft.near"
-"near","bsc-0x55d398326f99059ff775485246999027b3197955.omft.near","18","nep141:bsc-0x55d398326f99059ff775485246999027b3197955.omft.near"
-"bsc","0x55d398326f99059ff775485246999027b3197955","18","nep141:bsc-0x55d398326f99059ff775485246999027b3197955.omft.near"
-"near","bsc-0x8ac76a51cc950d9822d68b83fe1ad97b32cd580d.omft.near","18","nep141:bsc-0x8ac76a51cc950d9822d68b83fe1ad97b32cd580d.omft.near"
-"bsc","0x8ac76a51cc950d9822d68b83fe1ad97b32cd580d","18","nep141:bsc-0x8ac76a51cc950d9822d68b83fe1ad97b32cd580d.omft.near"
-"near","bsc.omft.near","18","nep141:bsc.omft.near"
-"bsc","","18","nep141:bsc.omft.near"
-"btc","","8","nep141:btc.omft.near"
-"doge","","8","nep141:doge.omft.near"
-"eth","0x1f9840a85d5af5bf1d1762f925bdaddc4201f984","18","nep141:eth-0x1f9840a85d5af5bf1d1762f925bdaddc4201f984.omft.near"
-"eth","0x514910771af9ca656af840dff83e8264ecf986ca","18","nep141:eth-0x514910771af9ca656af840dff83e8264ecf986ca.omft.near"
-"eth","0x6982508145454ce325ddbe47a25d4ec3d2311933","18","nep141:eth-0x6982508145454ce325ddbe47a25d4ec3d2311933.omft.near"
-"eth","0x6b175474e89094c44da98b954eedeac495271d0f","18","nep141:eth-0x6b175474e89094c44da98b954eedeac495271d0f.omft.near"
-"eth","0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9","18","nep141:eth-0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9.omft.near"
-"eth","0x95ad61b0a150d79219dcf64e1e6cc01f0b64c4ce","18","nep141:eth-0x95ad61b0a150d79219dcf64e1e6cc01f0b64c4ce.omft.near"
-"eth","0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48","6","nep141:eth-0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48.omft.near"
-"eth","0xa35923162c49cf95e6bf26623385eb431ad920d3","18","nep141:eth-0xa35923162c49cf95e6bf26623385eb431ad920d3.omft.near"
-"eth","0xaaaaaa20d9e0e2461697782ef11675f668207961","18","nep141:eth-0xaaaaaa20d9e0e2461697782ef11675f668207961.omft.near"
-"eth","0xaaee1a9723aadb7afa2810263653a34ba2c21c7a","18","nep141:eth-0xaaee1a9723aadb7afa2810263653a34ba2c21c7a.omft.near"
-"eth","0xdac17f958d2ee523a2206206994597c13d831ec7","6","nep141:eth-0xdac17f958d2ee523a2206206994597c13d831ec7.omft.near"
-"eth","","18","nep141:eth.omft.near"
-"near","gnear-229.meme-cooking.near","18","nep141:gnear-229.meme-cooking.near"
-"near","gnosis-0x177127622c4a00f3d409b75571e12cb3c8973d3c.omft.near","18","nep141:gnosis-0x177127622c4a00f3d409b75571e12cb3c8973d3c.omft.near"
-"gnosis","0x177127622c4a00f3d409b75571e12cb3c8973d3c","18","nep141:gnosis-0x177127622c4a00f3d409b75571e12cb3c8973d3c.omft.near"
-"near","gnosis-0x2a22f9c3b484c3629090feed35f17ff8f88f76f0.omft.near","6","nep141:gnosis-0x2a22f9c3b484c3629090feed35f17ff8f88f76f0.omft.near"
-"gnosis","0x2a22f9c3b484c3629090feed35f17ff8f88f76f0","6","nep141:gnosis-0x2a22f9c3b484c3629090feed35f17ff8f88f76f0.omft.near"
-"near","gnosis-0x4d18815d14fe5c3304e87b3fa18318baa5c23820.omft.near","18","nep141:gnosis-0x4d18815d14fe5c3304e87b3fa18318baa5c23820.omft.near"
-"gnosis","0x4d18815d14fe5c3304e87b3fa18318baa5c23820","18","nep141:gnosis-0x4d18815d14fe5c3304e87b3fa18318baa5c23820.omft.near"
-"near","gnosis-0x6a023ccd1ff6f2045c3309768ead9e68f978f6e1.omft.near","18","nep141:gnosis-0x6a023ccd1ff6f2045c3309768ead9e68f978f6e1.omft.near"
-"gnosis","0x6a023ccd1ff6f2045c3309768ead9e68f978f6e1","18","nep141:gnosis-0x6a023ccd1ff6f2045c3309768ead9e68f978f6e1.omft.near"
-"near","gnosis-0x9c58bacc331c9aa871afd802db6379a98e80cedb.omft.near","18","nep141:gnosis-0x9c58bacc331c9aa871afd802db6379a98e80cedb.omft.near"
-"gnosis","0x9c58bacc331c9aa871afd802db6379a98e80cedb","18","nep141:gnosis-0x9c58bacc331c9aa871afd802db6379a98e80cedb.omft.near"
-"near","gnosis-0xddafbb505ad214d7b80b1f830fccc89b60fb7a83.omft.near","6","nep141:gnosis-0xddafbb505ad214d7b80b1f830fccc89b60fb7a83.omft.near"
-"near","gnosis.omft.near","18","nep141:gnosis.omft.near"
-"gnosis","","18","nep141:gnosis.omft.near"
-"near","mpdao-token.near","6","nep141:mpdao-token.near"
-"near","noear-324.meme-cooking.near","18","nep141:noear-324.meme-cooking.near"
-"near","pol-0x3c499c542cef5e3811e1192ce70d8cc03d5c3359.omft.near","6","nep141:pol-0x3c499c542cef5e3811e1192ce70d8cc03d5c3359.omft.near"
-"pol","0x3c499c542cef5e3811e1192ce70d8cc03d5c3359","6","nep141:pol-0x3c499c542cef5e3811e1192ce70d8cc03d5c3359.omft.near"
-"near","pol.omft.near","18","nep141:pol.omft.near"
-"pol","","18","nep141:pol.omft.near"
-"near","purge-558.meme-cooking.near","18","nep141:purge-558.meme-cooking.near"
-"sol","2cff5b540505a2aa6a4e600ccc6fdd6d3a585a5d","9","nep141:sol-2cff5b540505a2aa6a4e600ccc6fdd6d3a585a5d.omft.near"
-"sol","57d087fd8c460f612f8701f5499ad8b2eec5ab68","6","nep141:sol-57d087fd8c460f612f8701f5499ad8b2eec5ab68.omft.near"
-"sol","5ce3bf3a31af18be40ba30f721101b4341690186","6","nep141:sol-5ce3bf3a31af18be40ba30f721101b4341690186.omft.near"
-"sol","b9c68f94ec8fd160137af8cdfe5e61cd68e2afba","6","nep141:sol-b9c68f94ec8fd160137af8cdfe5e61cd68e2afba.omft.near"
-"sol","c58e6539c2f2e097c251f8edf11f9c03e581f8d4","6","nep141:sol-c58e6539c2f2e097c251f8edf11f9c03e581f8d4.omft.near"
-"sol","c800a4bd850783ccb82c2b2c7e84175443606352","6","nep141:sol-c800a4bd850783ccb82c2b2c7e84175443606352.omft.near"
-"sol","d600e625449a4d9380eaf5e3265e54c90d34e260","6","nep141:sol-d600e625449a4d9380eaf5e3265e54c90d34e260.omft.near"
-"sol","df27d7abcc1c656d4ac3b1399bbfbba1994e6d8c","8","nep141:sol-df27d7abcc1c656d4ac3b1399bbfbba1994e6d8c.omft.near"
-"sol","","9","nep141:sol.omft.near"
-"near","token.0xshitzu.near","18","nep141:token.0xshitzu.near"
-"near","token.burrow.near","18","nep141:token.burrow.near"
-"near","token.sweat","18","nep141:token.sweat"
-"near","token.v2.ref-finance.near","18","nep141:token.v2.ref-finance.near"
-"near","usdt.tether-token.near","6","nep141:usdt.tether-token.near"
-"near","wrap.near","24","nep141:wrap.near"
-"xrp","","6","nep141:xrp.omft.near"
-"near","zec.omft.near","8","nep141:zec.omft.near"
-"zec","","8","nep141:zec.omft.near"

models/gold/defi/defi__ez_intents.sql

@@ -0,0 +1,60 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_strategy = 'merge',
+    incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"],
+    unique_key = ['ez_intents_id'],
+    merge_exclude_columns = ['inserted_timestamp'],
+    cluster_by = ['block_timestamp::DATE'],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id);",
+    tags = ['intents','curated','scheduled_non_core']
+) }}
+
+with
+intents as (
+    select
+        block_timestamp,
+        block_id,
+        tx_hash,
+        receipt_id,
+        receiver_id,
+        predecessor_id,
+        log_event,
+        log_index,
+        log_event_index,
+        owner_id,
+        old_owner_id,
+        new_owner_id,
+        memo,
+        amount_index,
+        amount_raw,
+        token_id,
+        referral,
+        dip4_version,
+        gas_burnt,
+        receipt_succeeded
+    from
+        {{ ref('defi__fact_intents') }}
+    {% if is_incremental() %}
+    where modified_timestamp >= (
+        select coalesce(max(modified_timestamp),'1970-01-01' :: timestamp)
+        from {{ this }}
+    )
+    {% endif %}
+),
+native_labels as (
+    select
+        contract_address,
+        name,
+        symbol,
+        decimals
+    from 
+        {{ ref('silver__ft_contract_metadata') }}
+),
+defuse_labels as (
+    select
+        defuse_asset_identifier,
+        asset_name as name,
+        decimals
+    from 
+        {{ ref('silver__defuse_tokens_metadata') }}
+)

models/silver/labels/silver__ft_contract_metadata.sql

@@ -1,3 +1,6 @@
+-- depends on: {{ ref('seeds__ft_token_details_final')}}
+-- depends on: {{ ref('bronze__nearblocks_ft_metadata')}}
+
 {{ config(
     materialized = 'incremental',
     unique_key = 'contract_address',
@@ -6,7 +9,17 @@
     tags = ['scheduled_non_core']
 ) }}
 
+
 WITH bronze AS (
+    {% if var('NEAR_MIGRATE_ARCHIVE', false) %}
+
+    SELECT
+        contract_address,
+        TRY_PARSE_JSON(DATA) AS DATA
+    FROM
+        {{ ref('seeds__ft_token_details_final')}}
+
+    {% else %}
 
     SELECT
         VALUE :CONTRACT_ADDRESS :: STRING AS contract_address,
@@ -16,21 +29,22 @@ WITH bronze AS (
     WHERE
         typeof(DATA) != 'NULL_VALUE'
 
-{% if is_incremental() %}
-AND
-    _inserted_timestamp >= (
-        SELECT
-            MAX(modified_timestamp)
-        FROM
-            {{ this }}
-    )
+    {% if is_incremental() %}
+    AND
+        _inserted_timestamp >= (
+            SELECT
+                MAX(modified_timestamp)
+            FROM
+                {{ this }}
+        )
+    {% endif %}
 {% endif %}
+
 ),
 flatten_results AS (
     SELECT
         VALUE :contract :: STRING AS contract_address,
         VALUE :decimals :: INT AS decimals,
-        VALUE :icon :: STRING AS icon,
         VALUE :name :: STRING AS NAME,
         VALUE :symbol :: STRING AS symbol,
         VALUE AS DATA
@@ -43,7 +57,6 @@ flatten_results AS (
 SELECT
     contract_address,
     decimals,
-    icon,
     NAME,
     symbol,
     DATA,

models/silver/labels/silver__nft_contract_metadata.sql

@@ -5,16 +5,13 @@
     incremental_strategy = 'merge',
     tags = ['livequery', 'nearblocks','scheduled_non_core'],
 ) }}
--- TODO: refactor? or drop support...
--- the livequery model is still running with no issue tbh
--- it's just the ft one that died 
 
 WITH livequery_results AS (
 
     SELECT
         *
     FROM
-        PLACEHOLDER
+        {{ ref('livequery__request_nearblocks_nft_metadata') }}
 
 {% if is_incremental() %}
 WHERE

models/streamline/external/token_metadata/nearblocks/complete/streamline__nearblocks_ft_complete.sql

@@ -9,18 +9,17 @@
 ) }}
 
 {% if var('NEAR_MIGRATE_ARCHIVE', false) %}
--- do not need to re-query for tokens we already have so just add to complete table once
--- especially with the low rate limit
+
 SELECT
     contract_address,
-    DATE_PART('EPOCH', _inserted_timestamp) :: INTEGER AS partition_key,
-    _inserted_timestamp,
+    DATE_PART('EPOCH', inserted_timestamp) :: INTEGER AS partition_key,
+    inserted_timestamp AS _inserted_timestamp,
     contract_address AS nearblocks_ft_complete_id,
-    COALESCE(inserted_timestamp, _inserted_timestamp) AS inserted_timestamp,
+    SYSDATE() AS inserted_timestamp,
     SYSDATE() AS modified_timestamp,
     '{{ invocation_id }}' AS _invocation_id
 FROM
-    near.silver.ft_contract_metadata
+    {{ ref('seeds__ft_token_details_final')}}
 
 {% else %}
 SELECT

models/streamline/external/token_metadata/nearblocks/legacy/livequery__request_nearblocks_nft_metadata.py

@@ -3,8 +3,6 @@ import snowflake.snowpark.types as T
 
 from datetime import datetime
 
-# Legacy model for NFT Contract Metadata
-# Still runs fine - keep? Or refactor?
 
 def model(dbt, session):
 

models/streamline/external/token_metadata/nearblocks/realtime/streamline__nearblocks_ft_realtime.sql

@@ -6,7 +6,7 @@
         target = "{{this.schema}}.{{this.identifier}}",
         params = {
             "external_table": "nearblocks_ft_metadata",
-            "sql_limit": "100",
+            "sql_limit": "500",
             "producer_batch_size": "50",
             "worker_batch_size": "10",
             "sql_source": "{{this.identifier}}"
@@ -15,11 +15,6 @@
     tags = ['streamline_non_core']
 ) }}
 
--- TODO plan to run this once daily
--- with the really low rate limit, we might get 30% of requests if we're lucky
--- have about 600 to catch up on and then likely very few per day, if any
--- exact batch size tbd
-
 WITH
 ft_tokenlist AS (