AN-3137/verified-token-list (#61)

.github/workflows/dbt_run_weekly.yml

@@ -42,4 +42,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run -m tag:tokenflow
+          dbt run -m tag:tokenflow tag:tokenlists

data/bronze__verified_tokenlist_seed.csv

@@ -0,0 +1,35 @@
+index,token_list,api_url
+1,1inch,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://tokens.1inch.eth.link
+2,Aave Token List,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://tokenlist.aave.eth.link
+3,Agora dataFi Tokens,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://datafi.theagora.eth.link
+4,BA ERC20 SEC Action,https://raw.githubusercontent.com/The-Blockchain-Association/sec-notice-list/master/ba-sec-list.json
+5,CMC DeFi,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://defi.cmc.eth.link
+6,CMC Stablecoin,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://stablecoin.cmc.eth.link
+7,CMC200 ERC20,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://erc20.cmc.eth.link
+8,8CompliFi Originals,https://compli.fi/complifi.tokenlist.json
+9,CoinGecko,https://tokens.coingecko.com/uniswap/all.json
+10,CoinGecko DeFi 100,https://www.coingecko.com/tokens_list/uniswap/defi_100/v_0_0_0.json
+11,Compound,https://raw.githubusercontent.com/compound-finance/token-list/master/compound.tokenlist.json
+12,Defiprime,https://defiprime.com/defiprime.tokenlist.json
+13,DFO Flexible Org,https://raw.githubusercontent.com/b-u-i-d-l/bazar-tokens-list/master/dist/decentralizedFlexibleOrganizations.json
+14,Dharma Token List,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://tokenlist.dharma.eth.link
+15,Furucombo,https://cdn.furucombo.app/furucombo.tokenlist.json
+16,Gemini Token List,https://www.gemini.com/uniswap/manifest.json
+17,Kleros Tokens,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://t2crtokens.eth.link
+18,Kyber,https://api.kyber.network/tokenlist
+19,Messari Verified,https://messari.io/tokenlist/messari-verified
+20,MyCrypto Token List,https://uniswap.mycryptoapi.com/
+21,Opyn Token List,https://raw.githubusercontent.com/opynfinance/opyn-tokenlist/master/opyn-v1.tokenlist.json
+22,Superchain Token List,https://static.optimism.io/optimism.tokenlist.json
+23,Roll Social Money,https://app.tryroll.com/tokens.json
+24,Set,https://raw.githubusercontent.com/SetProtocol/uniswap-tokenlist/main/set.tokenlist.json
+25,Synthetix,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://synths.snx.eth.link
+26,Testnet Token List,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://testnet.tokenlist.eth.link
+27,Token Name Service,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://list.tkn.eth.link
+28,Uniswap Default List,https://gateway.ipfs.io/ipns/tokens.uniswap.org
+29,Uniswap Token Pairs,https://raw.githubusercontent.com/jab416171/uniswap-pairtokens/master/uniswap_pair_tokens.json
+30,UMA,https://umaproject.org/uma.tokenlist.json
+31,Wrapped Tokens,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://wrapped.tokensoft.eth.link
+32,Yearn,https://yearn.science/static/tokenlist.json
+33,Zapper Token List,https://zapper.fi/api/token-list
+34,Zerion,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://tokenlist.zerion.eth.link

dbt_project.yml

@@ -51,4 +51,8 @@ on-run-end:
 vars:
   "dbt_date:time_zone": GMT
   UPDATE_UDFS_AND_SPS: False
-  UPDATE_SNOWFLAKE_TAGS: True
+  UPDATE_SNOWFLAKE_TAGS: True
+  API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' 
+  EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}'
+  ROLES: |
+      ["INTERNAL_DEV"]

models/doc_descriptions/general/__overview__.md

@@ -64,6 +64,12 @@ Note: These tables ceased updating on Feburary 4th, 2024.
 - [flashbots__fact_mevshare_transactions](https://flipsidecrypto.github.io/external-models/#!/model/model.external_models.flashbots__fact_mevshare_transactions)
 - [flashbots__fact_protect_transactions](https://flipsidecrypto.github.io/external-models/#!/model/model.external_models.flashbots__fact_protect_transactions)
 
+**Token Lists**
+
+[Token Lists Documentation](https://tokenlists.org/)
+
+- [tokenlists__ez_verified_tokens](https://flipsidecrypto.github.io/external-models/#!/model/model.external_models.tokenlists__ez_verified_tokens)
+
 ## **Helpful User-Defined Functions (UDFs)**
 
 UDFs are custom functions built by the Flipside team that can be used in your queries to make your life easier. 

models/doc_descriptions/general/export_columns.md

@@ -0,0 +1,19 @@
+{% docs pk %}
+
+The unique identifier for each row in the table.
+
+{% enddocs %}
+
+{% docs inserted_timestamp %}
+
+The utc timestamp at which the row was inserted into the table.
+
+{% enddocs %}
+
+{% docs modified_timestamp %}
+
+The utc timestamp at which the row was last modified.
+
+{% enddocs %}
+
+

models/tokenlists/bronze/bronze__all_tokenlists.sql

@@ -0,0 +1,41 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_strategy = 'delete+insert',
+    unique_key = 'all_tokenlists_id',
+    tags = ['tokenlists']
+) }}
+
+WITH calls AS ({% for item in range(5) %}
+    (
+
+    SELECT
+        api_url, 
+        {{ target.database }}.live.udf_api('GET', api_url,{},{}) AS request, 
+        INDEX AS row_num, 
+        SYSDATE() AS _inserted_timestamp
+    FROM
+        {{ ref('bronze__verified_tokenlist_seed') }}
+    WHERE
+        row_num BETWEEN {{ item * 10 + 1 }}
+        AND {{(item + 1) * 10 }}
+
+) {% if not loop.last %}
+UNION ALL
+{% endif %}
+{% endfor %})
+SELECT
+    api_url,
+    request,
+    request :data :keywords :: VARIANT AS keywords,
+    request :data :logoURI :: STRING AS logo_uri,
+    request :data :name :: STRING AS list_name,
+    request :data :tags :: VARIANT AS list_tags,
+    row_num,
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['api_url','request']
+    ) }} AS all_tokenlists_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp
+FROM
+    calls

models/tokenlists/bronze/bronze__all_tokenlists.yml

@@ -0,0 +1,16 @@
+version: 2
+models:
+  - name: bronze__all_tokenlists
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+            combination_of_columns:
+              - API_URL
+              - REQUEST
+    
+    columns:
+      - name: API_URL
+        tests:
+          - not_null
+      - name: REQUEST
+        tests: 
+          - not_null

models/tokenlists/gold/tokenlists__ez_verified_tokens.sql

@@ -0,0 +1,26 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true },
+    tags = ['tokenlists'],
+    meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'TOKEN LISTS' } } }
+) }}
+
+SELECT
+    LOWER(address) AS token_address,
+    NAME,
+    symbol,
+    decimals,
+    t.chain_id,
+    chain AS chain_name,
+    chain_symbol,
+    extensions AS token_extensions,
+    provider,
+    list_metadata,
+    tokenlists_verified_tokens_id AS ez_verified_tokens_id,
+    inserted_timestamp,
+    modified_timestamp
+FROM
+    {{ ref('silver__tokenlists_verified_tokens') }} t 
+LEFT JOIN {{ ref('defillama__dim_chains')}} d 
+    ON t.chain_id = d.chain_id

models/tokenlists/gold/tokenlists__ez_verified_tokens.yml

@@ -0,0 +1,32 @@
+version: 2
+models:
+  - name: tokenlists__ez_verified_tokens
+    description: A Uniswap Project, Token Lists is a community-led initiative to improve discoverability, reputation and trust in ERC20 token lists in a manner that is inclusive, transparent, and decentralized. This table contains dimensional information about the tokens listed on https://tokenlists.org/. This EZ view joins in Defillama's dim_chains for additional chain metadata.
+
+    columns:
+      - name: TOKEN_ADDRESS
+        description: The address of the token.
+      - name: NAME
+        description: The name of the token.
+      - name: SYMBOL
+        description: The symbol of the token.
+      - name: DECIMALS
+        description: The number of decimals of the token.
+      - name: CHAIN_ID
+        description: The unique identifier of the chain where the token is deployed.
+      - name: CHAIN_NAME
+        description: The name of the chain where the token is deployed. This is derived from `defillama.dim_chains`.
+      - name: CHAIN_SYMBOL
+        description: The official abbreviation or symbol representing the blockchain. This is derived from `defillama.dim_chains`.
+      - name: TOKEN_EXTENSIONS
+        description: Extended token information, including mappings of the tokens on other chains.
+      - name: PROVIDER
+        description: The provider of the token list.
+      - name: LIST_METADATA
+        description: The metadata of the token list.
+      - name: EZ_VERIFIED_TOKENS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 

models/tokenlists/silver/silver__tokenlists_verified_tokens.sql

@@ -0,0 +1,73 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_strategy = 'delete+insert',
+    unique_key = 'tokenlists_verified_tokens_id',
+    tags = ['tokenlists']
+) }}
+
+WITH base_lists AS (
+
+    SELECT
+        api_url,
+        request,
+        keywords,
+        logo_uri,
+        list_name,
+        list_tags,
+        VALUE :name :: STRING AS NAME,
+        VALUE :symbol :: STRING AS symbol,
+        VALUE :address :: STRING AS address,
+        VALUE :chainId :: STRING AS chain_id,
+        VALUE :decimals :: STRING AS decimals,
+        VALUE :extensions :: VARIANT AS extensions,
+        _inserted_timestamp,
+        all_tokenlists_id
+    FROM
+        {{ ref('bronze__all_tokenlists') }},
+        LATERAL FLATTEN (
+            input => request :data :tokens
+        )
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(_inserted_timestamp)
+        FROM
+            {{ this }}
+    )
+{% endif %}
+)
+SELECT
+    api_url,
+    NAME,
+    symbol,
+    address,
+    chain_id,
+    decimals,
+    extensions,
+    list_name AS provider,
+    list_tags AS list_metadata,
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['api_url', 'LOWER(address)', 'chain_id']
+    ) }} AS tokenlists_verified_tokens_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp
+FROM
+    base_lists
+WHERE
+    address IS NOT NULL
+
+{% if is_incremental() %}
+AND api_url || LOWER(address) || chain_id NOT IN (
+    SELECT
+        api_url || LOWER(address) || chain_id
+    FROM
+        {{ this }}
+)
+{% endif %}
+
+qualify(ROW_NUMBER() over (PARTITION BY api_url, LOWER(address), chain_id
+ORDER BY
+    _inserted_timestamp DESC)) = 1

models/tokenlists/silver/silver__tokenlists_verified_tokens.yml

@@ -0,0 +1,20 @@
+version: 2
+models:
+  - name: silver__tokenlists_verified_tokens
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+            combination_of_columns:
+              - API_URL
+              - LOWER(ADDRESS)
+              - CHAIN_ID
+    
+    columns:
+      - name: API_URL
+        tests:
+          - not_null
+      - name: ADDRESS
+        tests: 
+          - not_null
+      - name: CHAIN_ID
+        tests:
+          - not_null