diff --git a/.github/workflows/dbt_run_weekly.yml b/.github/workflows/dbt_run_weekly.yml index 68a1a9d..86e098a 100644 --- a/.github/workflows/dbt_run_weekly.yml +++ b/.github/workflows/dbt_run_weekly.yml @@ -42,4 +42,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m tag:tokenflow \ No newline at end of file + dbt run -m tag:tokenflow tag:tokenlists \ No newline at end of file diff --git a/data/bronze__verified_tokenlist_seed.csv b/data/bronze__verified_tokenlist_seed.csv new file mode 100644 index 0000000..ee4f55b --- /dev/null +++ b/data/bronze__verified_tokenlist_seed.csv @@ -0,0 +1,35 @@ +index,token_list,api_url +1,1inch,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://tokens.1inch.eth.link +2,Aave Token List,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://tokenlist.aave.eth.link +3,Agora dataFi Tokens,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://datafi.theagora.eth.link +4,BA ERC20 SEC Action,https://raw.githubusercontent.com/The-Blockchain-Association/sec-notice-list/master/ba-sec-list.json +5,CMC DeFi,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://defi.cmc.eth.link +6,CMC Stablecoin,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://stablecoin.cmc.eth.link +7,CMC200 ERC20,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://erc20.cmc.eth.link +8,8CompliFi Originals,https://compli.fi/complifi.tokenlist.json +9,CoinGecko,https://tokens.coingecko.com/uniswap/all.json +10,CoinGecko DeFi 100,https://www.coingecko.com/tokens_list/uniswap/defi_100/v_0_0_0.json +11,Compound,https://raw.githubusercontent.com/compound-finance/token-list/master/compound.tokenlist.json +12,Defiprime,https://defiprime.com/defiprime.tokenlist.json +13,DFO Flexible Org,https://raw.githubusercontent.com/b-u-i-d-l/bazar-tokens-list/master/dist/decentralizedFlexibleOrganizations.json +14,Dharma Token List,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://tokenlist.dharma.eth.link +15,Furucombo,https://cdn.furucombo.app/furucombo.tokenlist.json +16,Gemini Token List,https://www.gemini.com/uniswap/manifest.json +17,Kleros Tokens,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://t2crtokens.eth.link +18,Kyber,https://api.kyber.network/tokenlist +19,Messari Verified,https://messari.io/tokenlist/messari-verified +20,MyCrypto Token List,https://uniswap.mycryptoapi.com/ +21,Opyn Token List,https://raw.githubusercontent.com/opynfinance/opyn-tokenlist/master/opyn-v1.tokenlist.json +22,Superchain Token List,https://static.optimism.io/optimism.tokenlist.json +23,Roll Social Money,https://app.tryroll.com/tokens.json +24,Set,https://raw.githubusercontent.com/SetProtocol/uniswap-tokenlist/main/set.tokenlist.json +25,Synthetix,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://synths.snx.eth.link +26,Testnet Token List,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://testnet.tokenlist.eth.link +27,Token Name Service,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://list.tkn.eth.link +28,Uniswap Default List,https://gateway.ipfs.io/ipns/tokens.uniswap.org +29,Uniswap Token Pairs,https://raw.githubusercontent.com/jab416171/uniswap-pairtokens/master/uniswap_pair_tokens.json +30,UMA,https://umaproject.org/uma.tokenlist.json +31,Wrapped Tokens,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://wrapped.tokensoft.eth.link +32,Yearn,https://yearn.science/static/tokenlist.json +33,Zapper Token List,https://zapper.fi/api/token-list +34,Zerion,https://wispy-bird-88a7.uniswap.workers.dev/?url=https://tokenlist.zerion.eth.link \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index b1818dd..3185a1f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -51,4 +51,8 @@ on-run-end: vars: "dbt_date:time_zone": GMT UPDATE_UDFS_AND_SPS: False - UPDATE_SNOWFLAKE_TAGS: True \ No newline at end of file + UPDATE_SNOWFLAKE_TAGS: True + API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' + EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}' + ROLES: | + ["INTERNAL_DEV"] \ No newline at end of file diff --git a/models/doc_descriptions/general/__overview__.md b/models/doc_descriptions/general/__overview__.md index 77c2b24..7924526 100644 --- a/models/doc_descriptions/general/__overview__.md +++ b/models/doc_descriptions/general/__overview__.md @@ -64,6 +64,12 @@ Note: These tables ceased updating on Feburary 4th, 2024. - [flashbots__fact_mevshare_transactions](https://flipsidecrypto.github.io/external-models/#!/model/model.external_models.flashbots__fact_mevshare_transactions) - [flashbots__fact_protect_transactions](https://flipsidecrypto.github.io/external-models/#!/model/model.external_models.flashbots__fact_protect_transactions) +**Token Lists** + +[Token Lists Documentation](https://tokenlists.org/) + +- [tokenlists__ez_verified_tokens](https://flipsidecrypto.github.io/external-models/#!/model/model.external_models.tokenlists__ez_verified_tokens) + ## **Helpful User-Defined Functions (UDFs)** UDFs are custom functions built by the Flipside team that can be used in your queries to make your life easier. diff --git a/models/doc_descriptions/general/export_columns.md b/models/doc_descriptions/general/export_columns.md new file mode 100644 index 0000000..848af44 --- /dev/null +++ b/models/doc_descriptions/general/export_columns.md @@ -0,0 +1,19 @@ +{% docs pk %} + +The unique identifier for each row in the table. + +{% enddocs %} + +{% docs inserted_timestamp %} + +The utc timestamp at which the row was inserted into the table. + +{% enddocs %} + +{% docs modified_timestamp %} + +The utc timestamp at which the row was last modified. + +{% enddocs %} + + diff --git a/models/doc_descriptions/general/snapshot/snapshot_proposals.md b/models/doc_descriptions/snapshot/snapshot_proposals.md similarity index 100% rename from models/doc_descriptions/general/snapshot/snapshot_proposals.md rename to models/doc_descriptions/snapshot/snapshot_proposals.md diff --git a/models/doc_descriptions/general/snapshot/snapshot_spaces.md b/models/doc_descriptions/snapshot/snapshot_spaces.md similarity index 100% rename from models/doc_descriptions/general/snapshot/snapshot_spaces.md rename to models/doc_descriptions/snapshot/snapshot_spaces.md diff --git a/models/doc_descriptions/general/snapshot/snapshot_users.md b/models/doc_descriptions/snapshot/snapshot_users.md similarity index 100% rename from models/doc_descriptions/general/snapshot/snapshot_users.md rename to models/doc_descriptions/snapshot/snapshot_users.md diff --git a/models/doc_descriptions/general/snapshot/snapshot_votes.md b/models/doc_descriptions/snapshot/snapshot_votes.md similarity index 100% rename from models/doc_descriptions/general/snapshot/snapshot_votes.md rename to models/doc_descriptions/snapshot/snapshot_votes.md diff --git a/models/tokenlists/bronze/bronze__all_tokenlists.sql b/models/tokenlists/bronze/bronze__all_tokenlists.sql new file mode 100644 index 0000000..fbb8104 --- /dev/null +++ b/models/tokenlists/bronze/bronze__all_tokenlists.sql @@ -0,0 +1,41 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'all_tokenlists_id', + tags = ['tokenlists'] +) }} + +WITH calls AS ({% for item in range(5) %} + ( + + SELECT + api_url, + {{ target.database }}.live.udf_api('GET', api_url,{},{}) AS request, + INDEX AS row_num, + SYSDATE() AS _inserted_timestamp + FROM + {{ ref('bronze__verified_tokenlist_seed') }} + WHERE + row_num BETWEEN {{ item * 10 + 1 }} + AND {{(item + 1) * 10 }} + +) {% if not loop.last %} +UNION ALL +{% endif %} +{% endfor %}) +SELECT + api_url, + request, + request :data :keywords :: VARIANT AS keywords, + request :data :logoURI :: STRING AS logo_uri, + request :data :name :: STRING AS list_name, + request :data :tags :: VARIANT AS list_tags, + row_num, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['api_url','request'] + ) }} AS all_tokenlists_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + calls diff --git a/models/tokenlists/bronze/bronze__all_tokenlists.yml b/models/tokenlists/bronze/bronze__all_tokenlists.yml new file mode 100644 index 0000000..4703f07 --- /dev/null +++ b/models/tokenlists/bronze/bronze__all_tokenlists.yml @@ -0,0 +1,16 @@ +version: 2 +models: + - name: bronze__all_tokenlists + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - API_URL + - REQUEST + + columns: + - name: API_URL + tests: + - not_null + - name: REQUEST + tests: + - not_null \ No newline at end of file diff --git a/models/tokenlists/gold/tokenlists__ez_verified_tokens.sql b/models/tokenlists/gold/tokenlists__ez_verified_tokens.sql new file mode 100644 index 0000000..41f4f5c --- /dev/null +++ b/models/tokenlists/gold/tokenlists__ez_verified_tokens.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + tags = ['tokenlists'], + meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'TOKEN LISTS' } } } +) }} + +SELECT + LOWER(address) AS token_address, + NAME, + symbol, + decimals, + t.chain_id, + chain AS chain_name, + chain_symbol, + extensions AS token_extensions, + provider, + list_metadata, + tokenlists_verified_tokens_id AS ez_verified_tokens_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__tokenlists_verified_tokens') }} t +LEFT JOIN {{ ref('defillama__dim_chains')}} d + ON t.chain_id = d.chain_id diff --git a/models/tokenlists/gold/tokenlists__ez_verified_tokens.yml b/models/tokenlists/gold/tokenlists__ez_verified_tokens.yml new file mode 100644 index 0000000..d213703 --- /dev/null +++ b/models/tokenlists/gold/tokenlists__ez_verified_tokens.yml @@ -0,0 +1,32 @@ +version: 2 +models: + - name: tokenlists__ez_verified_tokens + description: A Uniswap Project, Token Lists is a community-led initiative to improve discoverability, reputation and trust in ERC20 token lists in a manner that is inclusive, transparent, and decentralized. This table contains dimensional information about the tokens listed on https://tokenlists.org/. This EZ view joins in Defillama's dim_chains for additional chain metadata. + + columns: + - name: TOKEN_ADDRESS + description: The address of the token. + - name: NAME + description: The name of the token. + - name: SYMBOL + description: The symbol of the token. + - name: DECIMALS + description: The number of decimals of the token. + - name: CHAIN_ID + description: The unique identifier of the chain where the token is deployed. + - name: CHAIN_NAME + description: The name of the chain where the token is deployed. This is derived from `defillama.dim_chains`. + - name: CHAIN_SYMBOL + description: The official abbreviation or symbol representing the blockchain. This is derived from `defillama.dim_chains`. + - name: TOKEN_EXTENSIONS + description: Extended token information, including mappings of the tokens on other chains. + - name: PROVIDER + description: The provider of the token list. + - name: LIST_METADATA + description: The metadata of the token list. + - name: EZ_VERIFIED_TOKENS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/tokenlists/silver/silver__tokenlists_verified_tokens.sql b/models/tokenlists/silver/silver__tokenlists_verified_tokens.sql new file mode 100644 index 0000000..6c0fc82 --- /dev/null +++ b/models/tokenlists/silver/silver__tokenlists_verified_tokens.sql @@ -0,0 +1,73 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'tokenlists_verified_tokens_id', + tags = ['tokenlists'] +) }} + +WITH base_lists AS ( + + SELECT + api_url, + request, + keywords, + logo_uri, + list_name, + list_tags, + VALUE :name :: STRING AS NAME, + VALUE :symbol :: STRING AS symbol, + VALUE :address :: STRING AS address, + VALUE :chainId :: STRING AS chain_id, + VALUE :decimals :: STRING AS decimals, + VALUE :extensions :: VARIANT AS extensions, + _inserted_timestamp, + all_tokenlists_id + FROM + {{ ref('bronze__all_tokenlists') }}, + LATERAL FLATTEN ( + input => request :data :tokens + ) + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + api_url, + NAME, + symbol, + address, + chain_id, + decimals, + extensions, + list_name AS provider, + list_tags AS list_metadata, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['api_url', 'LOWER(address)', 'chain_id'] + ) }} AS tokenlists_verified_tokens_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base_lists +WHERE + address IS NOT NULL + +{% if is_incremental() %} +AND api_url || LOWER(address) || chain_id NOT IN ( + SELECT + api_url || LOWER(address) || chain_id + FROM + {{ this }} +) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY api_url, LOWER(address), chain_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/tokenlists/silver/silver__tokenlists_verified_tokens.yml b/models/tokenlists/silver/silver__tokenlists_verified_tokens.yml new file mode 100644 index 0000000..c5b9f6f --- /dev/null +++ b/models/tokenlists/silver/silver__tokenlists_verified_tokens.yml @@ -0,0 +1,20 @@ +version: 2 +models: + - name: silver__tokenlists_verified_tokens + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - API_URL + - LOWER(ADDRESS) + - CHAIN_ID + + columns: + - name: API_URL + tests: + - not_null + - name: ADDRESS + tests: + - not_null + - name: CHAIN_ID + tests: + - not_null