From 5411f3076bc411dbb4e4933cb47039f9dcc9d22d Mon Sep 17 00:00:00 2001 From: Matt Romano <42412983+mattromano@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:57:57 -0700 Subject: [PATCH] AN-4298/add-abi-and-read-udfs-to-external (#55) --- .../deepnftvalue/bronze/bronze__dnv_collections.sql | 2 +- models/deepnftvalue/bronze/bronze__dnv_tokens.sql | 2 +- models/defillama/bronze/bronze__defillama_bridges.sql | 2 +- .../defillama/bronze/bronze__defillama_chainlist.sql | 2 +- models/defillama/bronze/bronze__defillama_chains.sql | 2 +- models/defillama/bronze/bronze__defillama_dexes.sql | 4 ++-- .../defillama/bronze/bronze__defillama_protocols.sql | 2 +- .../bronze/bronze__defillama_stablecoins.sql | 2 +- .../silver/silver__defillama_bridge_volume.sql | 2 +- .../defillama/silver/silver__defillama_chains_tvl.sql | 2 +- .../defillama/silver/silver__defillama_dex_volume.sql | 4 ++-- .../silver/silver__defillama_options_notional.sql | 4 ++-- .../silver/silver__defillama_options_premium.sql | 4 ++-- .../silver/silver__defillama_protocol_fees.sql | 4 ++-- .../silver/silver__defillama_protocol_revenue.sql | 4 ++-- models/snapshot/bronze/bronze__snapshot_proposals.sql | 11 ++++------- models/snapshot/bronze/bronze__snapshot_spaces.sql | 8 +------- models/snapshot/bronze/bronze__snapshot_users.sql | 10 ++-------- models/snapshot/bronze/bronze__snapshot_votes.sql | 10 ++-------- packages.yml | 2 +- 20 files changed, 31 insertions(+), 52 deletions(-) diff --git a/models/deepnftvalue/bronze/bronze__dnv_collections.sql b/models/deepnftvalue/bronze/bronze__dnv_collections.sql index 156a1f6..60e4add 100644 --- a/models/deepnftvalue/bronze/bronze__dnv_collections.sql +++ b/models/deepnftvalue/bronze/bronze__dnv_collections.sql @@ -57,7 +57,7 @@ row_nos AS ( ), batched AS ({% for item in range(10) %} SELECT - ethereum.streamline.udf_api(' GET ', api_url, PARSE_JSON(header),{}) AS resp, SYSDATE() _inserted_timestamp, collection_slug, CONCAT(collection_slug, '-', _inserted_timestamp) AS _id + live.udf_api(' GET ', api_url, PARSE_JSON(header),{}) AS resp, SYSDATE() _inserted_timestamp, collection_slug, CONCAT(collection_slug, '-', _inserted_timestamp) AS _id FROM row_nos rn WHERE diff --git a/models/deepnftvalue/bronze/bronze__dnv_tokens.sql b/models/deepnftvalue/bronze/bronze__dnv_tokens.sql index ed6da50..0b63ad9 100644 --- a/models/deepnftvalue/bronze/bronze__dnv_tokens.sql +++ b/models/deepnftvalue/bronze/bronze__dnv_tokens.sql @@ -60,7 +60,7 @@ row_nos AS ( ), batched AS ({% for item in range(9) %} SELECT - ethereum.streamline.udf_api(' GET ', api_url, PARSE_JSON(header),{}) AS resp, SYSDATE() _inserted_timestamp, collection_slug, _id + live.udf_api(' GET ', api_url, PARSE_JSON(header),{}) AS resp, SYSDATE() _inserted_timestamp, collection_slug, _id FROM row_nos rn WHERE diff --git a/models/defillama/bronze/bronze__defillama_bridges.sql b/models/defillama/bronze/bronze__defillama_bridges.sql index 6fd3afd..528d0cb 100644 --- a/models/defillama/bronze/bronze__defillama_bridges.sql +++ b/models/defillama/bronze/bronze__defillama_bridges.sql @@ -8,7 +8,7 @@ WITH bridge_base AS ( SELECT - ethereum.streamline.udf_api( + live.udf_api( 'GET','https://bridges.llama.fi/bridges?includeChains=true',{},{} ) AS read, SYSDATE() AS _inserted_timestamp diff --git a/models/defillama/bronze/bronze__defillama_chainlist.sql b/models/defillama/bronze/bronze__defillama_chainlist.sql index ea18480..f19e2e5 100644 --- a/models/defillama/bronze/bronze__defillama_chainlist.sql +++ b/models/defillama/bronze/bronze__defillama_chainlist.sql @@ -8,7 +8,7 @@ WITH chainlist_base AS ( SELECT - ethereum.streamline.udf_api( + live.udf_api( 'GET', 'https://chainid.network/chains.json',{},{} ) AS READ, diff --git a/models/defillama/bronze/bronze__defillama_chains.sql b/models/defillama/bronze/bronze__defillama_chains.sql index 45d6bc9..8dace83 100644 --- a/models/defillama/bronze/bronze__defillama_chains.sql +++ b/models/defillama/bronze/bronze__defillama_chains.sql @@ -7,7 +7,7 @@ WITH chain_base AS ( SELECT - ethereum.streamline.udf_api( + live.udf_api( 'GET','https://api.llama.fi/chains',{},{} ) AS read, SYSDATE() AS _inserted_timestamp diff --git a/models/defillama/bronze/bronze__defillama_dexes.sql b/models/defillama/bronze/bronze__defillama_dexes.sql index ffa3eee..06e981b 100644 --- a/models/defillama/bronze/bronze__defillama_dexes.sql +++ b/models/defillama/bronze/bronze__defillama_dexes.sql @@ -7,10 +7,10 @@ WITH base AS ( SELECT - ethereum.streamline.udf_api( + live.udf_api( 'GET','https://api.llama.fi/overview/dexs?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=totalVolume',{},{} ) AS dex_read, - ethereum.streamline.udf_api( + live.udf_api( 'GET','https://api.llama.fi/overview/options?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=totalPremiumVolume',{},{} ) AS options_read, SYSDATE() AS _inserted_timestamp diff --git a/models/defillama/bronze/bronze__defillama_protocols.sql b/models/defillama/bronze/bronze__defillama_protocols.sql index df59e0f..aac286a 100644 --- a/models/defillama/bronze/bronze__defillama_protocols.sql +++ b/models/defillama/bronze/bronze__defillama_protocols.sql @@ -7,7 +7,7 @@ WITH protocol_base AS ( SELECT - ethereum.streamline.udf_api( + live.udf_api( 'GET','https://api.llama.fi/protocols',{},{} ) AS read, SYSDATE() AS _inserted_timestamp diff --git a/models/defillama/bronze/bronze__defillama_stablecoins.sql b/models/defillama/bronze/bronze__defillama_stablecoins.sql index de59845..bf3b0a6 100644 --- a/models/defillama/bronze/bronze__defillama_stablecoins.sql +++ b/models/defillama/bronze/bronze__defillama_stablecoins.sql @@ -7,7 +7,7 @@ WITH stablecoin_base AS ( SELECT - ethereum.streamline.udf_api( + live.udf_api( 'GET','https://stablecoins.llama.fi/stablecoins?includePrices=false',{},{} ) AS read, SYSDATE() AS _inserted_timestamp diff --git a/models/defillama/silver/silver__defillama_bridge_volume.sql b/models/defillama/silver/silver__defillama_bridge_volume.sql index b4e5ca7..53f7183 100644 --- a/models/defillama/silver/silver__defillama_bridge_volume.sql +++ b/models/defillama/silver/silver__defillama_bridge_volume.sql @@ -13,7 +13,7 @@ SELECT bridge_id, bridge, bridge_name, - ethereum.streamline.udf_api( + live.udf_api( 'GET',CONCAT('https://bridges.llama.fi/bridgevolume/all?id=',bridge_id),{},{} ) AS read, SYSDATE() AS _inserted_timestamp diff --git a/models/defillama/silver/silver__defillama_chains_tvl.sql b/models/defillama/silver/silver__defillama_chains_tvl.sql index 5a6a131..2b163cd 100644 --- a/models/defillama/silver/silver__defillama_chains_tvl.sql +++ b/models/defillama/silver/silver__defillama_chains_tvl.sql @@ -12,7 +12,7 @@ WITH tvl_base AS ( SELECT chain_id, chain, - ethereum.streamline.udf_api( + live.udf_api( 'GET',CONCAT('https://api.llama.fi/charts/',chain),{},{} ) AS read, SYSDATE() AS _inserted_timestamp diff --git a/models/defillama/silver/silver__defillama_dex_volume.sql b/models/defillama/silver/silver__defillama_dex_volume.sql index 823aa77..358a05f 100644 --- a/models/defillama/silver/silver__defillama_dex_volume.sql +++ b/models/defillama/silver/silver__defillama_dex_volume.sql @@ -13,7 +13,7 @@ SELECT _inserted_timestamp FROM ( SELECT - ethereum.streamline.udf_api( + live.udf_api( 'GET','https://api.llama.fi/overview/dexs?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=dailyVolume',{},{} ) AS read, SYSDATE() AS _inserted_timestamp @@ -27,7 +27,7 @@ dex_base AS ( ( SELECT chain, - ethereum.streamline.udf_api( + live.udf_api( 'GET',CONCAT('https://api.llama.fi/overview/dexs/',chain,'?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=false&dataType=dailyVolume'),{},{} ) AS read, SYSDATE() AS _inserted_timestamp diff --git a/models/defillama/silver/silver__defillama_options_notional.sql b/models/defillama/silver/silver__defillama_options_notional.sql index 552f983..cd47014 100644 --- a/models/defillama/silver/silver__defillama_options_notional.sql +++ b/models/defillama/silver/silver__defillama_options_notional.sql @@ -13,7 +13,7 @@ SELECT _inserted_timestamp FROM ( SELECT - ethereum.streamline.udf_api( + live.udf_api( 'GET','https://api.llama.fi/overview/options?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=dailyNotionalVolume',{},{} ) AS read, SYSDATE() AS _inserted_timestamp @@ -27,7 +27,7 @@ options_base AS ( ( SELECT chain, - ethereum.streamline.udf_api( + live.udf_api( 'GET',CONCAT('https://api.llama.fi/overview/options/',chain,'?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=false&dataType=dailyNotionalVolume'),{},{} ) AS read, SYSDATE() AS _inserted_timestamp diff --git a/models/defillama/silver/silver__defillama_options_premium.sql b/models/defillama/silver/silver__defillama_options_premium.sql index b77b8cb..01d2220 100644 --- a/models/defillama/silver/silver__defillama_options_premium.sql +++ b/models/defillama/silver/silver__defillama_options_premium.sql @@ -13,7 +13,7 @@ SELECT _inserted_timestamp FROM ( SELECT - ethereum.streamline.udf_api( + live.udf_api( 'GET','https://api.llama.fi/overview/options?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=dailyPremiumVolume',{},{} ) AS read, SYSDATE() AS _inserted_timestamp @@ -27,7 +27,7 @@ options_base AS ( ( SELECT chain, - ethereum.streamline.udf_api( + live.udf_api( 'GET',CONCAT('https://api.llama.fi/overview/options/',chain,'?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=false&dataType=dailyPremiumVolume'),{},{} ) AS read, SYSDATE() AS _inserted_timestamp diff --git a/models/defillama/silver/silver__defillama_protocol_fees.sql b/models/defillama/silver/silver__defillama_protocol_fees.sql index d254e4e..64be38d 100644 --- a/models/defillama/silver/silver__defillama_protocol_fees.sql +++ b/models/defillama/silver/silver__defillama_protocol_fees.sql @@ -13,7 +13,7 @@ SELECT _inserted_timestamp FROM ( SELECT - ethereum.streamline.udf_api( + live.udf_api( 'GET','https://api.llama.fi/overview/fees?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=totalFees',{},{} ) AS read, SYSDATE() AS _inserted_timestamp @@ -27,7 +27,7 @@ fees_base AS ( ( SELECT chain, - ethereum.streamline.udf_api( + live.udf_api( 'GET',CONCAT('https://api.llama.fi/overview/fees/',chain,'?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=false&dataType=dailyFees'),{},{} ) AS read, SYSDATE() AS _inserted_timestamp diff --git a/models/defillama/silver/silver__defillama_protocol_revenue.sql b/models/defillama/silver/silver__defillama_protocol_revenue.sql index 9b3c189..7b3c533 100644 --- a/models/defillama/silver/silver__defillama_protocol_revenue.sql +++ b/models/defillama/silver/silver__defillama_protocol_revenue.sql @@ -13,7 +13,7 @@ SELECT _inserted_timestamp FROM ( SELECT - ethereum.streamline.udf_api( + live.udf_api( 'GET','https://api.llama.fi/overview/fees?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=totalRevenue',{},{} ) AS read, SYSDATE() AS _inserted_timestamp @@ -27,7 +27,7 @@ rev_base AS ( ( SELECT chain, - ethereum.streamline.udf_api( + live.udf_api( 'GET',CONCAT('https://api.llama.fi/overview/fees/',chain,'?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=false&dataType=dailyRevenue'),{},{} ) AS read, SYSDATE() AS _inserted_timestamp diff --git a/models/snapshot/bronze/bronze__snapshot_proposals.sql b/models/snapshot/bronze/bronze__snapshot_proposals.sql index 3876892..5a240ab 100644 --- a/models/snapshot/bronze/bronze__snapshot_proposals.sql +++ b/models/snapshot/bronze/bronze__snapshot_proposals.sql @@ -9,13 +9,10 @@ WITH requests AS ({% for item in range(6) %} ( SELECT - ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': ( - SELECT - api_key - FROM - {{ source('crosschain_silver', 'apis_keys') }} - WHERE - api_name = 'snapshot') },{ 'query':'query { proposals(orderBy: "created", orderDirection: asc,first:1000, skip: ' || {{ item * 1000 }} || ',where:{created_gte: ' || max_time_start || '}) { id space{id voting {delay quorum period type}} ipfs author created network type title body start end state votes choices scores_state scores } }' }) AS resp, SYSDATE() AS _inserted_timestamp + live.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey':'key' }, + { 'query':'query { proposals(orderBy: "created", orderDirection: asc,first:1000, skip: ' || {{ item * 1000 }} || ',where:{created_gte: ' || max_time_start || '}) { id space{id voting {delay quorum period type}} ipfs author created network type title body start end state votes choices scores_state scores } }' } + ,'Vault/prod/external/snapshot') AS resp, + SYSDATE() AS _inserted_timestamp FROM ( SELECT diff --git a/models/snapshot/bronze/bronze__snapshot_spaces.sql b/models/snapshot/bronze/bronze__snapshot_spaces.sql index 2d9b14c..ba155a2 100644 --- a/models/snapshot/bronze/bronze__snapshot_spaces.sql +++ b/models/snapshot/bronze/bronze__snapshot_spaces.sql @@ -10,13 +10,7 @@ WITH requests AS ({% for item in range(6) %} ( SELECT - ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': ( - SELECT - api_key - FROM - {{ source('crosschain_silver', 'apis_keys') }} - WHERE - api_name = 'snapshot') },{ 'query': 'query { spaces(orderBy: "created", orderDirection: desc, first: 1000, skip: ' || {{ item * 1000 }} || ') { id name about network symbol admins members categories domain private treasuries { address name network } verified } }' }) AS resp, SYSDATE() AS _inserted_timestamp) {% if not loop.last %} + live.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey':'key' },{ 'query': 'query { spaces(orderBy: "created", orderDirection: desc, first: 1000, skip: ' || {{ item * 1000 }} || ') { id name about network symbol admins members categories domain private treasuries { address name network } verified } }' },'Vault/prod/external/snapshot') AS resp, SYSDATE() AS _inserted_timestamp) {% if not loop.last %} UNION ALL {% endif %} {% endfor %}), diff --git a/models/snapshot/bronze/bronze__snapshot_users.sql b/models/snapshot/bronze/bronze__snapshot_users.sql index 59f1011..9dd9b96 100644 --- a/models/snapshot/bronze/bronze__snapshot_users.sql +++ b/models/snapshot/bronze/bronze__snapshot_users.sql @@ -10,13 +10,7 @@ WITH initial_request AS ({% for item in range(6) %} ( SELECT - ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': ( - SELECT - api_key - FROM - {{ source('crosschain_silver', 'apis_keys') }} - WHERE - api_name = 'snapshot') },{ 'query': 'query { users(orderBy: "created", orderDirection: asc, first: 1000, skip: ' || {{ item * 1000 }} || ', where:{created_gte: ' || max_time_start || '}) { id name about avatar ipfs created } }' }) AS resp, SYSDATE() AS _inserted_timestamp + live.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey':'key' },{ 'query': 'query { users(orderBy: "created", orderDirection: asc, first: 1000, skip: ' || {{ item * 1000 }} || ', where:{created_gte: ' || max_time_start || '}) { id name about avatar ipfs created } }' },'Vault/prod/external/snapshot') AS resp, SYSDATE() AS _inserted_timestamp FROM ( SELECT @@ -60,7 +54,7 @@ users_initial AS ( final_request AS ({% for item in range(6) %} ( SELECT - ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': ( + live.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': ( SELECT api_key FROM diff --git a/models/snapshot/bronze/bronze__snapshot_votes.sql b/models/snapshot/bronze/bronze__snapshot_votes.sql index 42e56c6..82c8798 100644 --- a/models/snapshot/bronze/bronze__snapshot_votes.sql +++ b/models/snapshot/bronze/bronze__snapshot_votes.sql @@ -9,13 +9,7 @@ WITH initial_votes_request AS ({% for item in range(6) %} ( SELECT - ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': ( - SELECT - api_key - FROM - {{ source('crosschain_silver', 'apis_keys') }} - WHERE - api_name = 'snapshot') },{ 'query': 'query { votes(orderBy: "created", orderDirection: asc, first: 1000, skip: ' || {{ item * 1000 }} || ', where:{created_gte: ' || max_time_start || ',created_lt: ' || max_time_end || '}) { id proposal{id} ipfs voter created choice vp } }' }) AS resp, SYSDATE() AS _inserted_timestamp + live.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey':'key' },{ 'query': 'query { votes(orderBy: "created", orderDirection: asc, first: 1000, skip: ' || {{ item * 1000 }} || ', where:{created_gte: ' || max_time_start || ',created_lt: ' || max_time_end || '}) { id proposal{id} ipfs voter created choice vp } }' },'Vault/prod/external/snapshot') AS resp, SYSDATE() AS _inserted_timestamp FROM ( SELECT @@ -61,7 +55,7 @@ votes_initial AS ( final_votes_request AS ({% for item in range(6) %} ( SELECT - ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': ( + live.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': ( SELECT api_key FROM diff --git a/packages.yml b/packages.yml index 8a8763a..e2fbd2e 100644 --- a/packages.yml +++ b/packages.yml @@ -6,4 +6,4 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.4.1 \ No newline at end of file + revision: v1.21.7 \ No newline at end of file