mirror of
https://github.com/FlipsideCrypto/external-models.git
synced 2026-02-06 11:21:59 +00:00
AN-5659/defillama-dex-volume-api-fixes (#88)
This commit is contained in:
Matt Romano
GitHub
parent
616eb9f4f3
commit
f32106dc6e
@ -1,5 +1,6 @@
|
||||
{{ config(
|
||||
materialized = 'table',
|
||||
enabled = false,
|
||||
unique_key = ['stablecoin_id','timestamp'],
|
||||
tags = ['defillama']
|
||||
) }}
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
persist_docs ={ "relation": true,
|
||||
"columns": true },
|
||||
tags = ['defillama'],
|
||||
enabled = false,
|
||||
meta={
|
||||
'database_tags':{
|
||||
'table': {
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
{{ config(
|
||||
materialized = 'view',
|
||||
enabled = false,
|
||||
persist_docs ={ "relation": true,
|
||||
"columns": true },
|
||||
tags = ['defillama'],
|
||||
|
||||
@ -20,6 +20,9 @@ models:
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 2
|
||||
- name: ID
|
||||
tests:
|
||||
- not_null
|
||||
@ -14,6 +14,9 @@ models:
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 2
|
||||
- name: ID
|
||||
tests:
|
||||
- not_null
|
||||
@ -5,79 +5,53 @@
|
||||
tags = ['defillama']
|
||||
) }}
|
||||
|
||||
WITH all_chains_dex_base AS (
|
||||
WITH api_pull AS (
|
||||
|
||||
SELECT
|
||||
LOWER(VALUE::STRING) AS chain,
|
||||
ROW_NUMBER() OVER (ORDER BY chain) AS row_num,
|
||||
_inserted_timestamp
|
||||
FROM (
|
||||
SELECT
|
||||
live.udf_api(
|
||||
'GET','https://api.llama.fi/overview/dexs?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=dailyVolume',{},{}
|
||||
) AS read,
|
||||
PARSE_JSON(
|
||||
live.udf_api(
|
||||
'GET',
|
||||
'https://api.llama.fi/overview/dexs?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=dailyVolume',{},{}
|
||||
)
|
||||
) :data :protocols AS response,
|
||||
SYSDATE() AS _inserted_timestamp
|
||||
),
|
||||
LATERAL FLATTEN (input=> read:data:allChains)
|
||||
),
|
||||
|
||||
dex_base AS (
|
||||
|
||||
{% for item in range(10) %}
|
||||
(
|
||||
SELECT
|
||||
chain,
|
||||
live.udf_api(
|
||||
'GET',CONCAT('https://api.llama.fi/overview/dexs/',chain,'?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=false&dataType=dailyVolume'),{},{}
|
||||
) AS read,
|
||||
SYSDATE() AS _inserted_timestamp
|
||||
FROM (
|
||||
SELECT
|
||||
DISTINCT chain,
|
||||
row_num
|
||||
FROM all_chains_dex_base
|
||||
WHERE row_num BETWEEN {{ item * 20 + 1 }} AND {{ (item + 1) * 20 }}
|
||||
)
|
||||
{% if is_incremental() %}
|
||||
WHERE chain NOT IN (
|
||||
lat_flat AS (
|
||||
SELECT
|
||||
chain
|
||||
FROM (
|
||||
SELECT
|
||||
DISTINCT chain,
|
||||
MAX(timestamp::DATE) AS max_timestamp
|
||||
FROM {{ this }}
|
||||
GROUP BY 1
|
||||
HAVING CURRENT_DATE = max_timestamp
|
||||
)
|
||||
)
|
||||
{% endif %}
|
||||
) {% if not loop.last %}
|
||||
UNION ALL
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
r.value AS VALUE,
|
||||
_inserted_timestamp
|
||||
FROM
|
||||
api_pull,
|
||||
LATERAL FLATTEN (
|
||||
input => response
|
||||
) AS r
|
||||
),
|
||||
|
||||
reads_output AS (
|
||||
|
||||
SELECT
|
||||
chain,
|
||||
TO_TIMESTAMP(VALUE[0]::INTEGER) AS timestamp,
|
||||
VALUE[1] AS dex_object,
|
||||
_inserted_timestamp
|
||||
FROM dex_base,
|
||||
LATERAL FLATTEN (input=> read:data:totalDataChartBreakdown)
|
||||
chain_breakdown AS (
|
||||
SELECT
|
||||
k.key AS chain,
|
||||
SYSDATE() :: DATE AS TIMESTAMP,
|
||||
v.key AS protocol,
|
||||
k.value AS dex_object,
|
||||
v.value :: INTEGER AS daily_volume,
|
||||
_inserted_timestamp
|
||||
FROM
|
||||
lat_flat,
|
||||
LATERAL FLATTEN(
|
||||
input => VALUE :breakdown24h
|
||||
) k,
|
||||
LATERAL FLATTEN(
|
||||
input => k.value
|
||||
) v
|
||||
)
|
||||
|
||||
SELECT
|
||||
chain,
|
||||
timestamp,
|
||||
LOWER(key::STRING) AS protocol,
|
||||
value::INTEGER AS daily_volume,
|
||||
TIMESTAMP,
|
||||
LOWER(protocol) AS protocol,
|
||||
daily_volume,
|
||||
dex_object,
|
||||
_inserted_timestamp,
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['chain', 'protocol', 'timestamp']
|
||||
) }} AS id
|
||||
FROM reads_output,
|
||||
LATERAL FLATTEN(input => PARSE_JSON(reads_output.dex_object))
|
||||
FROM
|
||||
chain_breakdown
|
||||
@ -16,6 +16,9 @@ models:
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 1
|
||||
- name: ID
|
||||
tests:
|
||||
- not_null
|
||||
@ -1,6 +1,7 @@
|
||||
{{ config(
|
||||
materialized = 'incremental',
|
||||
unique_key = 'id',
|
||||
enabled = false,
|
||||
full_refresh = false,
|
||||
tags = ['defillama']
|
||||
) }}
|
||||
|
||||
@ -16,6 +16,9 @@ models:
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 2
|
||||
- name: ID
|
||||
tests:
|
||||
- not_null
|
||||
@ -1,6 +1,7 @@
|
||||
{{ config(
|
||||
materialized = 'incremental',
|
||||
unique_key = 'id',
|
||||
enabled = false,
|
||||
full_refresh = false,
|
||||
tags = ['defillama']
|
||||
) }}
|
||||
|
||||
@ -16,6 +16,9 @@ models:
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 2
|
||||
- name: ID
|
||||
tests:
|
||||
- not_null
|
||||
@ -9,6 +9,12 @@ models:
|
||||
- name: _INSERTED_TIMESTAMP
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 2
|
||||
- name: TIMESTAMP
|
||||
tests:
|
||||
- not_null
|
||||
@ -16,6 +16,9 @@ models:
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 2
|
||||
- name: ID
|
||||
tests:
|
||||
- not_null
|
||||
@ -5,79 +5,54 @@
|
||||
tags = ['defillama']
|
||||
) }}
|
||||
|
||||
WITH all_chains_rev_base AS (
|
||||
WITH api_pull AS (
|
||||
|
||||
SELECT
|
||||
LOWER(VALUE::STRING) AS chain,
|
||||
ROW_NUMBER() OVER (ORDER BY chain) AS row_num,
|
||||
_inserted_timestamp
|
||||
FROM (
|
||||
SELECT
|
||||
live.udf_api(
|
||||
'GET','https://api.llama.fi/overview/fees?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=dailyRevenue',{},{}
|
||||
) AS read,
|
||||
PARSE_JSON(
|
||||
live.udf_api(
|
||||
'GET',
|
||||
'https://api.llama.fi/overview/fees?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=dailyRevenue',{},{}
|
||||
)
|
||||
) :data :protocols AS response,
|
||||
SYSDATE() AS _inserted_timestamp
|
||||
),
|
||||
LATERAL FLATTEN (input=> read:data:allChains)
|
||||
),
|
||||
|
||||
rev_base AS (
|
||||
|
||||
{% for item in range(7) %}
|
||||
(
|
||||
SELECT
|
||||
chain,
|
||||
live.udf_api(
|
||||
'GET',CONCAT('https://api.llama.fi/overview/fees/',chain,'?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=false&dataType=dailyRevenue'),{},{}
|
||||
) AS read,
|
||||
SYSDATE() AS _inserted_timestamp
|
||||
FROM (
|
||||
SELECT
|
||||
DISTINCT chain,
|
||||
row_num
|
||||
FROM all_chains_rev_base
|
||||
WHERE row_num BETWEEN {{ item * 15 + 1 }} AND {{ (item + 1) * 15 }}
|
||||
)
|
||||
{% if is_incremental() %}
|
||||
WHERE chain NOT IN (
|
||||
lat_flat AS (
|
||||
SELECT
|
||||
chain
|
||||
FROM (
|
||||
SELECT
|
||||
DISTINCT chain,
|
||||
MAX(timestamp::DATE) AS max_timestamp
|
||||
FROM {{ this }}
|
||||
GROUP BY 1
|
||||
HAVING CURRENT_DATE = max_timestamp
|
||||
)
|
||||
)
|
||||
{% endif %}
|
||||
) {% if not loop.last %}
|
||||
UNION ALL
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
r.value AS VALUE,
|
||||
r.value :displayName :: STRING AS protocol,
|
||||
_inserted_timestamp
|
||||
FROM
|
||||
api_pull,
|
||||
LATERAL FLATTEN (
|
||||
input => response
|
||||
) AS r
|
||||
),
|
||||
|
||||
reads_output AS (
|
||||
|
||||
SELECT
|
||||
chain,
|
||||
TO_TIMESTAMP(VALUE[0]::INTEGER) AS timestamp,
|
||||
VALUE[1] AS rev_object,
|
||||
_inserted_timestamp
|
||||
FROM rev_base,
|
||||
LATERAL FLATTEN (input=> read:data:totalDataChartBreakdown)
|
||||
chain_breakdown AS (
|
||||
SELECT
|
||||
k.key AS chain,
|
||||
SYSDATE() :: DATE AS TIMESTAMP,
|
||||
protocol,
|
||||
k.value AS rev_object,
|
||||
v.value :: INTEGER AS daily_rev,
|
||||
_inserted_timestamp
|
||||
FROM
|
||||
lat_flat,
|
||||
LATERAL FLATTEN(
|
||||
input => VALUE :breakdown24h
|
||||
) k,
|
||||
LATERAL FLATTEN(
|
||||
input => k.value
|
||||
) v
|
||||
)
|
||||
|
||||
SELECT
|
||||
chain,
|
||||
timestamp,
|
||||
key::STRING AS protocol,
|
||||
value::INTEGER AS daily_rev,
|
||||
TIMESTAMP,
|
||||
protocol,
|
||||
daily_rev,
|
||||
rev_object,
|
||||
_inserted_timestamp,
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['chain', 'protocol', 'timestamp']
|
||||
) }} AS id
|
||||
FROM reads_output,
|
||||
LATERAL FLATTEN(input => PARSE_JSON(reads_output.rev_object))
|
||||
FROM
|
||||
chain_breakdown
|
||||
|
||||
@ -16,6 +16,9 @@ models:
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 2
|
||||
- name: ID
|
||||
tests:
|
||||
- not_null
|
||||
@ -1,5 +1,6 @@
|
||||
{{ config(
|
||||
materialized = 'table',
|
||||
enabled = false,
|
||||
unique_key = 'defillama_stablecoin_supply_id',
|
||||
tags = ['defillama']
|
||||
) }}
|
||||
|
||||
@ -13,6 +13,9 @@ models:
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 2
|
||||
- name: DEFILLAMA_STABLECOIN_SUPPLY_ID
|
||||
tests:
|
||||
- not_null
|
||||
@ -2,10 +2,6 @@ version: 2
|
||||
models:
|
||||
- name: bronze__polymarket_markets
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- POLYMARKET_MARKET_ID
|
||||
|
||||
columns:
|
||||
- name: CONDITION_ID
|
||||
tests:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user