diff --git a/models/defillama/bronze/bronze__defillama_stablecoin_supply.sql b/models/defillama/bronze/bronze__defillama_stablecoin_supply.sql index 2401e4f..b4ee6ef 100644 --- a/models/defillama/bronze/bronze__defillama_stablecoin_supply.sql +++ b/models/defillama/bronze/bronze__defillama_stablecoin_supply.sql @@ -1,5 +1,6 @@ {{ config( materialized = 'table', + enabled = false, unique_key = ['stablecoin_id','timestamp'], tags = ['defillama'] ) }} diff --git a/models/defillama/gold/defillama__fact_options_volume.sql b/models/defillama/gold/defillama__fact_options_volume.sql index ad367c4..aa86604 100644 --- a/models/defillama/gold/defillama__fact_options_volume.sql +++ b/models/defillama/gold/defillama__fact_options_volume.sql @@ -3,6 +3,7 @@ persist_docs ={ "relation": true, "columns": true }, tags = ['defillama'], + enabled = false, meta={ 'database_tags':{ 'table': { diff --git a/models/defillama/gold/defillama__fact_stablecoin_supply.sql b/models/defillama/gold/defillama__fact_stablecoin_supply.sql index 60e3788..0f6a613 100644 --- a/models/defillama/gold/defillama__fact_stablecoin_supply.sql +++ b/models/defillama/gold/defillama__fact_stablecoin_supply.sql @@ -1,5 +1,6 @@ {{ config( materialized = 'view', + enabled = false, persist_docs ={ "relation": true, "columns": true }, tags = ['defillama'], diff --git a/models/defillama/silver/silver__defillama_bridge_volume.yml b/models/defillama/silver/silver__defillama_bridge_volume.yml index 3cb7342..15d5bad 100644 --- a/models/defillama/silver/silver__defillama_bridge_volume.yml +++ b/models/defillama/silver/silver__defillama_bridge_volume.yml @@ -20,6 +20,9 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 - name: ID tests: - not_null \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_chains_tvl.yml b/models/defillama/silver/silver__defillama_chains_tvl.yml index 07508a8..ba015fc 100644 --- a/models/defillama/silver/silver__defillama_chains_tvl.yml +++ b/models/defillama/silver/silver__defillama_chains_tvl.yml @@ -14,6 +14,9 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 - name: ID tests: - not_null \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_dex_volume.sql b/models/defillama/silver/silver__defillama_dex_volume.sql index 358a05f..6c6c800 100644 --- a/models/defillama/silver/silver__defillama_dex_volume.sql +++ b/models/defillama/silver/silver__defillama_dex_volume.sql @@ -5,79 +5,53 @@ tags = ['defillama'] ) }} -WITH all_chains_dex_base AS ( +WITH api_pull AS ( -SELECT - LOWER(VALUE::STRING) AS chain, - ROW_NUMBER() OVER (ORDER BY chain) AS row_num, - _inserted_timestamp -FROM ( SELECT - live.udf_api( - 'GET','https://api.llama.fi/overview/dexs?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=dailyVolume',{},{} - ) AS read, + PARSE_JSON( + live.udf_api( + 'GET', + 'https://api.llama.fi/overview/dexs?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=dailyVolume',{},{} + ) + ) :data :protocols AS response, SYSDATE() AS _inserted_timestamp - ), -LATERAL FLATTEN (input=> read:data:allChains) ), - -dex_base AS ( - -{% for item in range(10) %} -( -SELECT - chain, - live.udf_api( - 'GET',CONCAT('https://api.llama.fi/overview/dexs/',chain,'?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=false&dataType=dailyVolume'),{},{} - ) AS read, - SYSDATE() AS _inserted_timestamp -FROM ( - SELECT - DISTINCT chain, - row_num - FROM all_chains_dex_base - WHERE row_num BETWEEN {{ item * 20 + 1 }} AND {{ (item + 1) * 20 }} - ) -{% if is_incremental() %} -WHERE chain NOT IN ( +lat_flat AS ( SELECT - chain - FROM ( - SELECT - DISTINCT chain, - MAX(timestamp::DATE) AS max_timestamp - FROM {{ this }} - GROUP BY 1 - HAVING CURRENT_DATE = max_timestamp - ) -) -{% endif %} -) {% if not loop.last %} -UNION ALL -{% endif %} -{% endfor %} + r.value AS VALUE, + _inserted_timestamp + FROM + api_pull, + LATERAL FLATTEN ( + input => response + ) AS r ), - -reads_output AS ( - -SELECT - chain, - TO_TIMESTAMP(VALUE[0]::INTEGER) AS timestamp, - VALUE[1] AS dex_object, - _inserted_timestamp -FROM dex_base, - LATERAL FLATTEN (input=> read:data:totalDataChartBreakdown) +chain_breakdown AS ( + SELECT + k.key AS chain, + SYSDATE() :: DATE AS TIMESTAMP, + v.key AS protocol, + k.value AS dex_object, + v.value :: INTEGER AS daily_volume, + _inserted_timestamp + FROM + lat_flat, + LATERAL FLATTEN( + input => VALUE :breakdown24h + ) k, + LATERAL FLATTEN( + input => k.value + ) v ) - SELECT chain, - timestamp, - LOWER(key::STRING) AS protocol, - value::INTEGER AS daily_volume, + TIMESTAMP, + LOWER(protocol) AS protocol, + daily_volume, dex_object, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['chain', 'protocol', 'timestamp'] ) }} AS id -FROM reads_output, - LATERAL FLATTEN(input => PARSE_JSON(reads_output.dex_object)) +FROM + chain_breakdown \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_dex_volume.yml b/models/defillama/silver/silver__defillama_dex_volume.yml index 266796b..f048934 100644 --- a/models/defillama/silver/silver__defillama_dex_volume.yml +++ b/models/defillama/silver/silver__defillama_dex_volume.yml @@ -16,6 +16,9 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 - name: ID tests: - not_null \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_options_notional.sql b/models/defillama/silver/silver__defillama_options_notional.sql index cd47014..3e72dc1 100644 --- a/models/defillama/silver/silver__defillama_options_notional.sql +++ b/models/defillama/silver/silver__defillama_options_notional.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = 'id', + enabled = false, full_refresh = false, tags = ['defillama'] ) }} diff --git a/models/defillama/silver/silver__defillama_options_notional.yml b/models/defillama/silver/silver__defillama_options_notional.yml index 5cf70f7..32add79 100644 --- a/models/defillama/silver/silver__defillama_options_notional.yml +++ b/models/defillama/silver/silver__defillama_options_notional.yml @@ -16,6 +16,9 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 - name: ID tests: - not_null \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_options_premium.sql b/models/defillama/silver/silver__defillama_options_premium.sql index 01d2220..3e24967 100644 --- a/models/defillama/silver/silver__defillama_options_premium.sql +++ b/models/defillama/silver/silver__defillama_options_premium.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = 'id', + enabled = false, full_refresh = false, tags = ['defillama'] ) }} diff --git a/models/defillama/silver/silver__defillama_options_premium.yml b/models/defillama/silver/silver__defillama_options_premium.yml index ef1ef83..2e0bd26 100644 --- a/models/defillama/silver/silver__defillama_options_premium.yml +++ b/models/defillama/silver/silver__defillama_options_premium.yml @@ -16,6 +16,9 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 - name: ID tests: - not_null \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_pool_yields.yml b/models/defillama/silver/silver__defillama_pool_yields.yml index bc42b02..bd0b411 100644 --- a/models/defillama/silver/silver__defillama_pool_yields.yml +++ b/models/defillama/silver/silver__defillama_pool_yields.yml @@ -9,6 +9,12 @@ models: - name: _INSERTED_TIMESTAMP tests: - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 - name: TIMESTAMP tests: - not_null \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_protocol_fees.yml b/models/defillama/silver/silver__defillama_protocol_fees.yml index b58ab3e..d7efcde 100644 --- a/models/defillama/silver/silver__defillama_protocol_fees.yml +++ b/models/defillama/silver/silver__defillama_protocol_fees.yml @@ -16,6 +16,9 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 - name: ID tests: - not_null \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_protocol_revenue.sql b/models/defillama/silver/silver__defillama_protocol_revenue.sql index ba42bf0..0be4125 100644 --- a/models/defillama/silver/silver__defillama_protocol_revenue.sql +++ b/models/defillama/silver/silver__defillama_protocol_revenue.sql @@ -5,79 +5,54 @@ tags = ['defillama'] ) }} -WITH all_chains_rev_base AS ( +WITH api_pull AS ( -SELECT - LOWER(VALUE::STRING) AS chain, - ROW_NUMBER() OVER (ORDER BY chain) AS row_num, - _inserted_timestamp -FROM ( SELECT - live.udf_api( - 'GET','https://api.llama.fi/overview/fees?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=dailyRevenue',{},{} - ) AS read, + PARSE_JSON( + live.udf_api( + 'GET', + 'https://api.llama.fi/overview/fees?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true&dataType=dailyRevenue',{},{} + ) + ) :data :protocols AS response, SYSDATE() AS _inserted_timestamp - ), -LATERAL FLATTEN (input=> read:data:allChains) ), - -rev_base AS ( - -{% for item in range(7) %} -( -SELECT - chain, - live.udf_api( - 'GET',CONCAT('https://api.llama.fi/overview/fees/',chain,'?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=false&dataType=dailyRevenue'),{},{} - ) AS read, - SYSDATE() AS _inserted_timestamp -FROM ( - SELECT - DISTINCT chain, - row_num - FROM all_chains_rev_base - WHERE row_num BETWEEN {{ item * 15 + 1 }} AND {{ (item + 1) * 15 }} - ) -{% if is_incremental() %} -WHERE chain NOT IN ( +lat_flat AS ( SELECT - chain - FROM ( - SELECT - DISTINCT chain, - MAX(timestamp::DATE) AS max_timestamp - FROM {{ this }} - GROUP BY 1 - HAVING CURRENT_DATE = max_timestamp - ) -) -{% endif %} -) {% if not loop.last %} -UNION ALL -{% endif %} -{% endfor %} + r.value AS VALUE, + r.value :displayName :: STRING AS protocol, + _inserted_timestamp + FROM + api_pull, + LATERAL FLATTEN ( + input => response + ) AS r ), - -reads_output AS ( - -SELECT - chain, - TO_TIMESTAMP(VALUE[0]::INTEGER) AS timestamp, - VALUE[1] AS rev_object, - _inserted_timestamp -FROM rev_base, - LATERAL FLATTEN (input=> read:data:totalDataChartBreakdown) +chain_breakdown AS ( + SELECT + k.key AS chain, + SYSDATE() :: DATE AS TIMESTAMP, + protocol, + k.value AS rev_object, + v.value :: INTEGER AS daily_rev, + _inserted_timestamp + FROM + lat_flat, + LATERAL FLATTEN( + input => VALUE :breakdown24h + ) k, + LATERAL FLATTEN( + input => k.value + ) v ) - SELECT chain, - timestamp, - key::STRING AS protocol, - value::INTEGER AS daily_rev, + TIMESTAMP, + protocol, + daily_rev, rev_object, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['chain', 'protocol', 'timestamp'] ) }} AS id -FROM reads_output, - LATERAL FLATTEN(input => PARSE_JSON(reads_output.rev_object)) \ No newline at end of file +FROM + chain_breakdown diff --git a/models/defillama/silver/silver__defillama_protocol_revenue.yml b/models/defillama/silver/silver__defillama_protocol_revenue.yml index 5d65c27..87a6793 100644 --- a/models/defillama/silver/silver__defillama_protocol_revenue.yml +++ b/models/defillama/silver/silver__defillama_protocol_revenue.yml @@ -16,6 +16,9 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 - name: ID tests: - not_null \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_stablecoin_supply.sql b/models/defillama/silver/silver__defillama_stablecoin_supply.sql index 68ff827..3df77ef 100644 --- a/models/defillama/silver/silver__defillama_stablecoin_supply.sql +++ b/models/defillama/silver/silver__defillama_stablecoin_supply.sql @@ -1,5 +1,6 @@ {{ config( materialized = 'table', + enabled = false, unique_key = 'defillama_stablecoin_supply_id', tags = ['defillama'] ) }} diff --git a/models/defillama/silver/silver__defillama_stablecoin_supply.yml b/models/defillama/silver/silver__defillama_stablecoin_supply.yml index 4d90259..4cad7c1 100644 --- a/models/defillama/silver/silver__defillama_stablecoin_supply.yml +++ b/models/defillama/silver/silver__defillama_stablecoin_supply.yml @@ -13,6 +13,9 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 - name: DEFILLAMA_STABLECOIN_SUPPLY_ID tests: - not_null \ No newline at end of file diff --git a/models/polymarket/bronze/bronze__polymarket_markets.yml b/models/polymarket/bronze/bronze__polymarket_markets.yml index 723895f..06879a9 100644 --- a/models/polymarket/bronze/bronze__polymarket_markets.yml +++ b/models/polymarket/bronze/bronze__polymarket_markets.yml @@ -2,10 +2,6 @@ version: 2 models: - name: bronze__polymarket_markets tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - POLYMARKET_MARKET_ID - columns: - name: CONDITION_ID tests: