From e0058cb6dad685b64e587fabf28e17a7110dcddf Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Mon, 8 Sep 2025 14:59:33 -0400 Subject: [PATCH] fix (#136) --- .../bronze/bronze__defillama_protocols.sql | 52 ------------------ .../bronze/bronze__defillama_protocols.yml | 30 ----------- .../gold/defillama__dim_protocols.sql | 32 ++++++----- .../gold/defillama__dim_protocols.yml | 24 ++++++++- ...silver__defillama_protocol_tvl_history.sql | 2 +- .../silver/silver__defillama_protocols.sql | 54 +++++++++++++++++++ .../silver/silver__defillama_protocols.yml | 18 +++++++ .../bronze/bronze__defillama_protocols.sql | 9 ++++ .../bronze/bronze__defillama_protocols_FR.sql | 9 ++++ .../streamline__defillama_protocols.sql | 31 +++++++++++ ...reamline__defillama_stablecoin_metrics.sql | 0 ...eamline__defillama_protocol_historical.sql | 11 ++-- models/sources.yml | 1 + 13 files changed, 170 insertions(+), 103 deletions(-) delete mode 100644 models/defillama/bronze/bronze__defillama_protocols.sql delete mode 100644 models/defillama/bronze/bronze__defillama_protocols.yml create mode 100644 models/defillama/silver/silver__defillama_protocols.sql create mode 100644 models/defillama/silver/silver__defillama_protocols.yml create mode 100644 models/defillama/streamline/bronze/bronze__defillama_protocols.sql create mode 100644 models/defillama/streamline/bronze/bronze__defillama_protocols_FR.sql create mode 100644 models/defillama/streamline/realtime/streamline__defillama_protocols.sql rename models/defillama/streamline/{ => realtime}/streamline__defillama_stablecoin_metrics.sql (100%) diff --git a/models/defillama/bronze/bronze__defillama_protocols.sql b/models/defillama/bronze/bronze__defillama_protocols.sql deleted file mode 100644 index 0da6bac..0000000 --- a/models/defillama/bronze/bronze__defillama_protocols.sql +++ /dev/null @@ -1,52 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = 'protocol_id', - tags = ['defillama'] -) }} - -WITH protocol_base AS ( - -SELECT - live.udf_api( - 'GET','https://pro-api.llama.fi/{api_key}/api/protocols',{},{},'Vault/prod/external/defillama' - ) AS read, - SYSDATE() AS _inserted_timestamp -) - -SELECT - VALUE:id::STRING AS protocol_id, - VALUE:slug::STRING AS protocol_slug, - REGEXP_REPLACE(VALUE:parentProtocol::STRING, '^parent#', '') AS parent_protocol, - VALUE:name::STRING AS protocol, - CASE - WHEN VALUE:address::STRING = '-' THEN NULL - ELSE SUBSTRING(LOWER(VALUE:address::STRING), CHARINDEX(':', LOWER(VALUE:address::STRING))+1) - END AS address, - CASE - WHEN VALUE:symbol::STRING = '-' THEN NULL - ELSE VALUE:symbol::STRING - END AS symbol, - VALUE:description::STRING AS description, - VALUE:chain::STRING AS chain, - VALUE:audits::INTEGER AS num_audits, - VALUE:audit_note::STRING AS audit_note, - VALUE:category::STRING AS category, - VALUE:chains AS chains, - VALUE:url::STRING AS url, - VALUE:logo::STRING AS logo, - ROW_NUMBER() over ( - ORDER BY - protocol_id::int - ) AS row_num, - _inserted_timestamp -FROM protocol_base, - LATERAL FLATTEN (input=> read:data) - -{% if is_incremental() %} -WHERE protocol_id NOT IN ( - SELECT - DISTINCT protocol_id - FROM - {{ this }} -) -{% endif %} \ No newline at end of file diff --git a/models/defillama/bronze/bronze__defillama_protocols.yml b/models/defillama/bronze/bronze__defillama_protocols.yml deleted file mode 100644 index a75b941..0000000 --- a/models/defillama/bronze/bronze__defillama_protocols.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: 2 -models: - - name: bronze__defillama_protocols - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - PROTOCOL_ID - - columns: - - name: PROTOCOL_ID - tests: - - not_null - - name: PROTOCOL_SLUG - tests: - - not_null - - name: PROTOCOL - - name: ADDRESS - - name: SYMBOL - - name: DESCRIPTION - - name: CHAIN - - name: NUM_AUDITS - - name: AUDIT_NOTE - - name: CATEGORY - - name: CHAINS - - name: _INSERTED_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/defillama/gold/defillama__dim_protocols.sql b/models/defillama/gold/defillama__dim_protocols.sql index af0ad65..1f65531 100644 --- a/models/defillama/gold/defillama__dim_protocols.sql +++ b/models/defillama/gold/defillama__dim_protocols.sql @@ -1,15 +1,7 @@ {{ config( - materialized = 'view', - persist_docs ={ "relation": true, - "columns": true }, - tags = ['defillama'], - meta={ - 'database_tags':{ - 'table': { - 'PROTOCOL': 'DEFILLAMA' - } - } - } + materialized = 'incremental', + unique_key = 'protocol_id', + tags = ['defillama'] ) }} SELECT @@ -23,5 +15,19 @@ SELECT chains, category, num_audits, - audit_note -FROM {{ ref('bronze__defillama_protocols') }} \ No newline at end of file + audit_note, + tvl, + chain_tvls, + sysdate() as inserted_timestamp, + sysdate() as modified_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['protocol_id'] + ) }} AS dim_protocols_id +FROM {{ ref('silver__defillama_protocols') }} +WHERE 1=1 +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT MAX(modified_timestamp) FROM {{ this }} +) +{% endif %} +QUALIFY ROW_NUMBER() OVER (PARTITION BY dim_protocols_id ORDER BY modified_timestamp DESC) = 1 \ No newline at end of file diff --git a/models/defillama/gold/defillama__dim_protocols.yml b/models/defillama/gold/defillama__dim_protocols.yml index f51f763..bc9560c 100644 --- a/models/defillama/gold/defillama__dim_protocols.yml +++ b/models/defillama/gold/defillama__dim_protocols.yml @@ -25,4 +25,26 @@ models: - name: NUM_AUDITS description: Number of audits the protocol has undergone. - name: AUDIT_NOTE - description: Notes on the audits of the protocol. \ No newline at end of file + description: Notes on the audits of the protocol. + - name: TVL + description: Total value locked of the protocol as of the last run. + - name: CHAIN_TVLS + description: Total value locked of the protocol on each chain as of the last run. + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_protocol_tvl_history.sql b/models/defillama/silver/silver__defillama_protocol_tvl_history.sql index bc03bae..8da9ef5 100644 --- a/models/defillama/silver/silver__defillama_protocol_tvl_history.sql +++ b/models/defillama/silver/silver__defillama_protocol_tvl_history.sql @@ -97,7 +97,7 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM daily_tvl_with_lags d - LEFT JOIN {{ ref('bronze__defillama_protocols') }} p + LEFT JOIN {{ ref('defillama__dim_protocols') }} p ON p.protocol_id = d.protocol_id QUALIFY ROW_NUMBER() OVER ( PARTITION BY d.timestamp, d.protocol_id, d.chain diff --git a/models/defillama/silver/silver__defillama_protocols.sql b/models/defillama/silver/silver__defillama_protocols.sql new file mode 100644 index 0000000..a530d18 --- /dev/null +++ b/models/defillama/silver/silver__defillama_protocols.sql @@ -0,0 +1,54 @@ +-- depends_on: {{ ref('bronze__defillama_protocols') }} +{{ config( + materialized = 'incremental', + unique_key = 'defillama_protocols_id', + cluster_by = ['run_date'], + tags = ['defillama'] +) }} + +SELECT + TO_TIMESTAMP(VALUE:"RUN_TIMESTAMP"::INT) AS run_timestamp, + TO_TIMESTAMP(VALUE:"RUN_TIMESTAMP"::INT)::DATE AS run_date, + DATA:id::STRING AS protocol_id, + DATA:slug::STRING AS protocol_slug, + REGEXP_REPLACE(DATA:parentProtocol::STRING, '^parent#', '') AS parent_protocol, + DATA:name::STRING AS protocol, + CASE + WHEN DATA:address::STRING = '-' THEN NULL + ELSE SUBSTRING(LOWER(DATA:address::STRING), CHARINDEX(':', LOWER(DATA:address::STRING))+1) + END AS address, + CASE + WHEN DATA:symbol::STRING = '-' THEN NULL + ELSE DATA:symbol::STRING + END AS symbol, + DATA:description::STRING AS description, + DATA:chain::STRING AS chain, + DATA:audits::INTEGER AS num_audits, + DATA:audit_note::STRING AS audit_note, + DATA:category::STRING AS category, + DATA:url::STRING AS url, + DATA:logo::STRING AS logo, + DATA:tvl::FLOAT AS tvl, + DATA:chains AS chains, + DATA:chainTvls AS chain_tvls, + DATA, + _inserted_timestamp, + sysdate() as inserted_timestamp, + sysdate() as modified_timestamp, + '{{ invocation_id }}' as _invocation_id, + {{ dbt_utils.generate_surrogate_key( + ['protocol_id','run_date'] + ) }} AS defillama_protocols_id + from + {% if is_incremental() %} + {{ ref('bronze__defillama_protocols') }} + where _inserted_timestamp > ( + select coalesce(max(_inserted_timestamp), '2025-01-01') from {{ this }} + ) + {% else %} + {{ ref('bronze__defillama_protocols_FR') }} + {% endif %} + +QUALIFY( + ROW_NUMBER() OVER (PARTITION BY defillama_protocols_id ORDER BY _inserted_timestamp DESC) +) = 1 \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_protocols.yml b/models/defillama/silver/silver__defillama_protocols.yml new file mode 100644 index 0000000..4d1f89b --- /dev/null +++ b/models/defillama/silver/silver__defillama_protocols.yml @@ -0,0 +1,18 @@ +version: 2 +models: + - name: silver__defillama_protocols + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DEFILLAMA_PROTOCOLS_ID + + columns: + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 \ No newline at end of file diff --git a/models/defillama/streamline/bronze/bronze__defillama_protocols.sql b/models/defillama/streamline/bronze/bronze__defillama_protocols.sql new file mode 100644 index 0000000..b9b6fc5 --- /dev/null +++ b/models/defillama/streamline/bronze/bronze__defillama_protocols.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['defillama_streamline'] +) }} +{{ streamline_external_table_query_v2( + model = 'defillama_protocols', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)", + partition_name = "partition_key" +) }} diff --git a/models/defillama/streamline/bronze/bronze__defillama_protocols_FR.sql b/models/defillama/streamline/bronze/bronze__defillama_protocols_FR.sql new file mode 100644 index 0000000..bd14c9f --- /dev/null +++ b/models/defillama/streamline/bronze/bronze__defillama_protocols_FR.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['defillama_streamline'] +) }} +{{ streamline_external_table_FR_query_v2( + model = 'defillama_protocols', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)", + partition_name = "partition_key" +) }} diff --git a/models/defillama/streamline/realtime/streamline__defillama_protocols.sql b/models/defillama/streamline/realtime/streamline__defillama_protocols.sql new file mode 100644 index 0000000..1d1144c --- /dev/null +++ b/models/defillama/streamline/realtime/streamline__defillama_protocols.sql @@ -0,0 +1,31 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"defillama_protocols", + "sql_limit" :"10000", + "producer_batch_size" :"10", + "worker_batch_size" :"1", + "async_concurrent_requests" :"1", + "sql_source" :"{{this.identifier}}", + "exploded_key": tojson(['data']) + } + ), + tags = ['defillama_streamline'] +) }} + +SELECT + date_part('epoch_second', sysdate()) as run_timestamp, + date_part('epoch_second', sysdate()::DATE) AS partition_key, + {{ target.database }}.live.udf_api( + 'GET', + 'https://pro-api.llama.fi/{api_key}/api/protocols', + OBJECT_CONSTRUCT( + 'Content-Type', 'text/plain', + 'Accept', 'text/plain', + 'fsc-quantum-state', 'streamline' + ), + {}, + 'Vault/prod/external/defillama' + ) AS request \ No newline at end of file diff --git a/models/defillama/streamline/streamline__defillama_stablecoin_metrics.sql b/models/defillama/streamline/realtime/streamline__defillama_stablecoin_metrics.sql similarity index 100% rename from models/defillama/streamline/streamline__defillama_stablecoin_metrics.sql rename to models/defillama/streamline/realtime/streamline__defillama_stablecoin_metrics.sql diff --git a/models/defillama/streamline/streamline__defillama_protocol_historical.sql b/models/defillama/streamline/streamline__defillama_protocol_historical.sql index 8f60fda..7e9c57b 100644 --- a/models/defillama/streamline/streamline__defillama_protocol_historical.sql +++ b/models/defillama/streamline/streamline__defillama_protocol_historical.sql @@ -17,10 +17,9 @@ WITH base AS ( SELECT protocol_slug, - protocol_id, - row_num + protocol_id FROM - {{ ref('bronze__defillama_protocols') }} + {{ ref('defillama__dim_protocols') }} WHERE protocol_id NOT IN ( SELECT @@ -41,12 +40,12 @@ WITH base AS ( status_code = 200 ) ORDER BY - row_num ASC + protocol_id ASC LIMIT 200 ) SELECT protocol_id, - FLOOR(protocol_id / 10) * 10 AS partition_key, + date_part('epoch_second', sysdate()::DATE) AS partition_key, {{ target.database }}.live.udf_api( 'GET', 'https://pro-api.llama.fi/{api_key}/api/protocol/' || protocol_slug, @@ -60,4 +59,4 @@ SELECT FROM base ORDER BY - row_num ASC + protocol_id ASC diff --git a/models/sources.yml b/models/sources.yml index 79428f8..cb2d7d3 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -11,6 +11,7 @@ sources: - name: defillama_protocol_historical - name: valuations_parquet - name: defillama_stablecoin_metrics + - name: defillama_protocols - name: tokenflow_eth database: flipside_prod_db schema: tokenflow_eth