fix (#136)

2026-02-06 13:17:03 +00:00 · 2025-09-08 14:59:33 -04:00 · 2025-09-08 14:59:33 -04:00 · e0058cb6da
commit e0058cb6da
parent c92c2eda19
13 changed files with 170 additions and 103 deletions
--- a/models/defillama/bronze/bronze__defillama_protocols.sql
+++ b/models/defillama/bronze/bronze__defillama_protocols.sql
@ -1,52 +0,0 @@
-{{ config(
-    materialized = 'incremental',
-    unique_key = 'protocol_id',
-    tags = ['defillama']
-) }}
-
-WITH protocol_base AS (
-
-SELECT
-    live.udf_api(
-        'GET','https://pro-api.llama.fi/{api_key}/api/protocols',{},{},'Vault/prod/external/defillama'
-    ) AS read,
-    SYSDATE() AS _inserted_timestamp
-)
-
-SELECT
-    VALUE:id::STRING AS protocol_id,
-    VALUE:slug::STRING AS protocol_slug,
-    REGEXP_REPLACE(VALUE:parentProtocol::STRING, '^parent#', '') AS parent_protocol,
-    VALUE:name::STRING AS protocol,
-    CASE 
-        WHEN VALUE:address::STRING = '-' THEN NULL 
-        ELSE SUBSTRING(LOWER(VALUE:address::STRING), CHARINDEX(':', LOWER(VALUE:address::STRING))+1) 
-    END AS address,
-    CASE 
-        WHEN VALUE:symbol::STRING = '-' THEN NULL 
-        ELSE VALUE:symbol::STRING 
-    END AS symbol,
-    VALUE:description::STRING AS description,
-    VALUE:chain::STRING AS chain,
-    VALUE:audits::INTEGER AS num_audits,
-    VALUE:audit_note::STRING AS audit_note,
-    VALUE:category::STRING AS category,
-    VALUE:chains AS chains,
-    VALUE:url::STRING AS url,
-    VALUE:logo::STRING AS logo,
-    ROW_NUMBER() over (
-        ORDER BY 
-            protocol_id::int
-    ) AS row_num,
-    _inserted_timestamp
-FROM protocol_base,
-    LATERAL FLATTEN (input=> read:data)
-
-{% if is_incremental() %}
-WHERE protocol_id NOT IN (
-    SELECT
-        DISTINCT protocol_id
-    FROM
-        {{ this }}
-)
-{% endif %}
--- a/models/defillama/bronze/bronze__defillama_protocols.yml
+++ b/models/defillama/bronze/bronze__defillama_protocols.yml
@ -1,30 +0,0 @@
-version: 2
-models:
-  - name: bronze__defillama_protocols
-    tests:
-      - dbt_utils.unique_combination_of_columns:
-            combination_of_columns:
-              - PROTOCOL_ID
-    
-    columns:
-      - name: PROTOCOL_ID
-        tests: 
-          - not_null
-      - name: PROTOCOL_SLUG
-        tests: 
-          - not_null
-      - name: PROTOCOL
-      - name: ADDRESS
-      - name: SYMBOL
-      - name: DESCRIPTION
-      - name: CHAIN
-      - name: NUM_AUDITS
-      - name: AUDIT_NOTE
-      - name: CATEGORY
-      - name: CHAINS
-      - name: _INSERTED_TIMESTAMP
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - TIMESTAMP_NTZ
--- a/models/defillama/gold/defillama__dim_protocols.sql
+++ b/models/defillama/gold/defillama__dim_protocols.sql
@ -1,15 +1,7 @@
 {{ config(
-    materialized = 'view',
-    persist_docs ={ "relation": true,
-    "columns": true },
-    tags = ['defillama'],
-    meta={
-        'database_tags':{
-            'table': {
-                'PROTOCOL': 'DEFILLAMA'
-            }
-        }
-    }
+    materialized = 'incremental',
+    unique_key = 'protocol_id',
+    tags = ['defillama']
 ) }}

 SELECT
@ -23,5 +15,19 @@ SELECT
    chains,
    category,
    num_audits,
-    audit_note
-FROM {{ ref('bronze__defillama_protocols') }}
+    audit_note,
+    tvl,
+    chain_tvls,
+    sysdate() as inserted_timestamp,
+    sysdate() as modified_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['protocol_id']
+    ) }} AS dim_protocols_id
+FROM {{ ref('silver__defillama_protocols') }}
+WHERE 1=1
+{% if is_incremental() %}
+AND modified_timestamp > (
+    SELECT MAX(modified_timestamp) FROM {{ this }}
+)
+{% endif %}
+QUALIFY ROW_NUMBER() OVER (PARTITION BY dim_protocols_id ORDER BY modified_timestamp DESC) = 1
--- a/models/defillama/gold/defillama__dim_protocols.yml
+++ b/models/defillama/gold/defillama__dim_protocols.yml
@ -25,4 +25,26 @@ models:
      - name: NUM_AUDITS
        description: Number of audits the protocol has undergone.
      - name: AUDIT_NOTE
-        description: Notes on the audits of the protocol.
+        description: Notes on the audits of the protocol.
+      - name: TVL
+        description: Total value locked of the protocol as of the last run.
+      - name: CHAIN_TVLS
+        description: Total value locked of the protocol on each chain as of the last run.
+      - name: MODIFIED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 2
+      - name: INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 2
--- a/models/defillama/silver/silver__defillama_protocol_tvl_history.sql
+++ b/models/defillama/silver/silver__defillama_protocol_tvl_history.sql
@ -97,7 +97,7 @@ SELECT
    '{{ invocation_id }}' AS _invocation_id
 FROM
    daily_tvl_with_lags d
-    LEFT JOIN {{ ref('bronze__defillama_protocols') }} p
+    LEFT JOIN {{ ref('defillama__dim_protocols') }} p
      ON p.protocol_id = d.protocol_id
 QUALIFY ROW_NUMBER() OVER (
    PARTITION BY d.timestamp, d.protocol_id, d.chain
--- a/models/defillama/silver/silver__defillama_protocols.sql
+++ b/models/defillama/silver/silver__defillama_protocols.sql
@ -0,0 +1,54 @@
+-- depends_on: {{ ref('bronze__defillama_protocols') }}
+{{ config(
+    materialized = 'incremental',
+    unique_key = 'defillama_protocols_id',
+    cluster_by = ['run_date'],
+    tags = ['defillama']
+) }}
+
+SELECT
+    TO_TIMESTAMP(VALUE:"RUN_TIMESTAMP"::INT) AS run_timestamp,
+    TO_TIMESTAMP(VALUE:"RUN_TIMESTAMP"::INT)::DATE AS run_date,
+    DATA:id::STRING AS protocol_id,
+    DATA:slug::STRING AS protocol_slug,
+    REGEXP_REPLACE(DATA:parentProtocol::STRING, '^parent#', '') AS parent_protocol,
+    DATA:name::STRING AS protocol,
+    CASE 
+        WHEN DATA:address::STRING = '-' THEN NULL 
+        ELSE SUBSTRING(LOWER(DATA:address::STRING), CHARINDEX(':', LOWER(DATA:address::STRING))+1) 
+    END AS address,
+    CASE 
+        WHEN DATA:symbol::STRING = '-' THEN NULL 
+        ELSE DATA:symbol::STRING 
+    END AS symbol,
+    DATA:description::STRING AS description,
+    DATA:chain::STRING AS chain,
+    DATA:audits::INTEGER AS num_audits,
+    DATA:audit_note::STRING AS audit_note,
+    DATA:category::STRING AS category,
+    DATA:url::STRING AS url,
+    DATA:logo::STRING AS logo,
+    DATA:tvl::FLOAT AS tvl,
+    DATA:chains AS chains,
+    DATA:chainTvls AS chain_tvls,
+    DATA,
+    _inserted_timestamp,
+    sysdate() as inserted_timestamp,
+    sysdate() as modified_timestamp,
+    '{{ invocation_id }}' as _invocation_id,
+    {{ dbt_utils.generate_surrogate_key(
+        ['protocol_id','run_date']
+    ) }} AS defillama_protocols_id
+    from 
+    {% if is_incremental() %}
+    {{ ref('bronze__defillama_protocols') }}
+    where _inserted_timestamp > (
+        select coalesce(max(_inserted_timestamp), '2025-01-01') from {{ this }}
+    )
+    {% else %}
+    {{ ref('bronze__defillama_protocols_FR') }}
+    {% endif %}
+
+QUALIFY(
+    ROW_NUMBER() OVER (PARTITION BY defillama_protocols_id ORDER BY _inserted_timestamp DESC)
+) = 1
--- a/models/defillama/silver/silver__defillama_protocols.yml
+++ b/models/defillama/silver/silver__defillama_protocols.yml
@ -0,0 +1,18 @@
+version: 2
+models:
+  - name: silver__defillama_protocols
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+            combination_of_columns:
+              - DEFILLAMA_PROTOCOLS_ID
+    
+    columns:
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 2
--- a/models/defillama/streamline/bronze/bronze__defillama_protocols.sql
+++ b/models/defillama/streamline/bronze/bronze__defillama_protocols.sql
@ -0,0 +1,9 @@
+{{ config (
+    materialized = 'view',
+    tags = ['defillama_streamline']
+) }}
+{{ streamline_external_table_query_v2(
+    model = 'defillama_protocols',
+    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)",
+    partition_name = "partition_key"
+) }}
--- a/models/defillama/streamline/bronze/bronze__defillama_protocols_FR.sql
+++ b/models/defillama/streamline/bronze/bronze__defillama_protocols_FR.sql
@ -0,0 +1,9 @@
+{{ config (
+    materialized = 'view',
+    tags = ['defillama_streamline']
+) }}
+{{ streamline_external_table_FR_query_v2(
+    model = 'defillama_protocols',
+    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)",
+    partition_name = "partition_key"
+) }}
--- a/models/defillama/streamline/realtime/streamline__defillama_protocols.sql
+++ b/models/defillama/streamline/realtime/streamline__defillama_protocols.sql
@ -0,0 +1,31 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'streamline.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params ={ "external_table" :"defillama_protocols",
+        "sql_limit" :"10000",
+        "producer_batch_size" :"10",
+        "worker_batch_size" :"1",
+        "async_concurrent_requests" :"1",
+        "sql_source" :"{{this.identifier}}",
+        "exploded_key": tojson(['data'])
+        }
+    ),
+    tags = ['defillama_streamline']
+) }}
+
+SELECT
+    date_part('epoch_second', sysdate()) as run_timestamp,
+    date_part('epoch_second', sysdate()::DATE) AS partition_key,
+    {{ target.database }}.live.udf_api(
+        'GET',
+        'https://pro-api.llama.fi/{api_key}/api/protocols',
+        OBJECT_CONSTRUCT(
+            'Content-Type', 'text/plain',
+            'Accept', 'text/plain',
+            'fsc-quantum-state', 'streamline'
+        ),
+        {},
+        'Vault/prod/external/defillama'
+    ) AS request
--- a/models/defillama/streamline/realtime/streamline__defillama_stablecoin_metrics.sql
+++ b/models/defillama/streamline/realtime/streamline__defillama_stablecoin_metrics.sql
--- a/models/defillama/streamline/streamline__defillama_protocol_historical.sql
+++ b/models/defillama/streamline/streamline__defillama_protocol_historical.sql
@ -17,10 +17,9 @@ WITH base AS (

    SELECT
        protocol_slug,
-        protocol_id,
-        row_num
+        protocol_id
    FROM
-        {{ ref('bronze__defillama_protocols') }}
+        {{ ref('defillama__dim_protocols') }}
    WHERE
        protocol_id NOT IN (
            SELECT
@ -41,12 +40,12 @@ WITH base AS (
                status_code = 200
        )
    ORDER BY
-        row_num ASC
+        protocol_id ASC
    LIMIT 200
 )
 SELECT
    protocol_id,
-    FLOOR(protocol_id / 10) * 10 AS partition_key,
+    date_part('epoch_second', sysdate()::DATE) AS partition_key,
    {{ target.database }}.live.udf_api(
        'GET',
        'https://pro-api.llama.fi/{api_key}/api/protocol/' || protocol_slug,
@ -60,4 +59,4 @@ SELECT
 FROM
    base
 ORDER BY
-    row_num ASC
+    protocol_id ASC
--- a/models/sources.yml
+++ b/models/sources.yml
@ -11,6 +11,7 @@ sources:
      - name: defillama_protocol_historical
      - name: valuations_parquet
      - name: defillama_stablecoin_metrics
+      - name: defillama_protocols
  - name: tokenflow_eth
    database: flipside_prod_db
    schema: tokenflow_eth