DAT2-37/add-defillama-perps (#139)

models/defillama/bronze/bronze__defillama_perps.sql

@@ -0,0 +1,73 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = ['protocol_id'],
+    cluster_by = ['protocol_id'],
+    tags = ['defillama']
+) }}
+
+WITH api_pull AS (
+
+    SELECT
+        PARSE_JSON(
+            live.udf_api(
+                'GET',
+                'https://pro-api.llama.fi/{api_key}/api/overview/derivatives?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true',
+                {},
+                {},
+                'Vault/prod/external/defillama'
+            )
+        ) :data:protocols AS response,
+        SYSDATE() AS _inserted_timestamp
+),
+lat_flat AS (
+    SELECT
+        r.value AS VALUE,
+        _inserted_timestamp
+    FROM
+        api_pull,
+        LATERAL FLATTEN (response) AS r
+),
+protocol_expand AS (
+    SELECT
+        VALUE :defillamaId :: STRING AS protocol_id,
+        VALUE :category :: STRING AS category,
+        VALUE :name :: STRING AS NAME,
+        VALUE :displayName :: STRING AS display_name,
+        VALUE :module :: STRING AS module,
+        VALUE :logo :: STRING AS logo,
+        VALUE :chains AS chains,
+        VALUE :protocolType :: STRING AS protocol_type,
+        VALUE :methodologyURL :: STRING AS methodology_url,
+        VALUE :methodology AS methodology,
+        VALUE :parentProtocol :: STRING AS parent_protocol,
+        VALUE :slug :: STRING AS slug,
+        VALUE :linkedProtocols AS linked_protocols,
+        _inserted_timestamp
+    FROM
+        lat_flat
+    {% if is_incremental() %}
+    where VALUE :defillamaId :: STRING NOT IN (
+        select protocol_id from {{ this }}
+    )
+    {% endif %}
+)
+SELECT
+    protocol_id,
+    slug as protocol_slug,
+    category,
+    NAME,
+    display_name,
+    module,
+    logo,
+    chains,
+    protocol_type,
+    methodology_url,
+    methodology,
+    parent_protocol,
+    linked_protocols,
+    _inserted_timestamp,
+    sysdate() as inserted_timestamp,
+    sysdate() as modified_timestamp,
+    '{{ invocation_id }}' as _invocation_id
+FROM
+    protocol_expand

models/defillama/bronze/bronze__defillama_perps.yml

@@ -0,0 +1,30 @@
+version: 2
+models:
+  - name: bronze__defillama_perps
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+            combination_of_columns:
+              - protocol_id
+    
+    columns:
+      - name: protocol_id
+        tests: 
+          - not_null
+      - name: protocol_slug
+        tests:
+          - not_null
+      - name: category
+        tests:
+          - not_null
+      - name: name
+        tests:
+          - not_null
+      - name: display_name
+        tests:
+          - not_null
+      - name: _inserted_timestamp
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ

models/defillama/gold/defillama__dim_perps.sql

@@ -0,0 +1,24 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true },
+    tags = ['defillama'],
+    meta={
+        'database_tags':{
+            'table': {
+                'PROTOCOL': 'DEFILLAMA'
+            }
+        }
+    }
+) }}
+
+SELECT
+    protocol_id,
+    protocol_slug,
+    NAME as protocol,
+    category,
+    chains,
+    parent_protocol,
+    linked_protocols
+FROM
+    {{ ref('bronze__defillama_perps') }}

models/defillama/gold/defillama__dim_perps.yml

@@ -0,0 +1,20 @@
+version: 2
+models:
+  - name: defillama__dim_perps
+    description: This table contains dimensional information about the perpetual protocols listed on Defillama.
+
+    columns:
+      - name: protocol_id
+        description: Unique identifier of the protocol from Defillama.
+      - name: protocol_slug
+        description: URL-friendly slug identifier for the protocol.
+      - name: category
+        description: The category of protocol (e.g. Dexes, Options, Yield, Derivatives etc.).
+      - name: protocol
+        description: Name of the protocol.
+      - name: chains
+        description: Array of the various chains or networks that the protocol is deployed on.
+      - name: parent_protocol
+        description: Parent protocol identifier if this is a sub-protocol.
+      - name: linked_protocols
+        description: Array of related or linked protocols.

models/defillama/gold/defillama__fact_perp_volume.sql

@@ -0,0 +1,23 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true },
+    tags = ['defillama'],
+    meta={
+        'database_tags':{
+            'table': {
+                'PROTOCOL': 'DEFILLAMA'
+            }
+        }
+    }
+) }}
+
+SELECT
+    DATE,
+    chain,
+    protocol_id,
+    protocol_slug,
+    protocol,
+    volume
+FROM
+    {{ ref('silver__defillama_perp_daily_volume') }} f

models/defillama/gold/defillama__fact_perp_volume.yml

@@ -0,0 +1,18 @@
+version: 2
+models:
+  - name: defillama__fact_perp_volume
+    description: This table contains historical perpetual protocol volumes for the protocols listed on Defillama in `defillama__dim_perps`, where available.
+
+    columns:
+      - name: date
+        description: Date associated with the reported volume records.
+      - name: chain
+        description: The name of the blockchain where the volume occurred.
+      - name: protocol_id
+        description: Unique identifier of the protocol from Defillama.
+      - name: protocol_slug
+        description: URL-friendly slug identifier for the protocol.
+      - name: protocol
+        description: The name of the protocol.
+      - name: volume
+        description: The total volume driven by the protocol on a daily basis, denominated in USD.

models/defillama/silver/silver__defillama_perp_daily_volume.sql

@@ -0,0 +1,99 @@
+-- depends_on: {{ ref('silver__defillama_perp_metrics') }}
+{{ config(
+    materialized = 'incremental',
+    unique_key = 'defillama_perp_daily_volume_id',
+    cluster_by = ['date','chain','protocol_id'],
+    tags = ['defillama']
+) }}
+
+with base as (
+    select 
+        protocol_id,
+        protocol_slug,
+        name,
+        display_name,
+        total_data_chart_breakdown,
+        timestamp,
+        _inserted_timestamp,
+        defillama_perp_metrics_id
+    from {{ ref('silver__defillama_perp_metrics') }}
+    {% if is_incremental() %}
+    where _inserted_timestamp > (
+        select coalesce(max(_inserted_timestamp), '2025-01-01') from {{ this }}
+    )
+    {% endif %}
+),
+
+-- Flatten the total_data_chart_breakdown to get daily data points
+daily_data_flattened as (
+    select 
+        protocol_id,
+        protocol_slug,
+        name,
+        display_name,
+        timestamp,
+        _inserted_timestamp,
+        defillama_perp_metrics_id,
+        daily_date.value[0]::bigint as date_timestamp,
+        to_date(to_timestamp(daily_date.value[0]::bigint)) as date_day,
+        daily_date.value[1] as chain_breakdown_object,
+        daily_date.index as day_index
+    from base,
+    lateral flatten(input => total_data_chart_breakdown) as daily_date
+),
+
+-- Flatten the chain breakdown object to get blockchain/volume pairs
+chain_volume_flattened as (
+    select 
+        ddf.protocol_id,
+        ddf.protocol_slug,
+        ddf.name,
+        ddf.display_name,
+        ddf.timestamp,
+        ddf._inserted_timestamp,
+        ddf.defillama_perp_metrics_id,
+        ddf.date_day,
+        ddf.date_timestamp,
+        ddf.day_index,
+        chain_breakdown.key as chain,
+        chain_breakdown.value as protocol_volumes
+    from daily_data_flattened ddf,
+    lateral flatten(input => ddf.chain_breakdown_object) as chain_breakdown
+),
+
+-- Flatten the protocol volumes to get individual protocol volumes per blockchain
+final as (
+    select 
+        cvf.date_day as date,
+        cvf.chain,
+        cvf.protocol_id,
+        cvf.protocol_slug,
+        cvf.name as protocol,
+        protocol_vol.value::float as volume,
+        cvf._inserted_timestamp,
+        cvf.defillama_perp_metrics_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['cvf.protocol_id','cvf.date_day','cvf.chain']
+        ) }} as defillama_perp_daily_volume_id,
+        sysdate() as inserted_timestamp,
+        sysdate() as modified_timestamp,
+        '{{ invocation_id }}' as _invocation_id
+    from chain_volume_flattened cvf,
+    lateral flatten(input => cvf.protocol_volumes) as protocol_vol
+)
+
+select 
+    date,
+    chain,
+    protocol_id,
+    protocol_slug,
+    protocol,
+    volume,
+    defillama_perp_metrics_id,
+    defillama_perp_daily_volume_id,
+    _inserted_timestamp,
+    inserted_timestamp,
+    modified_timestamp,
+    _invocation_id
+from 
+    final qualify row_number() over (partition by defillama_perp_daily_volume_id order by _inserted_timestamp desc) = 1

models/defillama/silver/silver__defillama_perp_daily_volume.yml

@@ -0,0 +1,55 @@
+version: 2
+models:
+  - name: silver__defillama_perp_daily_volume
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+            combination_of_columns:
+              - protocol_id
+              - date
+              - chain
+    columns:
+      - name: date
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - DATE
+      - name: chain
+        tests:
+          - not_null
+      - name: protocol_id
+        tests:
+          - not_null
+      - name: protocol_slug
+        tests:
+          - not_null
+      - name: protocol
+        tests:
+          - not_null
+      - name: volume
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - FLOAT
+      - name: defillama_perp_metrics_id
+        tests:
+          - not_null
+      - name: defillama_perp_daily_volume_id
+        tests:
+          - not_null
+      - name: _inserted_timestamp
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+      - name: inserted_timestamp
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+      - name: modified_timestamp
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ

models/defillama/silver/silver__defillama_perp_metrics.sql

@@ -0,0 +1,79 @@
+-- depends_on: {{ ref('bronze__defillama_perps') }}
+{{ config(
+    materialized = 'incremental',
+    unique_key = 'defillama_perp_metrics_id',
+    cluster_by = ['timestamp','protocol_id'],
+    tags = ['defillama']
+) }}
+
+with base_raw as (
+    select 
+    data,
+    _inserted_timestamp
+    from 
+    {% if is_incremental() %}
+    {{ ref('bronze__defillama_perp_metrics') }}
+    where _inserted_timestamp > (
+        select coalesce(max(_inserted_timestamp), '2025-01-01') from {{ this }}
+    )
+    {% else %}
+    {{ ref('bronze__defillama_perp_metrics_FR') }}
+    {% endif %}
+),
+base as (
+    select 
+    _inserted_timestamp :: DATE AS timestamp,
+    DATA :defillamaId :: STRING AS protocol_id,
+    DATA :category :: STRING AS category,
+    DATA :name :: STRING AS name,
+    DATA :displayName :: STRING AS display_name,
+    DATA :module :: STRING AS module,
+    DATA :logo :: STRING AS logo,
+    DATA :chains AS chains,
+    DATA :protocolType :: STRING AS protocol_type,
+    DATA :methodologyURL :: STRING AS methodology_url,
+    DATA :methodology AS methodology,
+    DATA :parentProtocol :: STRING AS parent_protocol,
+    DATA :slug :: STRING AS slug,
+    DATA :linkedProtocols AS linked_protocols,
+    DATA :total24h :: FLOAT AS total_24h,
+    DATA :total48hto24h :: FLOAT AS total_48h_to_24h,
+    DATA :total7d :: FLOAT AS total_7d,
+    DATA :total30d :: FLOAT AS total_30d,
+    DATA :totalAllTime :: FLOAT AS total_all_time,
+    DATA :change_1d :: FLOAT AS change_1d,
+    DATA :totalDataChartBreakdown AS total_data_chart_breakdown,
+    _inserted_timestamp
+    from base_raw
+)
+select 
+    timestamp,
+    protocol_id,
+    category,
+    name,
+    display_name,
+    module,
+    logo,
+    chains,
+    protocol_type,
+    methodology_url,
+    methodology,
+    parent_protocol,
+    slug as protocol_slug,
+    linked_protocols,
+    total_24h,
+    total_48h_to_24h,
+    total_7d,
+    total_30d,
+    total_all_time,
+    change_1d,
+    total_data_chart_breakdown,
+    {{ dbt_utils.generate_surrogate_key(
+        ['protocol_id','timestamp']
+    ) }} as defillama_perp_metrics_id,
+    _inserted_timestamp,
+    sysdate() as inserted_timestamp,
+    sysdate() as modified_timestamp,
+    '{{ invocation_id }}' as _invocation_id
+from base
+qualify row_number() over (partition by defillama_perp_metrics_id order by _inserted_timestamp desc) = 1

models/defillama/silver/silver__defillama_perp_metrics.yml

@@ -0,0 +1,106 @@
+version: 2
+models:
+  - name: silver__defillama_perp_metrics
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+            combination_of_columns:
+              - protocol_id
+              - timestamp
+    
+    columns:
+      - name: timestamp
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - DATE
+      - name: protocol_id
+        tests:
+          - not_null
+      - name: category
+        tests:
+          - not_null
+      - name: name
+        tests:
+          - not_null
+      - name: display_name
+        tests:
+          - not_null
+      - name: module
+        tests:
+          - not_null
+      - name: logo
+        tests:
+          - not_null
+      - name: chains
+        tests:
+          - not_null
+      - name: protocol_type
+        tests:
+          - not_null
+      - name: methodology_url
+        tests:
+          - not_null
+      - name: methodology
+        tests:
+          - not_null
+      - name: parent_protocol
+        tests:
+          - not_null
+      - name: protocol_slug
+        tests:
+          - not_null
+      - name: linked_protocols
+        tests:
+          - not_null
+      - name: total_24h
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - FLOAT
+      - name: total_48h_to_24h
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - FLOAT
+      - name: total_7d
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - FLOAT
+      - name: total_30d
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - FLOAT
+      - name: total_all_time
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - FLOAT
+      - name: change_1d
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - FLOAT
+      - name: total_data_chart_breakdown
+        tests:
+          - not_null
+      - name: defillama_perp_metrics_id
+        tests:
+          - not_null
+      - name: _inserted_timestamp
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+      - name: inserted_timestamp
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+      - name: modified_timestamp
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ

models/defillama/streamline/bronze/bronze__defillama_perp_metrics.sql

@@ -0,0 +1,9 @@
+{{ config (
+    materialized = 'view',
+    tags = ['defillama_streamline']
+) }}
+{{ streamline_external_table_query_v2(
+    model = 'defillama_perp_metrics',
+    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)",
+    partition_name = "partition_key"
+) }}

models/defillama/streamline/bronze/bronze__defillama_perp_metrics_FR.sql

@@ -0,0 +1,9 @@
+{{ config (
+    materialized = 'view',
+    tags = ['defillama_streamline']
+) }}
+{{ streamline_external_table_FR_query_v2(
+    model = 'defillama_perp_metrics',
+    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)",
+    partition_name = "partition_key"
+) }}

models/defillama/streamline/realtime/streamline__defillama_perp_metrics.sql

@@ -0,0 +1,44 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'streamline.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params ={ "external_table" :"defillama_perp_metrics",
+        "sql_limit" :"10000",
+        "producer_batch_size" :"10",
+        "worker_batch_size" :"1",
+        "async_concurrent_requests" :"1",
+        "sql_source" :"{{this.identifier}}",
+        "exploded_key": tojson(['data'])
+        }
+    ),
+    tags = ['defillama_streamline']
+) }}
+
+WITH perps as (
+
+    select 
+        protocol_slug,
+        protocol_id
+    from {{ ref('bronze__defillama_perps') }}
+)
+SELECT
+    protocol_slug,
+    protocol_id,
+    date_part('epoch_second', sysdate()) as run_timestamp,
+    date_part('epoch_second', sysdate()::DATE) AS partition_key,
+    {{ target.database }}.live.udf_api(
+        'GET',
+        'https://pro-api.llama.fi/{api_key}/api/summary/derivatives/'|| protocol_slug || '?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=false',
+        OBJECT_CONSTRUCT(
+            'Content-Type', 'text/plain',
+            'Accept', 'text/plain',
+            'fsc-quantum-state', 'streamline'
+        ),
+        {},
+        'Vault/prod/external/defillama'
+    ) AS request
+FROM
+    perps
+where protocol_slug is not null
+limit 10000

models/sources.yml

@@ -12,6 +12,7 @@ sources:
       - name: valuations_parquet
       - name: defillama_stablecoin_metrics
       - name: defillama_protocols
+      - name: defillama_perp_metrics
   - name: tokenflow_eth
     database: flipside_prod_db
     schema: tokenflow_eth