From 01cbd0eb91df9f1f4b81a78f1a8de9fe207631db Mon Sep 17 00:00:00 2001 From: Matt Romano <42412983+mattromano@users.noreply.github.com> Date: Thu, 25 Sep 2025 09:46:44 -0700 Subject: [PATCH] DAT2-37/add-defillama-perps (#139) --- .../bronze/bronze__defillama_perps.sql | 73 ++++++++++++ .../bronze/bronze__defillama_perps.yml | 30 +++++ .../defillama/gold/defillama__dim_perps.sql | 24 ++++ .../defillama/gold/defillama__dim_perps.yml | 20 ++++ .../gold/defillama__fact_perp_volume.sql | 23 ++++ .../gold/defillama__fact_perp_volume.yml | 18 +++ .../silver__defillama_perp_daily_volume.sql | 99 ++++++++++++++++ .../silver__defillama_perp_daily_volume.yml | 55 +++++++++ .../silver/silver__defillama_perp_metrics.sql | 79 +++++++++++++ .../silver/silver__defillama_perp_metrics.yml | 106 ++++++++++++++++++ .../bronze/bronze__defillama_perp_metrics.sql | 9 ++ .../bronze__defillama_perp_metrics_FR.sql | 9 ++ .../streamline__defillama_perp_metrics.sql | 44 ++++++++ models/sources.yml | 1 + 14 files changed, 590 insertions(+) create mode 100644 models/defillama/bronze/bronze__defillama_perps.sql create mode 100644 models/defillama/bronze/bronze__defillama_perps.yml create mode 100644 models/defillama/gold/defillama__dim_perps.sql create mode 100644 models/defillama/gold/defillama__dim_perps.yml create mode 100644 models/defillama/gold/defillama__fact_perp_volume.sql create mode 100644 models/defillama/gold/defillama__fact_perp_volume.yml create mode 100644 models/defillama/silver/silver__defillama_perp_daily_volume.sql create mode 100644 models/defillama/silver/silver__defillama_perp_daily_volume.yml create mode 100644 models/defillama/silver/silver__defillama_perp_metrics.sql create mode 100644 models/defillama/silver/silver__defillama_perp_metrics.yml create mode 100644 models/defillama/streamline/bronze/bronze__defillama_perp_metrics.sql create mode 100644 models/defillama/streamline/bronze/bronze__defillama_perp_metrics_FR.sql create mode 100644 models/defillama/streamline/realtime/streamline__defillama_perp_metrics.sql diff --git a/models/defillama/bronze/bronze__defillama_perps.sql b/models/defillama/bronze/bronze__defillama_perps.sql new file mode 100644 index 0000000..77ccd54 --- /dev/null +++ b/models/defillama/bronze/bronze__defillama_perps.sql @@ -0,0 +1,73 @@ +{{ config( + materialized = 'incremental', + unique_key = ['protocol_id'], + cluster_by = ['protocol_id'], + tags = ['defillama'] +) }} + +WITH api_pull AS ( + + SELECT + PARSE_JSON( + live.udf_api( + 'GET', + 'https://pro-api.llama.fi/{api_key}/api/overview/derivatives?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true', + {}, + {}, + 'Vault/prod/external/defillama' + ) + ) :data:protocols AS response, + SYSDATE() AS _inserted_timestamp +), +lat_flat AS ( + SELECT + r.value AS VALUE, + _inserted_timestamp + FROM + api_pull, + LATERAL FLATTEN (response) AS r +), +protocol_expand AS ( + SELECT + VALUE :defillamaId :: STRING AS protocol_id, + VALUE :category :: STRING AS category, + VALUE :name :: STRING AS NAME, + VALUE :displayName :: STRING AS display_name, + VALUE :module :: STRING AS module, + VALUE :logo :: STRING AS logo, + VALUE :chains AS chains, + VALUE :protocolType :: STRING AS protocol_type, + VALUE :methodologyURL :: STRING AS methodology_url, + VALUE :methodology AS methodology, + VALUE :parentProtocol :: STRING AS parent_protocol, + VALUE :slug :: STRING AS slug, + VALUE :linkedProtocols AS linked_protocols, + _inserted_timestamp + FROM + lat_flat + {% if is_incremental() %} + where VALUE :defillamaId :: STRING NOT IN ( + select protocol_id from {{ this }} + ) + {% endif %} +) +SELECT + protocol_id, + slug as protocol_slug, + category, + NAME, + display_name, + module, + logo, + chains, + protocol_type, + methodology_url, + methodology, + parent_protocol, + linked_protocols, + _inserted_timestamp, + sysdate() as inserted_timestamp, + sysdate() as modified_timestamp, + '{{ invocation_id }}' as _invocation_id +FROM + protocol_expand diff --git a/models/defillama/bronze/bronze__defillama_perps.yml b/models/defillama/bronze/bronze__defillama_perps.yml new file mode 100644 index 0000000..32b3696 --- /dev/null +++ b/models/defillama/bronze/bronze__defillama_perps.yml @@ -0,0 +1,30 @@ +version: 2 +models: + - name: bronze__defillama_perps + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - protocol_id + + columns: + - name: protocol_id + tests: + - not_null + - name: protocol_slug + tests: + - not_null + - name: category + tests: + - not_null + - name: name + tests: + - not_null + - name: display_name + tests: + - not_null + - name: _inserted_timestamp + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ diff --git a/models/defillama/gold/defillama__dim_perps.sql b/models/defillama/gold/defillama__dim_perps.sql new file mode 100644 index 0000000..f33436c --- /dev/null +++ b/models/defillama/gold/defillama__dim_perps.sql @@ -0,0 +1,24 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + tags = ['defillama'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'DEFILLAMA' + } + } + } +) }} + +SELECT + protocol_id, + protocol_slug, + NAME as protocol, + category, + chains, + parent_protocol, + linked_protocols +FROM + {{ ref('bronze__defillama_perps') }} \ No newline at end of file diff --git a/models/defillama/gold/defillama__dim_perps.yml b/models/defillama/gold/defillama__dim_perps.yml new file mode 100644 index 0000000..43e18b7 --- /dev/null +++ b/models/defillama/gold/defillama__dim_perps.yml @@ -0,0 +1,20 @@ +version: 2 +models: + - name: defillama__dim_perps + description: This table contains dimensional information about the perpetual protocols listed on Defillama. + + columns: + - name: protocol_id + description: Unique identifier of the protocol from Defillama. + - name: protocol_slug + description: URL-friendly slug identifier for the protocol. + - name: category + description: The category of protocol (e.g. Dexes, Options, Yield, Derivatives etc.). + - name: protocol + description: Name of the protocol. + - name: chains + description: Array of the various chains or networks that the protocol is deployed on. + - name: parent_protocol + description: Parent protocol identifier if this is a sub-protocol. + - name: linked_protocols + description: Array of related or linked protocols. \ No newline at end of file diff --git a/models/defillama/gold/defillama__fact_perp_volume.sql b/models/defillama/gold/defillama__fact_perp_volume.sql new file mode 100644 index 0000000..5b9d226 --- /dev/null +++ b/models/defillama/gold/defillama__fact_perp_volume.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + tags = ['defillama'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'DEFILLAMA' + } + } + } +) }} + +SELECT + DATE, + chain, + protocol_id, + protocol_slug, + protocol, + volume +FROM + {{ ref('silver__defillama_perp_daily_volume') }} f diff --git a/models/defillama/gold/defillama__fact_perp_volume.yml b/models/defillama/gold/defillama__fact_perp_volume.yml new file mode 100644 index 0000000..6272f62 --- /dev/null +++ b/models/defillama/gold/defillama__fact_perp_volume.yml @@ -0,0 +1,18 @@ +version: 2 +models: + - name: defillama__fact_perp_volume + description: This table contains historical perpetual protocol volumes for the protocols listed on Defillama in `defillama__dim_perps`, where available. + + columns: + - name: date + description: Date associated with the reported volume records. + - name: chain + description: The name of the blockchain where the volume occurred. + - name: protocol_id + description: Unique identifier of the protocol from Defillama. + - name: protocol_slug + description: URL-friendly slug identifier for the protocol. + - name: protocol + description: The name of the protocol. + - name: volume + description: The total volume driven by the protocol on a daily basis, denominated in USD. \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_perp_daily_volume.sql b/models/defillama/silver/silver__defillama_perp_daily_volume.sql new file mode 100644 index 0000000..b265345 --- /dev/null +++ b/models/defillama/silver/silver__defillama_perp_daily_volume.sql @@ -0,0 +1,99 @@ +-- depends_on: {{ ref('silver__defillama_perp_metrics') }} +{{ config( + materialized = 'incremental', + unique_key = 'defillama_perp_daily_volume_id', + cluster_by = ['date','chain','protocol_id'], + tags = ['defillama'] +) }} + +with base as ( + select + protocol_id, + protocol_slug, + name, + display_name, + total_data_chart_breakdown, + timestamp, + _inserted_timestamp, + defillama_perp_metrics_id + from {{ ref('silver__defillama_perp_metrics') }} + {% if is_incremental() %} + where _inserted_timestamp > ( + select coalesce(max(_inserted_timestamp), '2025-01-01') from {{ this }} + ) + {% endif %} +), + +-- Flatten the total_data_chart_breakdown to get daily data points +daily_data_flattened as ( + select + protocol_id, + protocol_slug, + name, + display_name, + timestamp, + _inserted_timestamp, + defillama_perp_metrics_id, + daily_date.value[0]::bigint as date_timestamp, + to_date(to_timestamp(daily_date.value[0]::bigint)) as date_day, + daily_date.value[1] as chain_breakdown_object, + daily_date.index as day_index + from base, + lateral flatten(input => total_data_chart_breakdown) as daily_date +), + +-- Flatten the chain breakdown object to get blockchain/volume pairs +chain_volume_flattened as ( + select + ddf.protocol_id, + ddf.protocol_slug, + ddf.name, + ddf.display_name, + ddf.timestamp, + ddf._inserted_timestamp, + ddf.defillama_perp_metrics_id, + ddf.date_day, + ddf.date_timestamp, + ddf.day_index, + chain_breakdown.key as chain, + chain_breakdown.value as protocol_volumes + from daily_data_flattened ddf, + lateral flatten(input => ddf.chain_breakdown_object) as chain_breakdown +), + +-- Flatten the protocol volumes to get individual protocol volumes per blockchain +final as ( + select + cvf.date_day as date, + cvf.chain, + cvf.protocol_id, + cvf.protocol_slug, + cvf.name as protocol, + protocol_vol.value::float as volume, + cvf._inserted_timestamp, + cvf.defillama_perp_metrics_id, + {{ dbt_utils.generate_surrogate_key( + ['cvf.protocol_id','cvf.date_day','cvf.chain'] + ) }} as defillama_perp_daily_volume_id, + sysdate() as inserted_timestamp, + sysdate() as modified_timestamp, + '{{ invocation_id }}' as _invocation_id + from chain_volume_flattened cvf, + lateral flatten(input => cvf.protocol_volumes) as protocol_vol +) + +select + date, + chain, + protocol_id, + protocol_slug, + protocol, + volume, + defillama_perp_metrics_id, + defillama_perp_daily_volume_id, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + _invocation_id +from + final qualify row_number() over (partition by defillama_perp_daily_volume_id order by _inserted_timestamp desc) = 1 diff --git a/models/defillama/silver/silver__defillama_perp_daily_volume.yml b/models/defillama/silver/silver__defillama_perp_daily_volume.yml new file mode 100644 index 0000000..8aec986 --- /dev/null +++ b/models/defillama/silver/silver__defillama_perp_daily_volume.yml @@ -0,0 +1,55 @@ +version: 2 +models: + - name: silver__defillama_perp_daily_volume + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - protocol_id + - date + - chain + columns: + - name: date + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - DATE + - name: chain + tests: + - not_null + - name: protocol_id + tests: + - not_null + - name: protocol_slug + tests: + - not_null + - name: protocol + tests: + - not_null + - name: volume + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - name: defillama_perp_metrics_id + tests: + - not_null + - name: defillama_perp_daily_volume_id + tests: + - not_null + - name: _inserted_timestamp + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: inserted_timestamp + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: modified_timestamp + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ diff --git a/models/defillama/silver/silver__defillama_perp_metrics.sql b/models/defillama/silver/silver__defillama_perp_metrics.sql new file mode 100644 index 0000000..22085b1 --- /dev/null +++ b/models/defillama/silver/silver__defillama_perp_metrics.sql @@ -0,0 +1,79 @@ +-- depends_on: {{ ref('bronze__defillama_perps') }} +{{ config( + materialized = 'incremental', + unique_key = 'defillama_perp_metrics_id', + cluster_by = ['timestamp','protocol_id'], + tags = ['defillama'] +) }} + +with base_raw as ( + select + data, + _inserted_timestamp + from + {% if is_incremental() %} + {{ ref('bronze__defillama_perp_metrics') }} + where _inserted_timestamp > ( + select coalesce(max(_inserted_timestamp), '2025-01-01') from {{ this }} + ) + {% else %} + {{ ref('bronze__defillama_perp_metrics_FR') }} + {% endif %} +), +base as ( + select + _inserted_timestamp :: DATE AS timestamp, + DATA :defillamaId :: STRING AS protocol_id, + DATA :category :: STRING AS category, + DATA :name :: STRING AS name, + DATA :displayName :: STRING AS display_name, + DATA :module :: STRING AS module, + DATA :logo :: STRING AS logo, + DATA :chains AS chains, + DATA :protocolType :: STRING AS protocol_type, + DATA :methodologyURL :: STRING AS methodology_url, + DATA :methodology AS methodology, + DATA :parentProtocol :: STRING AS parent_protocol, + DATA :slug :: STRING AS slug, + DATA :linkedProtocols AS linked_protocols, + DATA :total24h :: FLOAT AS total_24h, + DATA :total48hto24h :: FLOAT AS total_48h_to_24h, + DATA :total7d :: FLOAT AS total_7d, + DATA :total30d :: FLOAT AS total_30d, + DATA :totalAllTime :: FLOAT AS total_all_time, + DATA :change_1d :: FLOAT AS change_1d, + DATA :totalDataChartBreakdown AS total_data_chart_breakdown, + _inserted_timestamp + from base_raw +) +select + timestamp, + protocol_id, + category, + name, + display_name, + module, + logo, + chains, + protocol_type, + methodology_url, + methodology, + parent_protocol, + slug as protocol_slug, + linked_protocols, + total_24h, + total_48h_to_24h, + total_7d, + total_30d, + total_all_time, + change_1d, + total_data_chart_breakdown, + {{ dbt_utils.generate_surrogate_key( + ['protocol_id','timestamp'] + ) }} as defillama_perp_metrics_id, + _inserted_timestamp, + sysdate() as inserted_timestamp, + sysdate() as modified_timestamp, + '{{ invocation_id }}' as _invocation_id +from base +qualify row_number() over (partition by defillama_perp_metrics_id order by _inserted_timestamp desc) = 1 \ No newline at end of file diff --git a/models/defillama/silver/silver__defillama_perp_metrics.yml b/models/defillama/silver/silver__defillama_perp_metrics.yml new file mode 100644 index 0000000..b8e3573 --- /dev/null +++ b/models/defillama/silver/silver__defillama_perp_metrics.yml @@ -0,0 +1,106 @@ +version: 2 +models: + - name: silver__defillama_perp_metrics + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - protocol_id + - timestamp + + columns: + - name: timestamp + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - DATE + - name: protocol_id + tests: + - not_null + - name: category + tests: + - not_null + - name: name + tests: + - not_null + - name: display_name + tests: + - not_null + - name: module + tests: + - not_null + - name: logo + tests: + - not_null + - name: chains + tests: + - not_null + - name: protocol_type + tests: + - not_null + - name: methodology_url + tests: + - not_null + - name: methodology + tests: + - not_null + - name: parent_protocol + tests: + - not_null + - name: protocol_slug + tests: + - not_null + - name: linked_protocols + tests: + - not_null + - name: total_24h + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - name: total_48h_to_24h + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - name: total_7d + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - name: total_30d + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - name: total_all_time + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - name: change_1d + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - name: total_data_chart_breakdown + tests: + - not_null + - name: defillama_perp_metrics_id + tests: + - not_null + - name: _inserted_timestamp + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: inserted_timestamp + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: modified_timestamp + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/defillama/streamline/bronze/bronze__defillama_perp_metrics.sql b/models/defillama/streamline/bronze/bronze__defillama_perp_metrics.sql new file mode 100644 index 0000000..164e3a0 --- /dev/null +++ b/models/defillama/streamline/bronze/bronze__defillama_perp_metrics.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['defillama_streamline'] +) }} +{{ streamline_external_table_query_v2( + model = 'defillama_perp_metrics', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)", + partition_name = "partition_key" +) }} diff --git a/models/defillama/streamline/bronze/bronze__defillama_perp_metrics_FR.sql b/models/defillama/streamline/bronze/bronze__defillama_perp_metrics_FR.sql new file mode 100644 index 0000000..d193f67 --- /dev/null +++ b/models/defillama/streamline/bronze/bronze__defillama_perp_metrics_FR.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['defillama_streamline'] +) }} +{{ streamline_external_table_FR_query_v2( + model = 'defillama_perp_metrics', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER)", + partition_name = "partition_key" +) }} diff --git a/models/defillama/streamline/realtime/streamline__defillama_perp_metrics.sql b/models/defillama/streamline/realtime/streamline__defillama_perp_metrics.sql new file mode 100644 index 0000000..845311b --- /dev/null +++ b/models/defillama/streamline/realtime/streamline__defillama_perp_metrics.sql @@ -0,0 +1,44 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"defillama_perp_metrics", + "sql_limit" :"10000", + "producer_batch_size" :"10", + "worker_batch_size" :"1", + "async_concurrent_requests" :"1", + "sql_source" :"{{this.identifier}}", + "exploded_key": tojson(['data']) + } + ), + tags = ['defillama_streamline'] +) }} + +WITH perps as ( + + select + protocol_slug, + protocol_id + from {{ ref('bronze__defillama_perps') }} +) +SELECT + protocol_slug, + protocol_id, + date_part('epoch_second', sysdate()) as run_timestamp, + date_part('epoch_second', sysdate()::DATE) AS partition_key, + {{ target.database }}.live.udf_api( + 'GET', + 'https://pro-api.llama.fi/{api_key}/api/summary/derivatives/'|| protocol_slug || '?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=false', + OBJECT_CONSTRUCT( + 'Content-Type', 'text/plain', + 'Accept', 'text/plain', + 'fsc-quantum-state', 'streamline' + ), + {}, + 'Vault/prod/external/defillama' + ) AS request +FROM + perps +where protocol_slug is not null +limit 10000 \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index cb2d7d3..e64b5b1 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -12,6 +12,7 @@ sources: - name: valuations_parquet - name: defillama_stablecoin_metrics - name: defillama_protocols + - name: defillama_perp_metrics - name: tokenflow_eth database: flipside_prod_db schema: tokenflow_eth