From fa535ccb8c9cecb8e4fa3642a7c697d2c60be8b1 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 17 Nov 2025 13:49:23 -0700 Subject: [PATCH 001/164] initial models --- .../reads/bronze/bronze__curated_reads.sql | 16 +++ .../reads/bronze/bronze__curated_reads_fr.sql | 16 +++ .../reads/silver/silver__curated_reads.sql | 51 +++++++ .../reads/silver/silver__curated_reads.yml | 7 + .../streamline__curated_reads_complete.sql | 49 +++++++ .../streamline__curated_reads_history.sql | 120 +++++++++++++++++ .../streamline__curated_reads_realtime.sql | 127 ++++++++++++++++++ 7 files changed, 386 insertions(+) create mode 100644 models/curated_package/reads/bronze/bronze__curated_reads.sql create mode 100644 models/curated_package/reads/bronze/bronze__curated_reads_fr.sql create mode 100644 models/curated_package/reads/silver/silver__curated_reads.sql create mode 100644 models/curated_package/reads/silver/silver__curated_reads.yml create mode 100644 models/curated_package/reads/streamline/complete/streamline__curated_reads_complete.sql create mode 100644 models/curated_package/reads/streamline/history/streamline__curated_reads_history.sql create mode 100644 models/curated_package/reads/streamline/realtime/streamline__curated_reads_realtime.sql diff --git a/models/curated_package/reads/bronze/bronze__curated_reads.sql b/models/curated_package/reads/bronze/bronze__curated_reads.sql new file mode 100644 index 00000000..dacfdce9 --- /dev/null +++ b/models/curated_package/reads/bronze/bronze__curated_reads.sql @@ -0,0 +1,16 @@ +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze','curated_reads','phase_4'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query( + source_name = 'curated_reads', + block_number = false, + contract_address = true, + data_not_null = false +) }} \ No newline at end of file diff --git a/models/curated_package/reads/bronze/bronze__curated_reads_fr.sql b/models/curated_package/reads/bronze/bronze__curated_reads_fr.sql new file mode 100644 index 00000000..1bc1c4b1 --- /dev/null +++ b/models/curated_package/reads/bronze/bronze__curated_reads_fr.sql @@ -0,0 +1,16 @@ +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'view', + tags = ['bronze','curated_reads','phase_4'] +) }} + +{# Main query starts here #} +{{ streamline_external_table_query_fr( + source_name = 'curated_reads', + block_number = false, + contract_address = true, + data_not_null = false +) }} \ No newline at end of file diff --git a/models/curated_package/reads/silver/silver__curated_reads.sql b/models/curated_package/reads/silver/silver__curated_reads.sql new file mode 100644 index 00000000..1020af69 --- /dev/null +++ b/models/curated_package/reads/silver/silver__curated_reads.sql @@ -0,0 +1,51 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +-- depends_on: {{ ref('bronze__curated_reads') }} +{{ config( + materialized = 'incremental', + unique_key = 'curated_reads_id', + cluster_by = ['block_date'], + incremental_predicates = ["dynamic_range", "block_number"], + merge_exclude_columns = ["inserted_timestamp"], + full_refresh = vars.GLOBAL_SILVER_FR_ENABLED, + tags = ['silver','curated_reads','curated_daily'] +) }} + +SELECT + VALUE :"BLOCK_NUMBER" :: NUMBER AS block_number, + ( + VALUE :"BLOCK_DATE_UNIX" :: TIMESTAMP + ) :: DATE AS block_date, + contract_address, + DATA :result :: STRING AS amount_hex, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address'] + ) }} AS curated_reads_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze__curated_reads') }} +WHERE + _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1970-01-01') + FROM + {{ this }}) + AND DATA :result :: STRING <> '0x' + {% else %} + {{ ref('bronze__curated_reads_fr') }} + WHERE + DATA :result :: STRING <> '0x' + {% endif %} + + qualify(ROW_NUMBER() over (PARTITION BY block_number, contract_address + ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/curated_package/reads/silver/silver__curated_reads.yml b/models/curated_package/reads/silver/silver__curated_reads.yml new file mode 100644 index 00000000..eb6583c1 --- /dev/null +++ b/models/curated_package/reads/silver/silver__curated_reads.yml @@ -0,0 +1,7 @@ +version: 2 +models: + - name: silver__curated_reads + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - curated_reads_id \ No newline at end of file diff --git a/models/curated_package/reads/streamline/complete/streamline__curated_reads_complete.sql b/models/curated_package/reads/streamline/complete/streamline__curated_reads_complete.sql new file mode 100644 index 00000000..38d90323 --- /dev/null +++ b/models/curated_package/reads/streamline/complete/streamline__curated_reads_complete.sql @@ -0,0 +1,49 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +-- depends on: {{ ref('bronze__curated_reads') }} + +{# Set up dbt configuration #} +{{ config ( + materialized = 'incremental', + unique_key = 'curated_reads_complete_id', + incremental_predicates = ['dynamic_range', 'partition_key'], + full_refresh = vars.GLOBAL_STREAMLINE_FR_ENABLED, + tags = ['streamline','curated_reads','complete','phase_4'] +) }} + +SELECT + partition_key, + contract_address, + VALUE :"BLOCK_NUMBER" :: NUMBER AS block_number, + ( + VALUE :"BLOCK_DATE_UNIX" :: TIMESTAMP + ) :: DATE AS block_date, + file_name, + {{ dbt_utils.generate_surrogate_key( + ['contract_address', 'block_number'] + ) }} AS curated_reads_complete_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze__curated_reads') }} +WHERE + _inserted_timestamp >= ( + SELECT + COALESCE (MAX(_inserted_timestamp), '1970-01-01' :: TIMESTAMP) + FROM + {{ this }}) + {% else %} + {{ ref('bronze__curated_reads_fr') }} + {% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY curated_reads_complete_id +ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/curated_package/reads/streamline/history/streamline__curated_reads_history.sql b/models/curated_package/reads/streamline/history/streamline__curated_reads_history.sql new file mode 100644 index 00000000..62c73367 --- /dev/null +++ b/models/curated_package/reads/streamline/history/streamline__curated_reads_history.sql @@ -0,0 +1,120 @@ +{# Get variables +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +{{ config ( + materialized = "view", + tags = ['streamline','curated_reads','history','phase_4'] +) }} + +WITH max_blocks AS ( + SELECT + block_number, + block_date + FROM + {{ ref("_max_block_by_date") }} + WHERE + block_date >= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE +), +base AS ( + SELECT + c.contract_address, + m.block_number, + m.block_date + FROM + max_blocks m + LEFT JOIN {{ ref('streamline__curated_reads_complete') }} c + ON m.block_number = c.block_number + WHERE + c.contract_address IS NULL + AND m.block_date < ( + SELECT MAX(block_date) + FROM max_blocks + ) +), +function_sigs AS ( + SELECT + '0x18160ddd' AS function_sig, + 'totalSupply' AS function_name +), +ready_reads AS ( + SELECT + contract_address, + block_number, + block_date, + function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input + FROM + base + JOIN function_sigs + ON 1 = 1 +) +SELECT + contract_address, + block_number, + DATE_PART('EPOCH_SECONDS', block_date) :: INT AS block_date_unix, + ROUND(block_number,-3) AS partition_key, + function_sig, + input, + live.udf_api( + 'POST', + '{{ vars.GLOBAL_NODE_URL }}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', 'streamline' + ), + OBJECT_CONSTRUCT( + 'method', 'eth_call', + 'jsonrpc', '2.0', + 'params', [{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(block_number)], + 'id', concat_ws( + '-', + contract_address, + input, + block_number + ) + ), + '{{ vars.GLOBAL_NODE_VAULT_PATH }}' + ) AS request +FROM + ready_reads +WHERE + EXISTS ( + SELECT + 1 + FROM + ready_reads + LIMIT + 1 + ) + +{# Streamline Function Call #} +{% if execute %} + {% set params = { + "external_table": 'stablecoin_reads', + "sql_limit": vars.CURATED_SL_STABLECOIN_READS_HISTORY_SQL_LIMIT, + "producer_batch_size": vars.CURATED_SL_STABLECOIN_READS_HISTORY_PRODUCER_BATCH_SIZE, + "worker_batch_size": vars.CURATED_SL_STABLECOIN_READS_HISTORY_WORKER_BATCH_SIZE, + "async_concurrent_requests": vars.CURATED_SL_STABLECOIN_READS_HISTORY_ASYNC_CONCURRENT_REQUESTS, + "sql_source": 'stablecoin_reads_history' + } %} + + {% set function_call_sql %} + {{ fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = this.schema ~ "." ~ this.identifier, + params = params + ) }} + {% endset %} + + {% do run_query(function_call_sql) %} + {{ log("Streamline function call: " ~ function_call_sql, info=true) }} +{% endif %} + #} diff --git a/models/curated_package/reads/streamline/realtime/streamline__curated_reads_realtime.sql b/models/curated_package/reads/streamline/realtime/streamline__curated_reads_realtime.sql new file mode 100644 index 00000000..19946d8b --- /dev/null +++ b/models/curated_package/reads/streamline/realtime/streamline__curated_reads_realtime.sql @@ -0,0 +1,127 @@ +{# Get variables +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +{{ config ( + materialized = "view", + tags = ['streamline','stablecoin_reads','realtime','phase_4'] +) }} + +WITH verified_stablecoins AS ( + SELECT + contract_address + FROM + {{ ref('defi__dim_stablecoins') }} + WHERE + is_verified + AND contract_address IS NOT NULL +), +max_blocks AS ( + SELECT + block_number, + block_date + FROM + {{ ref("_max_block_by_date") }} + WHERE + block_date >= DATEADD('day',-4,SYSDATE()) --last 3 max block_number by date +), +base AS ( + SELECT + s.contract_address, + m.block_number, + m.block_date + FROM + verified_stablecoins s + CROSS JOIN max_blocks m + LEFT JOIN {{ ref('streamline__stablecoin_reads_complete') }} c + ON s.contract_address = c.contract_address + AND m.block_number = c.block_number + WHERE + c.contract_address IS NULL +), +function_sigs AS ( + SELECT + '0x18160ddd' AS function_sig, + 'totalSupply' AS function_name +), +ready_reads AS ( + SELECT + contract_address, + block_number, + block_date, + function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input + FROM + base + JOIN function_sigs + ON 1 = 1 +) +SELECT + contract_address, + block_number, + DATE_PART('EPOCH_SECONDS', block_date) :: INT AS block_date_unix, + ROUND(block_number,-3) AS partition_key, + function_sig, + input, + live.udf_api( + 'POST', + '{{ vars.GLOBAL_NODE_URL }}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', 'streamline' + ), + OBJECT_CONSTRUCT( + 'method', 'eth_call', + 'jsonrpc', '2.0', + 'params', [{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(block_number)], + 'id', concat_ws( + '-', + contract_address, + input, + block_number + ) + ), + '{{ vars.GLOBAL_NODE_VAULT_PATH }}' + ) AS request +FROM + ready_reads +WHERE + EXISTS ( + SELECT + 1 + FROM + ready_reads + LIMIT + 1 + ) + +{# Streamline Function Call #} +{% if execute %} + {% set params = { + "external_table" :"stablecoin_reads", + "sql_limit" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_SQL_LIMIT, + "producer_batch_size" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_PRODUCER_BATCH_SIZE, + "worker_batch_size" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_WORKER_BATCH_SIZE, + "async_concurrent_requests" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_ASYNC_CONCURRENT_REQUESTS, + "sql_source" : 'stablecoin_reads_realtime' + } %} + + {% set function_call_sql %} + {{ fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = this.schema ~ "." ~ this.identifier, + params = params + ) }} + {% endset %} + + {% do run_query(function_call_sql) %} + {{ log("Streamline function call: " ~ function_call_sql, info=true) }} +{% endif %} + #} From 640496ee105a366c5dab51d4047b1cca73685134 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 17 Nov 2025 14:12:26 -0700 Subject: [PATCH 002/164] initial models --- .../bronze/bronze__curated_reads.sql | 0 .../bronze/bronze__curated_reads_fr.sql | 0 .../silver/silver__curated_reads.sql | 11 +- .../silver/silver__curated_reads.yml | 0 .../streamline__curated_reads_complete.sql | 6 +- ...reamline__curated_reads_daily_realtime.sql | 39 ++++++ ..._curated_reads_daily_realtime_requests.sql | 107 +++++++++++++++ ...treamline__curated_reads_daily_records.sql | 0 .../streamline__curated_reads_history.sql | 120 ----------------- .../streamline__curated_reads_realtime.sql | 127 ------------------ 10 files changed, 158 insertions(+), 252 deletions(-) rename models/curated_package/{reads => curated_reads}/bronze/bronze__curated_reads.sql (100%) rename models/curated_package/{reads => curated_reads}/bronze/bronze__curated_reads_fr.sql (100%) rename models/curated_package/{reads => curated_reads}/silver/silver__curated_reads.sql (77%) rename models/curated_package/{reads => curated_reads}/silver/silver__curated_reads.yml (100%) rename models/curated_package/{reads => curated_reads}/streamline/complete/streamline__curated_reads_complete.sql (79%) create mode 100644 models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime.sql create mode 100644 models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime_requests.sql create mode 100644 models/curated_package/curated_reads/streamline/streamline__curated_reads_daily_records.sql delete mode 100644 models/curated_package/reads/streamline/history/streamline__curated_reads_history.sql delete mode 100644 models/curated_package/reads/streamline/realtime/streamline__curated_reads_realtime.sql diff --git a/models/curated_package/reads/bronze/bronze__curated_reads.sql b/models/curated_package/curated_reads/bronze/bronze__curated_reads.sql similarity index 100% rename from models/curated_package/reads/bronze/bronze__curated_reads.sql rename to models/curated_package/curated_reads/bronze/bronze__curated_reads.sql diff --git a/models/curated_package/reads/bronze/bronze__curated_reads_fr.sql b/models/curated_package/curated_reads/bronze/bronze__curated_reads_fr.sql similarity index 100% rename from models/curated_package/reads/bronze/bronze__curated_reads_fr.sql rename to models/curated_package/curated_reads/bronze/bronze__curated_reads_fr.sql diff --git a/models/curated_package/reads/silver/silver__curated_reads.sql b/models/curated_package/curated_reads/silver/silver__curated_reads.sql similarity index 77% rename from models/curated_package/reads/silver/silver__curated_reads.sql rename to models/curated_package/curated_reads/silver/silver__curated_reads.sql index 1020af69..3aee60a7 100644 --- a/models/curated_package/reads/silver/silver__curated_reads.sql +++ b/models/curated_package/curated_reads/silver/silver__curated_reads.sql @@ -16,15 +16,18 @@ ) }} SELECT + VALUE :"ADDRESS" :: STRING AS address, + VALUE :"FUNCTION_NAME" :: STRING AS function_name, + VALUE :"FUNCTION_SIG" :: STRING AS function_sig, + VALUE :"INPUT" :: STRING AS input, VALUE :"BLOCK_NUMBER" :: NUMBER AS block_number, ( VALUE :"BLOCK_DATE_UNIX" :: TIMESTAMP ) :: DATE AS block_date, - contract_address, - DATA :result :: STRING AS amount_hex, + DATA :result :: STRING AS result_hex, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address'] + ['block_number','contract_address', 'address', 'function_name', 'function_sig', 'input'] ) }} AS curated_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, @@ -46,6 +49,6 @@ WHERE DATA :result :: STRING <> '0x' {% endif %} - qualify(ROW_NUMBER() over (PARTITION BY block_number, contract_address + qualify(ROW_NUMBER() over (PARTITION BY curated_reads_id ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/curated_package/reads/silver/silver__curated_reads.yml b/models/curated_package/curated_reads/silver/silver__curated_reads.yml similarity index 100% rename from models/curated_package/reads/silver/silver__curated_reads.yml rename to models/curated_package/curated_reads/silver/silver__curated_reads.yml diff --git a/models/curated_package/reads/streamline/complete/streamline__curated_reads_complete.sql b/models/curated_package/curated_reads/streamline/complete/streamline__curated_reads_complete.sql similarity index 79% rename from models/curated_package/reads/streamline/complete/streamline__curated_reads_complete.sql rename to models/curated_package/curated_reads/streamline/complete/streamline__curated_reads_complete.sql index 38d90323..f6e4952d 100644 --- a/models/curated_package/reads/streamline/complete/streamline__curated_reads_complete.sql +++ b/models/curated_package/curated_reads/streamline/complete/streamline__curated_reads_complete.sql @@ -18,13 +18,17 @@ SELECT partition_key, contract_address, + VALUE :"ADDRESS" :: STRING AS address, VALUE :"BLOCK_NUMBER" :: NUMBER AS block_number, ( VALUE :"BLOCK_DATE_UNIX" :: TIMESTAMP ) :: DATE AS block_date, + VALUE :"FUNCTION_NAME" :: STRING AS function_name, + VALUE :"FUNCTION_SIG" :: STRING AS function_sig, + VALUE :"INPUT" :: STRING AS input, file_name, {{ dbt_utils.generate_surrogate_key( - ['contract_address', 'block_number'] + ['contract_address', 'address', 'block_number', 'function_name', 'function_sig', 'input'] ) }} AS curated_reads_complete_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime.sql b/models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime.sql new file mode 100644 index 00000000..119ca639 --- /dev/null +++ b/models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime.sql @@ -0,0 +1,39 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +{{ config ( + materialized = "view", + tags = ['streamline','curated_reads','realtime','phase_4'] +) }} + +SELECT + * +FROM + {{ ref('streamline__curated_reads_daily_realtime_requests') }} + +{# Streamline Function Call #} +{% if execute %} + {% set params = { + "external_table": 'curated_reads', + "sql_limit": vars.CURATED_SL_READS_DAILY_REALTIME_SQL_LIMIT, + "producer_batch_size": vars.CURATED_SL_READS_DAILY_REALTIME_PRODUCER_BATCH_SIZE, + "worker_batch_size": vars.CURATED_SL_READS_DAILY_REALTIME_WORKER_BATCH_SIZE, + "async_concurrent_requests": vars.CURATED_SL_READS_DAILY_REALTIME_ASYNC_CONCURRENT_REQUESTS, + "sql_source": 'curated_reads_daily_realtime' + } %} + + {% set function_call_sql %} + {{ fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = this.schema ~ "." ~ this.identifier, + params = params + ) }} + {% endset %} + + {% do run_query(function_call_sql) %} + {{ log("Streamline function call: " ~ function_call_sql, info=true) }} +{% endif %} \ No newline at end of file diff --git a/models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime_requests.sql b/models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime_requests.sql new file mode 100644 index 00000000..fb759bbb --- /dev/null +++ b/models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime_requests.sql @@ -0,0 +1,107 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +{{ config ( + materialized = "table", + tags = ['streamline','curated_reads','realtime','phase_4'] +) }} + +WITH last_x_days AS ( + + SELECT + block_number, + block_date + FROM + {{ ref("_max_block_by_date") }} + WHERE block_date >= DATEADD('day',-4,SYSDATE()) --last 3 max block_number by date +), +to_do AS ( + SELECT + DISTINCT + d.block_number, + d.block_date, + t.address, + t.contract_address, + t.function_name, + t.function_sig, + t.input + FROM + {{ ref("streamline__curated_reads_daily_records") }} t + INNER JOIN last_x_days d + ON t.block_date = d.block_date + --max daily block_number from 1 day ago, for each contract_address/address pair + WHERE + t.block_date IS NOT NULL + EXCEPT + SELECT + block_number, + block_date, + address, + contract_address, + function_name, + function_sig, + input + FROM + {{ ref("streamline__curated_reads_daily_complete") }} + WHERE + block_date >= ( + SELECT MIN(block_date) + FROM last_x_days + ) + AND block_date IS NOT NULL + AND _inserted_timestamp :: DATE >= DATEADD( + 'day', + -7, + SYSDATE() + ) +) +SELECT + block_number, + DATE_PART('EPOCH_SECONDS', block_date) :: INT AS block_date_unix, + address, + contract_address, + function_name, + function_sig, + input, + ROUND( + block_number, + -3 + ) AS partition_key, + OBJECT_CONSTRUCT( + 'data', OBJECT_CONSTRUCT( + 'id', CONCAT( + address, + '-', + contract_address, + '-', + input, + '-', + block_number + ), + 'jsonrpc', '2.0', + 'method', 'eth_call', + 'params', ARRAY_CONSTRUCT( + OBJECT_CONSTRUCT( + 'to', contract_address, + 'data', input + ), + utils.udf_int_to_hex(block_number) + ) + ), + 'headers', OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'x-fsc-livequery', 'true' + ), + 'method', 'POST', + 'secret_name', '{{ vars.GLOBAL_NODE_VAULT_PATH }}', + 'url', '{{ vars.GLOBAL_NODE_URL }}' + ) AS request +FROM + to_do +ORDER BY partition_key DESC, block_number DESC + +LIMIT {{ vars.CURATED_SL_READS_DAILY_REALTIME_SQL_LIMIT }} \ No newline at end of file diff --git a/models/curated_package/curated_reads/streamline/streamline__curated_reads_daily_records.sql b/models/curated_package/curated_reads/streamline/streamline__curated_reads_daily_records.sql new file mode 100644 index 00000000..e69de29b diff --git a/models/curated_package/reads/streamline/history/streamline__curated_reads_history.sql b/models/curated_package/reads/streamline/history/streamline__curated_reads_history.sql deleted file mode 100644 index 62c73367..00000000 --- a/models/curated_package/reads/streamline/history/streamline__curated_reads_history.sql +++ /dev/null @@ -1,120 +0,0 @@ -{# Get variables -{% set vars = return_vars() %} - -{# Log configuration details #} -{{ log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = "view", - tags = ['streamline','curated_reads','history','phase_4'] -) }} - -WITH max_blocks AS ( - SELECT - block_number, - block_date - FROM - {{ ref("_max_block_by_date") }} - WHERE - block_date >= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE -), -base AS ( - SELECT - c.contract_address, - m.block_number, - m.block_date - FROM - max_blocks m - LEFT JOIN {{ ref('streamline__curated_reads_complete') }} c - ON m.block_number = c.block_number - WHERE - c.contract_address IS NULL - AND m.block_date < ( - SELECT MAX(block_date) - FROM max_blocks - ) -), -function_sigs AS ( - SELECT - '0x18160ddd' AS function_sig, - 'totalSupply' AS function_name -), -ready_reads AS ( - SELECT - contract_address, - block_number, - block_date, - function_sig, - RPAD( - function_sig, - 64, - '0' - ) AS input - FROM - base - JOIN function_sigs - ON 1 = 1 -) -SELECT - contract_address, - block_number, - DATE_PART('EPOCH_SECONDS', block_date) :: INT AS block_date_unix, - ROUND(block_number,-3) AS partition_key, - function_sig, - input, - live.udf_api( - 'POST', - '{{ vars.GLOBAL_NODE_URL }}', - OBJECT_CONSTRUCT( - 'Content-Type', 'application/json', - 'fsc-quantum-state', 'streamline' - ), - OBJECT_CONSTRUCT( - 'method', 'eth_call', - 'jsonrpc', '2.0', - 'params', [{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(block_number)], - 'id', concat_ws( - '-', - contract_address, - input, - block_number - ) - ), - '{{ vars.GLOBAL_NODE_VAULT_PATH }}' - ) AS request -FROM - ready_reads -WHERE - EXISTS ( - SELECT - 1 - FROM - ready_reads - LIMIT - 1 - ) - -{# Streamline Function Call #} -{% if execute %} - {% set params = { - "external_table": 'stablecoin_reads', - "sql_limit": vars.CURATED_SL_STABLECOIN_READS_HISTORY_SQL_LIMIT, - "producer_batch_size": vars.CURATED_SL_STABLECOIN_READS_HISTORY_PRODUCER_BATCH_SIZE, - "worker_batch_size": vars.CURATED_SL_STABLECOIN_READS_HISTORY_WORKER_BATCH_SIZE, - "async_concurrent_requests": vars.CURATED_SL_STABLECOIN_READS_HISTORY_ASYNC_CONCURRENT_REQUESTS, - "sql_source": 'stablecoin_reads_history' - } %} - - {% set function_call_sql %} - {{ fsc_utils.if_data_call_function_v2( - func = 'streamline.udf_bulk_rest_api_v2', - target = this.schema ~ "." ~ this.identifier, - params = params - ) }} - {% endset %} - - {% do run_query(function_call_sql) %} - {{ log("Streamline function call: " ~ function_call_sql, info=true) }} -{% endif %} - #} diff --git a/models/curated_package/reads/streamline/realtime/streamline__curated_reads_realtime.sql b/models/curated_package/reads/streamline/realtime/streamline__curated_reads_realtime.sql deleted file mode 100644 index 19946d8b..00000000 --- a/models/curated_package/reads/streamline/realtime/streamline__curated_reads_realtime.sql +++ /dev/null @@ -1,127 +0,0 @@ -{# Get variables -{% set vars = return_vars() %} - -{# Log configuration details #} -{{ log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = "view", - tags = ['streamline','stablecoin_reads','realtime','phase_4'] -) }} - -WITH verified_stablecoins AS ( - SELECT - contract_address - FROM - {{ ref('defi__dim_stablecoins') }} - WHERE - is_verified - AND contract_address IS NOT NULL -), -max_blocks AS ( - SELECT - block_number, - block_date - FROM - {{ ref("_max_block_by_date") }} - WHERE - block_date >= DATEADD('day',-4,SYSDATE()) --last 3 max block_number by date -), -base AS ( - SELECT - s.contract_address, - m.block_number, - m.block_date - FROM - verified_stablecoins s - CROSS JOIN max_blocks m - LEFT JOIN {{ ref('streamline__stablecoin_reads_complete') }} c - ON s.contract_address = c.contract_address - AND m.block_number = c.block_number - WHERE - c.contract_address IS NULL -), -function_sigs AS ( - SELECT - '0x18160ddd' AS function_sig, - 'totalSupply' AS function_name -), -ready_reads AS ( - SELECT - contract_address, - block_number, - block_date, - function_sig, - RPAD( - function_sig, - 64, - '0' - ) AS input - FROM - base - JOIN function_sigs - ON 1 = 1 -) -SELECT - contract_address, - block_number, - DATE_PART('EPOCH_SECONDS', block_date) :: INT AS block_date_unix, - ROUND(block_number,-3) AS partition_key, - function_sig, - input, - live.udf_api( - 'POST', - '{{ vars.GLOBAL_NODE_URL }}', - OBJECT_CONSTRUCT( - 'Content-Type', 'application/json', - 'fsc-quantum-state', 'streamline' - ), - OBJECT_CONSTRUCT( - 'method', 'eth_call', - 'jsonrpc', '2.0', - 'params', [{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(block_number)], - 'id', concat_ws( - '-', - contract_address, - input, - block_number - ) - ), - '{{ vars.GLOBAL_NODE_VAULT_PATH }}' - ) AS request -FROM - ready_reads -WHERE - EXISTS ( - SELECT - 1 - FROM - ready_reads - LIMIT - 1 - ) - -{# Streamline Function Call #} -{% if execute %} - {% set params = { - "external_table" :"stablecoin_reads", - "sql_limit" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_SQL_LIMIT, - "producer_batch_size" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_PRODUCER_BATCH_SIZE, - "worker_batch_size" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_WORKER_BATCH_SIZE, - "async_concurrent_requests" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_ASYNC_CONCURRENT_REQUESTS, - "sql_source" : 'stablecoin_reads_realtime' - } %} - - {% set function_call_sql %} - {{ fsc_utils.if_data_call_function_v2( - func = 'streamline.udf_bulk_rest_api_v2', - target = this.schema ~ "." ~ this.identifier, - params = params - ) }} - {% endset %} - - {% do run_query(function_call_sql) %} - {{ log("Streamline function call: " ~ function_call_sql, info=true) }} -{% endif %} - #} From 944cf93a062083457f2d63604a78de50c8550703 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 17 Nov 2025 14:21:51 -0700 Subject: [PATCH 003/164] naming --- ...ated_reads.sql => bronze__contract_reads.sql} | 4 ++-- ...eads_fr.sql => bronze__contract_reads_fr.sql} | 4 ++-- ...ated_reads.sql => silver__contract_reads.sql} | 14 +++++++------- ...ated_reads.yml => silver__contract_reads.yml} | 4 ++-- ...l => streamline__contract_reads_complete.sql} | 14 +++++++------- ...treamline__contract_reads_daily_realtime.sql} | 16 ++++++++-------- ...__contract_reads_daily_realtime_requests.sql} | 8 ++++---- ...streamline__contract_reads_daily_records.sql} | 0 8 files changed, 32 insertions(+), 32 deletions(-) rename models/curated_package/curated_reads/bronze/{bronze__curated_reads.sql => bronze__contract_reads.sql} (75%) rename models/curated_package/curated_reads/bronze/{bronze__curated_reads_fr.sql => bronze__contract_reads_fr.sql} (76%) rename models/curated_package/curated_reads/silver/{silver__curated_reads.sql => silver__contract_reads.sql} (81%) rename models/curated_package/curated_reads/silver/{silver__curated_reads.yml => silver__contract_reads.yml} (63%) rename models/curated_package/curated_reads/streamline/complete/{streamline__curated_reads_complete.sql => streamline__contract_reads_complete.sql} (75%) rename models/curated_package/curated_reads/streamline/realtime/{streamline__curated_reads_daily_realtime.sql => streamline__contract_reads_daily_realtime.sql} (50%) rename models/curated_package/curated_reads/streamline/realtime/{streamline__curated_reads_daily_realtime_requests.sql => streamline__contract_reads_daily_realtime_requests.sql} (90%) rename models/curated_package/curated_reads/streamline/{streamline__curated_reads_daily_records.sql => streamline__contract_reads_daily_records.sql} (100%) diff --git a/models/curated_package/curated_reads/bronze/bronze__curated_reads.sql b/models/curated_package/curated_reads/bronze/bronze__contract_reads.sql similarity index 75% rename from models/curated_package/curated_reads/bronze/bronze__curated_reads.sql rename to models/curated_package/curated_reads/bronze/bronze__contract_reads.sql index dacfdce9..502e14be 100644 --- a/models/curated_package/curated_reads/bronze/bronze__curated_reads.sql +++ b/models/curated_package/curated_reads/bronze/bronze__contract_reads.sql @@ -4,12 +4,12 @@ {# Set up dbt configuration #} {{ config ( materialized = 'view', - tags = ['bronze','curated_reads','phase_4'] + tags = ['bronze','contract_reads','phase_4'] ) }} {# Main query starts here #} {{ streamline_external_table_query( - source_name = 'curated_reads', + source_name = 'contract_reads', block_number = false, contract_address = true, data_not_null = false diff --git a/models/curated_package/curated_reads/bronze/bronze__curated_reads_fr.sql b/models/curated_package/curated_reads/bronze/bronze__contract_reads_fr.sql similarity index 76% rename from models/curated_package/curated_reads/bronze/bronze__curated_reads_fr.sql rename to models/curated_package/curated_reads/bronze/bronze__contract_reads_fr.sql index 1bc1c4b1..e58798b5 100644 --- a/models/curated_package/curated_reads/bronze/bronze__curated_reads_fr.sql +++ b/models/curated_package/curated_reads/bronze/bronze__contract_reads_fr.sql @@ -4,12 +4,12 @@ {# Set up dbt configuration #} {{ config ( materialized = 'view', - tags = ['bronze','curated_reads','phase_4'] + tags = ['bronze','contract_reads','phase_4'] ) }} {# Main query starts here #} {{ streamline_external_table_query_fr( - source_name = 'curated_reads', + source_name = 'contract_reads', block_number = false, contract_address = true, data_not_null = false diff --git a/models/curated_package/curated_reads/silver/silver__curated_reads.sql b/models/curated_package/curated_reads/silver/silver__contract_reads.sql similarity index 81% rename from models/curated_package/curated_reads/silver/silver__curated_reads.sql rename to models/curated_package/curated_reads/silver/silver__contract_reads.sql index 3aee60a7..c0f09836 100644 --- a/models/curated_package/curated_reads/silver/silver__curated_reads.sql +++ b/models/curated_package/curated_reads/silver/silver__contract_reads.sql @@ -4,15 +4,15 @@ {# Log configuration details #} {{ log_model_details() }} --- depends_on: {{ ref('bronze__curated_reads') }} +-- depends_on: {{ ref('bronze__contract_reads') }} {{ config( materialized = 'incremental', - unique_key = 'curated_reads_id', + unique_key = 'contract_reads_id', cluster_by = ['block_date'], incremental_predicates = ["dynamic_range", "block_number"], merge_exclude_columns = ["inserted_timestamp"], full_refresh = vars.GLOBAL_SILVER_FR_ENABLED, - tags = ['silver','curated_reads','curated_daily'] + tags = ['silver','contract_reads','curated_daily'] ) }} SELECT @@ -28,14 +28,14 @@ SELECT _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['block_number','contract_address', 'address', 'function_name', 'function_sig', 'input'] - ) }} AS curated_reads_id, + ) }} AS contract_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM {% if is_incremental() %} -{{ ref('bronze__curated_reads') }} +{{ ref('bronze__contract_reads') }} WHERE _inserted_timestamp >= ( SELECT @@ -44,11 +44,11 @@ WHERE {{ this }}) AND DATA :result :: STRING <> '0x' {% else %} - {{ ref('bronze__curated_reads_fr') }} + {{ ref('bronze__contract_reads_fr') }} WHERE DATA :result :: STRING <> '0x' {% endif %} - qualify(ROW_NUMBER() over (PARTITION BY curated_reads_id + qualify(ROW_NUMBER() over (PARTITION BY contract_reads_id ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/curated_package/curated_reads/silver/silver__curated_reads.yml b/models/curated_package/curated_reads/silver/silver__contract_reads.yml similarity index 63% rename from models/curated_package/curated_reads/silver/silver__curated_reads.yml rename to models/curated_package/curated_reads/silver/silver__contract_reads.yml index eb6583c1..1446c63d 100644 --- a/models/curated_package/curated_reads/silver/silver__curated_reads.yml +++ b/models/curated_package/curated_reads/silver/silver__contract_reads.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: silver__curated_reads + - name: silver__contract_reads tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - curated_reads_id \ No newline at end of file + - contract_reads_id \ No newline at end of file diff --git a/models/curated_package/curated_reads/streamline/complete/streamline__curated_reads_complete.sql b/models/curated_package/curated_reads/streamline/complete/streamline__contract_reads_complete.sql similarity index 75% rename from models/curated_package/curated_reads/streamline/complete/streamline__curated_reads_complete.sql rename to models/curated_package/curated_reads/streamline/complete/streamline__contract_reads_complete.sql index f6e4952d..c08fb96f 100644 --- a/models/curated_package/curated_reads/streamline/complete/streamline__curated_reads_complete.sql +++ b/models/curated_package/curated_reads/streamline/complete/streamline__contract_reads_complete.sql @@ -4,15 +4,15 @@ {# Log configuration details #} {{ log_model_details() }} --- depends on: {{ ref('bronze__curated_reads') }} +-- depends on: {{ ref('bronze__contract_reads') }} {# Set up dbt configuration #} {{ config ( materialized = 'incremental', - unique_key = 'curated_reads_complete_id', + unique_key = 'contract_reads_complete_id', incremental_predicates = ['dynamic_range', 'partition_key'], full_refresh = vars.GLOBAL_STREAMLINE_FR_ENABLED, - tags = ['streamline','curated_reads','complete','phase_4'] + tags = ['streamline','contract_reads','complete','phase_4'] ) }} SELECT @@ -29,7 +29,7 @@ SELECT file_name, {{ dbt_utils.generate_surrogate_key( ['contract_address', 'address', 'block_number', 'function_name', 'function_sig', 'input'] - ) }} AS curated_reads_complete_id, + ) }} AS contract_reads_complete_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, _inserted_timestamp, @@ -37,7 +37,7 @@ SELECT FROM {% if is_incremental() %} -{{ ref('bronze__curated_reads') }} +{{ ref('bronze__contract_reads') }} WHERE _inserted_timestamp >= ( SELECT @@ -45,9 +45,9 @@ WHERE FROM {{ this }}) {% else %} - {{ ref('bronze__curated_reads_fr') }} + {{ ref('bronze__contract_reads_fr') }} {% endif %} -qualify(ROW_NUMBER() over (PARTITION BY curated_reads_complete_id +qualify(ROW_NUMBER() over (PARTITION BY contract_reads_complete_id ORDER BY _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime.sql b/models/curated_package/curated_reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql similarity index 50% rename from models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime.sql rename to models/curated_package/curated_reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql index 119ca639..cc0e289a 100644 --- a/models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime.sql +++ b/models/curated_package/curated_reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql @@ -7,23 +7,23 @@ {# Set up dbt configuration #} {{ config ( materialized = "view", - tags = ['streamline','curated_reads','realtime','phase_4'] + tags = ['streamline','contract_reads','realtime','phase_4'] ) }} SELECT * FROM - {{ ref('streamline__curated_reads_daily_realtime_requests') }} + {{ ref('streamline__contract_reads_daily_realtime_requests') }} {# Streamline Function Call #} {% if execute %} {% set params = { - "external_table": 'curated_reads', - "sql_limit": vars.CURATED_SL_READS_DAILY_REALTIME_SQL_LIMIT, - "producer_batch_size": vars.CURATED_SL_READS_DAILY_REALTIME_PRODUCER_BATCH_SIZE, - "worker_batch_size": vars.CURATED_SL_READS_DAILY_REALTIME_WORKER_BATCH_SIZE, - "async_concurrent_requests": vars.CURATED_SL_READS_DAILY_REALTIME_ASYNC_CONCURRENT_REQUESTS, - "sql_source": 'curated_reads_daily_realtime' + "external_table": 'contract_reads', + "sql_limit": vars.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_SQL_LIMIT, + "producer_batch_size": vars.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_PRODUCER_BATCH_SIZE, + "worker_batch_size": vars.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_WORKER_BATCH_SIZE, + "async_concurrent_requests": vars.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_ASYNC_CONCURRENT_REQUESTS, + "sql_source": 'contract_reads_daily_realtime' } %} {% set function_call_sql %} diff --git a/models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime_requests.sql b/models/curated_package/curated_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql similarity index 90% rename from models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime_requests.sql rename to models/curated_package/curated_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index fb759bbb..ff797b9e 100644 --- a/models/curated_package/curated_reads/streamline/realtime/streamline__curated_reads_daily_realtime_requests.sql +++ b/models/curated_package/curated_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -7,7 +7,7 @@ {# Set up dbt configuration #} {{ config ( materialized = "table", - tags = ['streamline','curated_reads','realtime','phase_4'] + tags = ['streamline','contract_reads','realtime','phase_4'] ) }} WITH last_x_days AS ( @@ -30,7 +30,7 @@ to_do AS ( t.function_sig, t.input FROM - {{ ref("streamline__curated_reads_daily_records") }} t + {{ ref("streamline__contract_reads_daily_records") }} t INNER JOIN last_x_days d ON t.block_date = d.block_date --max daily block_number from 1 day ago, for each contract_address/address pair @@ -46,7 +46,7 @@ to_do AS ( function_sig, input FROM - {{ ref("streamline__curated_reads_daily_complete") }} + {{ ref("streamline__contract_reads_daily_complete") }} WHERE block_date >= ( SELECT MIN(block_date) @@ -104,4 +104,4 @@ FROM to_do ORDER BY partition_key DESC, block_number DESC -LIMIT {{ vars.CURATED_SL_READS_DAILY_REALTIME_SQL_LIMIT }} \ No newline at end of file +LIMIT {{ vars.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_SQL_LIMIT }} \ No newline at end of file diff --git a/models/curated_package/curated_reads/streamline/streamline__curated_reads_daily_records.sql b/models/curated_package/curated_reads/streamline/streamline__contract_reads_daily_records.sql similarity index 100% rename from models/curated_package/curated_reads/streamline/streamline__curated_reads_daily_records.sql rename to models/curated_package/curated_reads/streamline/streamline__contract_reads_daily_records.sql From 306bb9c5ed5259c0669ef27e8badfcc94b134fa9 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 17 Nov 2025 14:22:46 -0700 Subject: [PATCH 004/164] folder --- .../bronze/bronze__contract_reads.sql | 0 .../bronze/bronze__contract_reads_fr.sql | 0 .../silver/silver__contract_reads.sql | 0 .../silver/silver__contract_reads.yml | 0 .../streamline/complete/streamline__contract_reads_complete.sql | 0 .../realtime/streamline__contract_reads_daily_realtime.sql | 0 .../streamline__contract_reads_daily_realtime_requests.sql | 0 .../streamline/streamline__contract_reads_daily_records.sql | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename models/curated_package/{curated_reads => contract_reads}/bronze/bronze__contract_reads.sql (100%) rename models/curated_package/{curated_reads => contract_reads}/bronze/bronze__contract_reads_fr.sql (100%) rename models/curated_package/{curated_reads => contract_reads}/silver/silver__contract_reads.sql (100%) rename models/curated_package/{curated_reads => contract_reads}/silver/silver__contract_reads.yml (100%) rename models/curated_package/{curated_reads => contract_reads}/streamline/complete/streamline__contract_reads_complete.sql (100%) rename models/curated_package/{curated_reads => contract_reads}/streamline/realtime/streamline__contract_reads_daily_realtime.sql (100%) rename models/curated_package/{curated_reads => contract_reads}/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql (100%) rename models/curated_package/{curated_reads => contract_reads}/streamline/streamline__contract_reads_daily_records.sql (100%) diff --git a/models/curated_package/curated_reads/bronze/bronze__contract_reads.sql b/models/curated_package/contract_reads/bronze/bronze__contract_reads.sql similarity index 100% rename from models/curated_package/curated_reads/bronze/bronze__contract_reads.sql rename to models/curated_package/contract_reads/bronze/bronze__contract_reads.sql diff --git a/models/curated_package/curated_reads/bronze/bronze__contract_reads_fr.sql b/models/curated_package/contract_reads/bronze/bronze__contract_reads_fr.sql similarity index 100% rename from models/curated_package/curated_reads/bronze/bronze__contract_reads_fr.sql rename to models/curated_package/contract_reads/bronze/bronze__contract_reads_fr.sql diff --git a/models/curated_package/curated_reads/silver/silver__contract_reads.sql b/models/curated_package/contract_reads/silver/silver__contract_reads.sql similarity index 100% rename from models/curated_package/curated_reads/silver/silver__contract_reads.sql rename to models/curated_package/contract_reads/silver/silver__contract_reads.sql diff --git a/models/curated_package/curated_reads/silver/silver__contract_reads.yml b/models/curated_package/contract_reads/silver/silver__contract_reads.yml similarity index 100% rename from models/curated_package/curated_reads/silver/silver__contract_reads.yml rename to models/curated_package/contract_reads/silver/silver__contract_reads.yml diff --git a/models/curated_package/curated_reads/streamline/complete/streamline__contract_reads_complete.sql b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql similarity index 100% rename from models/curated_package/curated_reads/streamline/complete/streamline__contract_reads_complete.sql rename to models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql diff --git a/models/curated_package/curated_reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql similarity index 100% rename from models/curated_package/curated_reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql rename to models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql diff --git a/models/curated_package/curated_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql similarity index 100% rename from models/curated_package/curated_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql rename to models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql diff --git a/models/curated_package/curated_reads/streamline/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/streamline__contract_reads_daily_records.sql similarity index 100% rename from models/curated_package/curated_reads/streamline/streamline__contract_reads_daily_records.sql rename to models/curated_package/contract_reads/streamline/streamline__contract_reads_daily_records.sql From 193acf191d2a08544516d111a95656d8714d601a Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 17 Nov 2025 14:33:58 -0700 Subject: [PATCH 005/164] lido --- .../records/lido/silver_reads__lido_tvl.sql | 50 ++++++++++++++++ ...reamline__contract_reads_daily_records.sql | 58 +++++++++++++++++++ ...reamline__contract_reads_daily_records.sql | 0 3 files changed, 108 insertions(+) create mode 100644 models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql create mode 100644 models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql delete mode 100644 models/curated_package/contract_reads/streamline/streamline__contract_reads_daily_records.sql diff --git a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql new file mode 100644 index 00000000..b47747b5 --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql @@ -0,0 +1,50 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'lido_tvl_id', + tags = ['silver','contract_reads','curated_daily'] +) }} + +WITH pooled_assets AS ( +SELECT + '0xae7ab96520de3a18e5e111b5eaab095312d7fe84' AS contract_address, + NULL AS address, + 'getTotalPooledEther' AS function_name, + '0x37cfdaca' AS function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input +UNION ALL +SELECT + '0x9ee91f9f426fa633d227f7a9b000e28b9dfd8599' AS contract_address, + NULL AS address, + 'getTotalPooledMatic' AS function_name, + '0xe00222a0' AS function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input +) + +SELECT + contract_address, + address, + function_name, + function_sig, + input, + {{ dbt_utils.generate_surrogate_key( + ['contract_address', 'address', 'function_name', 'function_sig', 'input'] + ) }} AS lido_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM pooled_assets diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql new file mode 100644 index 00000000..b0d20cbc --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql @@ -0,0 +1,58 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +{{ config ( + materialized = "incremental", + unique_key = "contract_reads_daily_records_id", + cluster_by = "block_date", + full_refresh = vars.GLOBAL_STREAMLINE_FR_ENABLED, + tags = ['streamline','contract_reads','records','phase_4'] +) }} + +WITH lido_tvl AS ( + + SELECT + contract_address, + address, + function_name, + function_sig, + input + FROM + {{ ref('silver_reads__lido_tvl') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} +), +all_records AS ( + SELECT + * + FROM + lido_tvl +) +SELECT + contract_address, + address, + function_name, + function_sig, + input, + {{ dbt_utils.generate_surrogate_key( + ['contract_address', 'address', 'function_name', 'function_sig', 'input'] + ) }} AS contract_reads_daily_records_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + all_records qualify (ROW_NUMBER() over (PARTITION BY contract_reads_daily_records_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/contract_reads/streamline/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/streamline__contract_reads_daily_records.sql deleted file mode 100644 index e69de29b..00000000 From f6ffdbab13dce550a6368300e163af8b858cb889 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 17 Nov 2025 15:10:08 -0700 Subject: [PATCH 006/164] uni v2 --- .../silver/silver__contract_reads.sql | 4 +- .../streamline__contract_reads_complete.sql | 12 +++-- ...contract_reads_daily_realtime_requests.sql | 22 +++++--- .../records/lido/silver_reads__lido_tvl.sql | 10 ++-- ...reamline__contract_reads_daily_records.sql | 36 ++++++++++++- .../uniswap/silver_reads__uniswap_v2_tvl.sql | 54 +++++++++++++++++++ 6 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_tvl.sql diff --git a/models/curated_package/contract_reads/silver/silver__contract_reads.sql b/models/curated_package/contract_reads/silver/silver__contract_reads.sql index c0f09836..cf789d22 100644 --- a/models/curated_package/contract_reads/silver/silver__contract_reads.sql +++ b/models/curated_package/contract_reads/silver/silver__contract_reads.sql @@ -25,9 +25,11 @@ SELECT VALUE :"BLOCK_DATE_UNIX" :: TIMESTAMP ) :: DATE AS block_date, DATA :result :: STRING AS result_hex, + (VALUE :"METADATA" :: STRING) :: VARIANT AS metadata, + VALUE :"PLATFORM" :: STRING AS platform, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( - ['block_number','contract_address', 'address', 'function_name', 'function_sig', 'input'] + ['block_number','contract_address', 'address', 'input', 'platform'] ) }} AS contract_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql index c08fb96f..a511a073 100644 --- a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql +++ b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql @@ -26,9 +26,13 @@ SELECT VALUE :"FUNCTION_NAME" :: STRING AS function_name, VALUE :"FUNCTION_SIG" :: STRING AS function_sig, VALUE :"INPUT" :: STRING AS input, + ( + VALUE :"METADATA" :: STRING + ) :: variant AS metadata, + VALUE :"PLATFORM" :: STRING AS platform, file_name, {{ dbt_utils.generate_surrogate_key( - ['contract_address', 'address', 'block_number', 'function_name', 'function_sig', 'input'] + ['contract_address', 'address', 'block_number', 'function_name', 'function_sig', 'input', 'platform'] ) }} AS contract_reads_complete_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, @@ -48,6 +52,6 @@ WHERE {{ ref('bronze__contract_reads_fr') }} {% endif %} -qualify(ROW_NUMBER() over (PARTITION BY contract_reads_complete_id -ORDER BY - _inserted_timestamp DESC)) = 1 \ No newline at end of file + qualify(ROW_NUMBER() over (PARTITION BY contract_reads_complete_id + ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index ff797b9e..bb6464a8 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -24,11 +24,13 @@ to_do AS ( DISTINCT d.block_number, d.block_date, - t.address, t.contract_address, + t.address, t.function_name, t.function_sig, - t.input + t.input, + t.metadata, + t.platform FROM {{ ref("streamline__contract_reads_daily_records") }} t INNER JOIN last_x_days d @@ -40,11 +42,13 @@ to_do AS ( SELECT block_number, block_date, - address, contract_address, + address, function_name, function_sig, - input + input, + metadata, + platform FROM {{ ref("streamline__contract_reads_daily_complete") }} WHERE @@ -62,11 +66,13 @@ to_do AS ( SELECT block_number, DATE_PART('EPOCH_SECONDS', block_date) :: INT AS block_date_unix, - address, contract_address, + address, function_name, function_sig, input, + metadata, + platform, ROUND( block_number, -3 @@ -74,13 +80,13 @@ SELECT OBJECT_CONSTRUCT( 'data', OBJECT_CONSTRUCT( 'id', CONCAT( - address, - '-', contract_address, '-', + address, + '-', input, '-', - block_number + block_number, ), 'jsonrpc', '2.0', 'method', 'eth_call', diff --git a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql index b47747b5..14ee32a2 100644 --- a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql @@ -14,7 +14,6 @@ WITH pooled_assets AS ( SELECT '0xae7ab96520de3a18e5e111b5eaab095312d7fe84' AS contract_address, - NULL AS address, 'getTotalPooledEther' AS function_name, '0x37cfdaca' AS function_sig, RPAD( @@ -25,7 +24,6 @@ SELECT UNION ALL SELECT '0x9ee91f9f426fa633d227f7a9b000e28b9dfd8599' AS contract_address, - NULL AS address, 'getTotalPooledMatic' AS function_name, '0xe00222a0' AS function_sig, RPAD( @@ -37,12 +35,16 @@ SELECT SELECT contract_address, - address, + NULL AS address, function_name, function_sig, input, + NULL AS metadata, + 'lido' AS protocol, + 'v1' AS version, + CONCAT(protocol, '-', version) AS platform, {{ dbt_utils.generate_surrogate_key( - ['contract_address', 'address', 'function_name', 'function_sig', 'input'] + ['contract_address','address','input','platform'] ) }} AS lido_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql index b0d20cbc..e9dd15bc 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql @@ -20,7 +20,11 @@ WITH lido_tvl AS ( address, function_name, function_sig, - input + input, + metadata, + protocol, + version, + platform FROM {{ ref('silver_reads__lido_tvl') }} @@ -34,11 +38,35 @@ WHERE ) {% endif %} ), +uniswap_v2_tvl AS ( + SELECT + contract_address, + address, + function_name, + function_sig, + input, + metadata, + protocol, + version, + platform + FROM {{ ref('silver_reads__uniswap_v2_tvl') }} + {% if is_incremental() %} + WHERE modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} +) all_records AS ( SELECT * FROM lido_tvl + UNION ALL + SELECT + * + FROM + uniswap_v2_tvl ) SELECT contract_address, @@ -46,8 +74,12 @@ SELECT function_name, function_sig, input, + metadata, + protocol, + version, + platform, {{ dbt_utils.generate_surrogate_key( - ['contract_address', 'address', 'function_name', 'function_sig', 'input'] + ['contract_address','address','input','platform'] ) }} AS contract_reads_daily_records_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_tvl.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_tvl.sql new file mode 100644 index 00000000..6dbc13e7 --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_tvl.sql @@ -0,0 +1,54 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'lido_tvl_id', + tags = ['silver','contract_reads','curated_daily'] +) }} + +WITH liquidity_pools AS ( + SELECT + DISTINCT + pool_address AS contract_address, + token0, + token1, + protocol, + version, + platform + FROM {{ ref('silver_dex__paircreated_evt_v2_pools') }} + {% if is_incremental() %} + WHERE modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} +) +SELECT + contract_address, + NULL AS address, + 'getReserves' AS function_name, + '0x0902f1ac' AS function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input, + OBJECT_CONSTRUCT( + 'token0', token0, + 'token1', token1 + ) AS metadata, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','address','input','platform'] + ) }} AS uniswap_v2_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM liquidity_pools From 658f8d9ed449e780794552acf4372440ef74ea44 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 17 Nov 2025 16:13:54 -0700 Subject: [PATCH 007/164] uni v3 --- ...reamline__contract_reads_daily_records.sql | 24 ++++++ .../uniswap/silver_reads__uniswap_v3_tvl.sql | 76 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_tvl.sql diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql index e9dd15bc..7efb0131 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql @@ -57,6 +57,25 @@ uniswap_v2_tvl AS ( ) {% endif %} ) +uniswap_v3_tvl AS ( + SELECT + contract_address, + address, + function_name, + function_sig, + input, + metadata, + protocol, + version, + platform + FROM {{ ref('silver_reads__uniswap_v3_tvl') }} + {% if is_incremental() %} + WHERE modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} +) all_records AS ( SELECT * @@ -67,6 +86,11 @@ all_records AS ( * FROM uniswap_v2_tvl + UNION ALL + SELECT + * + FROM + uniswap_v3_tvl ) SELECT contract_address, diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_tvl.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_tvl.sql new file mode 100644 index 00000000..5c9d5e6d --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_tvl.sql @@ -0,0 +1,76 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'uniswap_v3_tvl_id', + tags = ['silver','contract_reads','curated_daily'] +) }} + +WITH liquidity_pools AS ( + SELECT + DISTINCT + pool_address, + token0, + token1, + protocol, + version, + platform + FROM {{ ref('silver_dex__poolcreated_evt_v3_pools') }} + {% if is_incremental() %} + WHERE modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} +), + +lp_balances AS ( + SELECT + token0 AS contract_address, + pool_address AS address, + token0, + token1, + protocol, + version, + platform + FROM liquidity_pools + UNION + SELECT + token1 AS contract_address, + pool_address AS address, + token0, + token1, + protocol, + version, + platform + FROM liquidity_pools +) + +SELECT + contract_address, + address, + 'balanceOf' AS function_name, + '0x70a08231' AS function_sig, + CONCAT( + '0x70a08231', + LPAD(SUBSTR(address, 3), 64, '0') + ) AS input, + OBJECT_CONSTRUCT( + 'token0', token0, + 'token1', token1 + ) AS metadata, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','address','input','platform'] + ) }} AS uniswap_v3_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM lp_balances From 2db47031eb3f624e911ba734f2229599f1120703 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 17 Nov 2025 16:55:57 -0700 Subject: [PATCH 008/164] aave v1 --- .../aave/silver_reads__aave_v1_tvl.sql | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql new file mode 100644 index 00000000..a0a4443f --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql @@ -0,0 +1,111 @@ +{# Get variables #} +{% set vars = return_vars() %} +{# Log configuration details #} +{{ log_model_details() }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'uniswap_v3_tvl_id', + tags = ['silver','contract_reads','curated_daily'] +) }} + +WITH all_tokens AS ( + + SELECT + underlying_address AS contract_address, + protocol, + version, + CONCAT( + protocol, + '-', + version + ) AS platform + FROM + {{ ref('silver_lending__aave_ethereum_tokens') }} --relevant for ethereum only + WHERE + protocol = 'aave' + AND version = 'v1' + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +UNION +SELECT + underlying_address AS contract_address, + protocol, + version, + CONCAT( + protocol, + '-', + version + ) AS platform +FROM + {{ ref('silver_lending__aave_tokens') }} +WHERE + protocol = 'aave' + AND version = 'v1' + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +lending_pools AS ( + SELECT + contract_address, + '0x3dfd23a6c5e8bbcfc9581d2e864a68feb6a076d3' AS address, + --Aave: LendingPoolCore + function_name, + function_sig, + input, + metadata, + protocol, + version, + platform + FROM + all_tokens + UNION ALL + SELECT + contract_address, + '0x1012cff81a1582ddd0616517efb97d02c5c17e25' AS address, + --Uniswap: LendingPoolCore in Aave v1 holds Uniswap v1 LP tokens as collateral + function_name, + function_sig, + input, + metadata, + protocol, + version, + platform + FROM + all_tokens +) +SELECT + contract_address, + address, + 'balanceOf' AS function_name, + '0x70a08231' AS function_sig, + CONCAT( + '0x70a08231', + LPAD(SUBSTR(address, 3), 64, '0') + ) AS input, + NULL AS metadata, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','address','input','platform'] + ) }} AS uniswap_v3_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + lp_balances From f35191bc8b6f192f578d522bdbf54dbb255e28b8 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 17 Nov 2025 17:30:05 -0700 Subject: [PATCH 009/164] eth logic --- .../records/aave/silver_reads__aave_v1_tvl.sql | 16 ++++++++++++---- .../records/lido/silver_reads__lido_tvl.sql | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql index a0a4443f..4e6c13f7 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql @@ -21,7 +21,7 @@ WITH all_tokens AS ( version ) AS platform FROM - {{ ref('silver_lending__aave_ethereum_tokens') }} --relevant for ethereum only + {{ ref('silver_lending__aave_tokens') }} WHERE protocol = 'aave' AND version = 'v1' @@ -34,6 +34,8 @@ AND modified_timestamp > ( {{ this }} ) {% endif %} + +{% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} UNION SELECT underlying_address AS contract_address, @@ -45,7 +47,8 @@ SELECT version ) AS platform FROM - {{ ref('silver_lending__aave_tokens') }} + {{ ref('silver_lending__aave_ethereum_tokens') }} + --relevant for ethereum only WHERE protocol = 'aave' AND version = 'v1' @@ -58,11 +61,14 @@ AND modified_timestamp > ( {{ this }} ) {% endif %} +{% endif %} ), lending_pools AS ( SELECT contract_address, - '0x3dfd23a6c5e8bbcfc9581d2e864a68feb6a076d3' AS address, + CASE + WHEN '{{ vars.GLOBAL_PROJECT_NAME }}' = 'ethereum' THEN '0x3dfd23a6c5e8bbcfc9581d2e864a68feb6a076d3' + END AS address, --Aave: LendingPoolCore function_name, function_sig, @@ -76,7 +82,9 @@ lending_pools AS ( UNION ALL SELECT contract_address, - '0x1012cff81a1582ddd0616517efb97d02c5c17e25' AS address, + CASE + WHEN '{{ vars.GLOBAL_PROJECT_NAME }}' = 'ethereum' THEN '0x1012cff81a1582ddd0616517efb97d02c5c17e25' + END AS address, --Uniswap: LendingPoolCore in Aave v1 holds Uniswap v1 LP tokens as collateral function_name, function_sig, diff --git a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql index 14ee32a2..0916fc82 100644 --- a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql @@ -50,3 +50,4 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM pooled_assets +--need to add logic to bypass model / not read if not on ethereum \ No newline at end of file From b4cb1e9ccf72427188927209cefa69b1ab895e7c Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 18 Nov 2025 09:01:39 -0700 Subject: [PATCH 010/164] aave v2 --- .../aave/silver_reads__aave_v1_tvl.sql | 4 +- .../aave/silver_reads__aave_v2_tvl.sql | 87 +++++++++++++++++++ .../records/lido/silver_reads__lido_tvl.sql | 2 +- .../uniswap/silver_reads__uniswap_v2_tvl.sql | 2 +- .../uniswap/silver_reads__uniswap_v3_tvl.sql | 2 +- 5 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_tvl.sql diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql index 4e6c13f7..6c01dfbb 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql @@ -5,7 +5,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - unique_key = 'uniswap_v3_tvl_id', + unique_key = 'aave_v1_tvl_id', tags = ['silver','contract_reads','curated_daily'] ) }} @@ -111,7 +111,7 @@ SELECT platform, {{ dbt_utils.generate_surrogate_key( ['contract_address','address','input','platform'] - ) }} AS uniswap_v3_tvl_id, + ) }} AS aave_v1_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_tvl.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_tvl.sql new file mode 100644 index 00000000..23b104ac --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_tvl.sql @@ -0,0 +1,87 @@ +{# Get variables #} +{% set vars = return_vars() %} +{# Log configuration details #} +{{ log_model_details() }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'aave_v2_tvl_id', + tags = ['silver','contract_reads','curated_daily'] +) }} + +WITH all_tokens AS ( + + SELECT + atoken_address AS contract_address, + protocol, + version, + CONCAT( + protocol, + '-', + version + ) AS platform + FROM + {{ ref('silver_lending__aave_tokens') }} + WHERE + protocol = 'aave' + AND version = 'v2' + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} + +{% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} +UNION +SELECT + atoken_address AS contract_address, + protocol, + version, + CONCAT( + protocol, + '-', + version + ) AS platform +FROM + {{ ref('silver_lending__aave_ethereum_tokens') }} + --relevant for ethereum only +WHERE + protocol = 'aave' + AND version IN ('v2','v2.1') + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +{% endif %} +) +SELECT + contract_address, + NULL AS address, + 'totalSupply' AS function_name, + '0x18160ddd' AS function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input + NULL AS metadata, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','input','platform'] + ) }} AS aave_v2_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + lp_balances diff --git a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql index 0916fc82..b4c839b7 100644 --- a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql @@ -44,7 +44,7 @@ SELECT 'v1' AS version, CONCAT(protocol, '-', version) AS platform, {{ dbt_utils.generate_surrogate_key( - ['contract_address','address','input','platform'] + ['contract_address','input','platform'] ) }} AS lido_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_tvl.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_tvl.sql index 6dbc13e7..72269290 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_tvl.sql @@ -46,7 +46,7 @@ SELECT version, platform, {{ dbt_utils.generate_surrogate_key( - ['contract_address','address','input','platform'] + ['contract_address','input','platform'] ) }} AS uniswap_v2_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_tvl.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_tvl.sql index 5c9d5e6d..dfcf8b1a 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_tvl.sql @@ -68,7 +68,7 @@ SELECT version, platform, {{ dbt_utils.generate_surrogate_key( - ['contract_address','address','input','platform'] + ['contract_address','input','platform'] ) }} AS uniswap_v3_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, From 7511cc4e829ec1b0fef14828ceaf13043234204f Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 18 Nov 2025 09:03:14 -0700 Subject: [PATCH 011/164] aave v3 --- .../aave/silver_reads__aave_v3_tvl.sql | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_tvl.sql diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_tvl.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_tvl.sql new file mode 100644 index 00000000..52fe92fc --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_tvl.sql @@ -0,0 +1,59 @@ +{# Get variables #} +{% set vars = return_vars() %} +{# Log configuration details #} +{{ log_model_details() }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'aave_v3_tvl_id', + tags = ['silver','contract_reads','curated_daily'] +) }} + +WITH all_tokens AS ( + + SELECT + atoken_address AS contract_address, + protocol, + version, + CONCAT( + protocol, + '-', + version + ) AS platform + FROM + {{ ref('silver_lending__aave_tokens') }} + WHERE + protocol = 'aave' + AND version = 'v3' + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + contract_address, + NULL AS address, + 'totalSupply' AS function_name, + '0x18160ddd' AS function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input + NULL AS metadata, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','input','platform'] + ) }} AS aave_v3_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + lp_balances From 2d6ef3d6207b5dd11be2eccf42f2f23df4e07647 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 18 Nov 2025 09:43:33 -0700 Subject: [PATCH 012/164] uni v1 and naming --- .../variables/project_vars/ethereum_vars.sql | 3 + ...vl.sql => silver_reads__aave_v1_reads.sql} | 4 +- ...vl.sql => silver_reads__aave_v2_reads.sql} | 4 +- ...vl.sql => silver_reads__aave_v3_reads.sql} | 4 +- ...o_tvl.sql => silver_reads__lido_reads.sql} | 4 +- ...reamline__contract_reads_daily_records.sql | 21 ++--- ...ine__contract_reads_daily_records_test.sql | 70 ++++++++++++++++ ...sql => silver_reads__uniswap_v2_reads.sql} | 4 +- ...sql => silver_reads__uniswap_v3_reads.sql} | 4 +- .../uniswap/silver_dex__uniswap_v1_pools.sql | 84 +++++++++++++++++++ 10 files changed, 180 insertions(+), 22 deletions(-) rename models/curated_package/contract_reads/streamline/records/aave/{silver_reads__aave_v1_tvl.sql => silver_reads__aave_v1_reads.sql} (97%) rename models/curated_package/contract_reads/streamline/records/aave/{silver_reads__aave_v2_tvl.sql => silver_reads__aave_v2_reads.sql} (96%) rename models/curated_package/contract_reads/streamline/records/aave/{silver_reads__aave_v3_tvl.sql => silver_reads__aave_v3_reads.sql} (94%) rename models/curated_package/contract_reads/streamline/records/lido/{silver_reads__lido_tvl.sql => silver_reads__lido_reads.sql} (95%) create mode 100644 models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql rename models/curated_package/contract_reads/streamline/records/uniswap/{silver_reads__uniswap_v2_tvl.sql => silver_reads__uniswap_v2_reads.sql} (94%) rename models/curated_package/contract_reads/streamline/records/uniswap/{silver_reads__uniswap_v3_tvl.sql => silver_reads__uniswap_v3_reads.sql} (95%) create mode 100644 models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pools.sql diff --git a/macros/global/variables/project_vars/ethereum_vars.sql b/macros/global/variables/project_vars/ethereum_vars.sql index e0a24c03..90ee66f1 100644 --- a/macros/global/variables/project_vars/ethereum_vars.sql +++ b/macros/global/variables/project_vars/ethereum_vars.sql @@ -31,6 +31,9 @@ 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['ronin_axie_bridge-v1','multichain-v7','symbiosis-v1','axie_infinity-v2','across-v1','near_rainbow_bridge-v1','hop-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'uniswap': { + 'v1': { + 'factory': '0xc0a47dfe034b400b47bdad5fecda2621de6c4d95' + }, 'v2': { 'uni_v2_pair_created': '0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f' }, diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql similarity index 97% rename from models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql rename to models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql index 6c01dfbb..6fe09c83 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql @@ -5,7 +5,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - unique_key = 'aave_v1_tvl_id', + unique_key = 'aave_v1_reads_id', tags = ['silver','contract_reads','curated_daily'] ) }} @@ -111,7 +111,7 @@ SELECT platform, {{ dbt_utils.generate_surrogate_key( ['contract_address','address','input','platform'] - ) }} AS aave_v1_tvl_id, + ) }} AS aave_v1_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_tvl.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql similarity index 96% rename from models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_tvl.sql rename to models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql index 23b104ac..5b601e87 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql @@ -5,7 +5,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - unique_key = 'aave_v2_tvl_id', + unique_key = 'aave_v2_reads_id', tags = ['silver','contract_reads','curated_daily'] ) }} @@ -79,7 +79,7 @@ SELECT platform, {{ dbt_utils.generate_surrogate_key( ['contract_address','input','platform'] - ) }} AS aave_v2_tvl_id, + ) }} AS aave_v2_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_tvl.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql similarity index 94% rename from models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_tvl.sql rename to models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql index 52fe92fc..0ae69b3b 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql @@ -5,7 +5,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - unique_key = 'aave_v3_tvl_id', + unique_key = 'aave_v3_reads_id', tags = ['silver','contract_reads','curated_daily'] ) }} @@ -51,7 +51,7 @@ SELECT platform, {{ dbt_utils.generate_surrogate_key( ['contract_address','input','platform'] - ) }} AS aave_v3_tvl_id, + ) }} AS aave_v3_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql similarity index 95% rename from models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql rename to models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql index b4c839b7..3c939d6a 100644 --- a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql @@ -7,7 +7,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - unique_key = 'lido_tvl_id', + unique_key = 'lido_reads_id', tags = ['silver','contract_reads','curated_daily'] ) }} @@ -45,7 +45,7 @@ SELECT CONCAT(protocol, '-', version) AS platform, {{ dbt_utils.generate_surrogate_key( ['contract_address','input','platform'] - ) }} AS lido_tvl_id, + ) }} AS lido_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql index 7efb0131..ec69b660 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql @@ -13,7 +13,7 @@ tags = ['streamline','contract_reads','records','phase_4'] ) }} -WITH lido_tvl AS ( +WITH lido AS ( SELECT contract_address, @@ -26,7 +26,7 @@ WITH lido_tvl AS ( version, platform FROM - {{ ref('silver_reads__lido_tvl') }} + {{ ref('silver_reads__lido_reads') }} {% if is_incremental() %} WHERE @@ -38,7 +38,7 @@ WHERE ) {% endif %} ), -uniswap_v2_tvl AS ( +uniswap_v2 AS ( SELECT contract_address, address, @@ -49,7 +49,7 @@ uniswap_v2_tvl AS ( protocol, version, platform - FROM {{ ref('silver_reads__uniswap_v2_tvl') }} + FROM {{ ref('silver_reads__uniswap_v2_reads') }} {% if is_incremental() %} WHERE modified_timestamp > ( SELECT MAX(modified_timestamp) @@ -57,7 +57,7 @@ uniswap_v2_tvl AS ( ) {% endif %} ) -uniswap_v3_tvl AS ( +uniswap_v3 AS ( SELECT contract_address, address, @@ -68,29 +68,30 @@ uniswap_v3_tvl AS ( protocol, version, platform - FROM {{ ref('silver_reads__uniswap_v3_tvl') }} + FROM {{ ref('silver_reads__uniswap_v3_reads') }} {% if is_incremental() %} WHERE modified_timestamp > ( SELECT MAX(modified_timestamp) FROM {{ this }} ) {% endif %} -) +), + all_records AS ( SELECT * FROM - lido_tvl + lido UNION ALL SELECT * FROM - uniswap_v2_tvl + uniswap_v2 UNION ALL SELECT * FROM - uniswap_v3_tvl + uniswap_v3 ) SELECT contract_address, diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql new file mode 100644 index 00000000..5fe51525 --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql @@ -0,0 +1,70 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +{{ config ( + materialized = "incremental", + unique_key = "contract_reads_daily_records_id", + cluster_by = "block_date", + full_refresh = vars.GLOBAL_STREAMLINE_FR_ENABLED, + tags = ['streamline','contract_reads','records','phase_4'] +) }} + +{% set models = [ +(ref('silver_reads__lido_reads')), +(ref('silver_reads__uniswap_v2_reads')), +(ref('silver_reads__uniswap_v3_reads')), +(ref('silver_reads__aave_v1_reads')), +(ref('silver_reads__aave_v2_reads')), +(ref('silver_reads__aave_v3_reads')) + ] +%} + +WITH all_records AS ( + {% for model in models %} + SELECT + contract_address, + address, + function_name, + function_sig, + input, + metadata, + protocol, + version, + platform + FROM {{ models[0] }} + {% if not loop.last %} + {% if is_incremental() %} + WHERE modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} + UNION ALL + {% endif %} + {% endfor %} +) + +SELECT + contract_address, + address, + function_name, + function_sig, + input, + metadata, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','address','input','platform'] + ) }} AS contract_reads_daily_records_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + all_records qualify (ROW_NUMBER() over (PARTITION BY contract_reads_daily_records_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_tvl.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql similarity index 94% rename from models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_tvl.sql rename to models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql index 72269290..73a0d079 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql @@ -7,7 +7,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - unique_key = 'lido_tvl_id', + unique_key = 'lido_reads_id', tags = ['silver','contract_reads','curated_daily'] ) }} @@ -47,7 +47,7 @@ SELECT platform, {{ dbt_utils.generate_surrogate_key( ['contract_address','input','platform'] - ) }} AS uniswap_v2_tvl_id, + ) }} AS uniswap_v2_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_tvl.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql similarity index 95% rename from models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_tvl.sql rename to models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql index dfcf8b1a..34cf105d 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_tvl.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql @@ -7,7 +7,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - unique_key = 'uniswap_v3_tvl_id', + unique_key = 'uniswap_v3_reads_id', tags = ['silver','contract_reads','curated_daily'] ) }} @@ -69,7 +69,7 @@ SELECT platform, {{ dbt_utils.generate_surrogate_key( ['contract_address','input','platform'] - ) }} AS uniswap_v3_tvl_id, + ) }} AS uniswap_v3_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pools.sql b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pools.sql new file mode 100644 index 00000000..e0898e93 --- /dev/null +++ b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pools.sql @@ -0,0 +1,84 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'pool_address', + tags = ['silver_dex','defi','dex','curated'] +) }} + +WITH contract_mapping AS ( + {{ curated_contract_mapping( + vars.CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING + ) }} + WHERE + version = 'v1' +), +pools AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + l.contract_address, + CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS token0, --All pools are token-ETH only, no token-token pairs + '0x0000000000000000000000000000000000000000' AS token1, --native ETH, for pricing purposes + CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS pool_address, + m.protocol, + m.version, + m.type, + CONCAT( + m.protocol, + '-', + m.version + ) AS platform, + 'NewExchange' AS event_name, + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp + FROM + {{ ref('core__fact_event_logs') }} + l + INNER JOIN contract_mapping m + ON l.contract_address = m.contract_address + WHERE + topics [0] :: STRING = '0x9d42cb017eb05bd8944ab536a8b35bc68085931dd5f4356489801453923953f9' --NewExchange + AND tx_succeeded + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) - INTERVAL '{{ vars.CURATED_LOOKBACK_HOURS }}' + FROM + {{ this }} +) +AND modified_timestamp >= SYSDATE() - INTERVAL '{{ vars.CURATED_LOOKBACK_DAYS }}' +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_index, + event_name, + token0, + token1, + pool_address, + platform, + protocol, + version, + type, + _log_id, + modified_timestamp +FROM + pools qualify(ROW_NUMBER() over (PARTITION BY pool_address +ORDER BY + modified_timestamp DESC)) = 1 From 3978939ed5de8f61e672ed14a7e1e179186d5988 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 18 Nov 2025 10:24:28 -0700 Subject: [PATCH 013/164] typos --- .../streamline/records/aave/silver_reads__aave_v1_reads.sql | 2 ++ .../streamline/records/aave/silver_reads__aave_v2_reads.sql | 2 ++ .../streamline/records/aave/silver_reads__aave_v3_reads.sql | 2 +- .../records/uniswap/silver_reads__uniswap_v3_reads.sql | 4 ++-- models/sources.yml | 1 + 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql index 6fe09c83..79fe9c9a 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql @@ -2,6 +2,8 @@ {% set vars = return_vars() %} {# Log configuration details #} {{ log_model_details() }} + +-- depends_on: {{ ref('silver_lending__aave_ethereum_tokens') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql index 5b601e87..d2ffae70 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql @@ -2,6 +2,8 @@ {% set vars = return_vars() %} {# Log configuration details #} {{ log_model_details() }} + +-- depends_on: {{ ref('silver_lending__aave_ethereum_tokens') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql index 0ae69b3b..38623d00 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql @@ -44,7 +44,7 @@ SELECT function_sig, 64, '0' - ) AS input + ) AS input, NULL AS metadata, protocol, version, diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql index 34cf105d..ced5eab3 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql @@ -15,8 +15,8 @@ WITH liquidity_pools AS ( SELECT DISTINCT pool_address, - token0, - token1, + token0_address AS token0, + token1_address AS token1, protocol, version, platform diff --git a/models/sources.yml b/models/sources.yml index 2b605943..8b31a3d9 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -43,6 +43,7 @@ sources: - name: token_reads - name: balances_erc20 - name: balances_native + - name: contract_reads - name: complete_streamline database: "{{ target.database }}" schema: streamline From 675c0efb1e31fde17f2254ef03c81c57517f70a5 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 18 Nov 2025 10:33:07 -0700 Subject: [PATCH 014/164] aave --- .../aave/silver_reads__aave_v1_reads.sql | 2 +- .../aave/silver_reads__aave_v2_reads.sql | 2 +- .../aave/silver_reads__aave_v3_reads.sql | 2 +- ...reamline__contract_reads_daily_records.sql | 74 ++++++++++++++++++- 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql index 79fe9c9a..e87a7df1 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql @@ -118,4 +118,4 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - lp_balances + lending_pools diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql index d2ffae70..506d40bc 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql @@ -74,7 +74,7 @@ SELECT function_sig, 64, '0' - ) AS input + ) AS input, NULL AS metadata, protocol, version, diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql index 38623d00..e8155029 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql @@ -56,4 +56,4 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - lp_balances + all_tokens diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql index ec69b660..07a4ad44 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql @@ -56,7 +56,7 @@ uniswap_v2 AS ( FROM {{ this }} ) {% endif %} -) +), uniswap_v3 AS ( SELECT contract_address, @@ -76,6 +76,63 @@ uniswap_v3 AS ( ) {% endif %} ), +aave_v1 AS ( + SELECT + contract_address, + address, + function_name, + function_sig, + input, + metadata, + protocol, + version, + platform + FROM {{ ref('silver_reads__aave_v1_reads') }} + {% if is_incremental() %} + WHERE modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} +), +aave_v2 AS ( + SELECT + contract_address, + address, + function_name, + function_sig, + input, + metadata, + protocol, + version, + platform + FROM {{ ref('silver_reads__aave_v2_reads') }} + {% if is_incremental() %} + WHERE modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} +), +aave_v3 AS ( + SELECT + contract_address, + address, + function_name, + function_sig, + input, + metadata, + protocol, + version, + platform + FROM {{ ref('silver_reads__aave_v3_reads') }} + {% if is_incremental() %} + WHERE modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} +), all_records AS ( SELECT @@ -92,6 +149,21 @@ all_records AS ( * FROM uniswap_v3 + UNION ALL + SELECT + * + FROM + aave_v1 + UNION ALL + SELECT + * + FROM + aave_v2 + UNION ALL + SELECT + * + FROM + aave_v3 ) SELECT contract_address, From b6944e8347449259b2e46918b24b439fc60c44ad Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 18 Nov 2025 10:37:41 -0700 Subject: [PATCH 015/164] columns --- .../records/aave/silver_reads__aave_v1_reads.sql | 8 -------- .../records/aave/silver_reads__aave_v2_reads.sql | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql index e87a7df1..a87de13b 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql @@ -72,10 +72,6 @@ lending_pools AS ( WHEN '{{ vars.GLOBAL_PROJECT_NAME }}' = 'ethereum' THEN '0x3dfd23a6c5e8bbcfc9581d2e864a68feb6a076d3' END AS address, --Aave: LendingPoolCore - function_name, - function_sig, - input, - metadata, protocol, version, platform @@ -88,10 +84,6 @@ lending_pools AS ( WHEN '{{ vars.GLOBAL_PROJECT_NAME }}' = 'ethereum' THEN '0x1012cff81a1582ddd0616517efb97d02c5c17e25' END AS address, --Uniswap: LendingPoolCore in Aave v1 holds Uniswap v1 LP tokens as collateral - function_name, - function_sig, - input, - metadata, protocol, version, platform diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql index 506d40bc..50111ed3 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql @@ -86,4 +86,4 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - lp_balances + all_tokens From 0e5f8b9a3ca784c345f63eba21bb4ed2750d0249 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 18 Nov 2025 10:42:57 -0700 Subject: [PATCH 016/164] remove cluster by --- .../records/streamline__contract_reads_daily_records.sql | 1 - .../records/streamline__contract_reads_daily_records_test.sql | 1 - 2 files changed, 2 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql index 07a4ad44..b519d313 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql @@ -8,7 +8,6 @@ {{ config ( materialized = "incremental", unique_key = "contract_reads_daily_records_id", - cluster_by = "block_date", full_refresh = vars.GLOBAL_STREAMLINE_FR_ENABLED, tags = ['streamline','contract_reads','records','phase_4'] ) }} diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql index 5fe51525..5aa0b7f7 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql @@ -8,7 +8,6 @@ {{ config ( materialized = "incremental", unique_key = "contract_reads_daily_records_id", - cluster_by = "block_date", full_refresh = vars.GLOBAL_STREAMLINE_FR_ENABLED, tags = ['streamline','contract_reads','records','phase_4'] ) }} From 6bd2550bcf110b8ef2b7c66a796f464eea508d25 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 18 Nov 2025 10:44:52 -0700 Subject: [PATCH 017/164] variant type --- .../streamline/records/aave/silver_reads__aave_v1_reads.sql | 2 +- .../streamline/records/aave/silver_reads__aave_v2_reads.sql | 2 +- .../streamline/records/aave/silver_reads__aave_v3_reads.sql | 2 +- .../streamline/records/lido/silver_reads__lido_reads.sql | 2 +- .../records/uniswap/silver_reads__uniswap_v2_reads.sql | 2 +- .../records/uniswap/silver_reads__uniswap_v3_reads.sql | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql index a87de13b..36bd09a3 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql @@ -99,7 +99,7 @@ SELECT '0x70a08231', LPAD(SUBSTR(address, 3), 64, '0') ) AS input, - NULL AS metadata, + NULL :: VARIANT AS metadata, protocol, version, platform, diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql index 50111ed3..03cc9dec 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql @@ -75,7 +75,7 @@ SELECT 64, '0' ) AS input, - NULL AS metadata, + NULL :: VARIANT AS metadata, protocol, version, platform, diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql index e8155029..a2497a98 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql @@ -45,7 +45,7 @@ SELECT 64, '0' ) AS input, - NULL AS metadata, + NULL :: VARIANT AS metadata, protocol, version, platform, diff --git a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql index 3c939d6a..3e714c0c 100644 --- a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql @@ -39,7 +39,7 @@ SELECT function_name, function_sig, input, - NULL AS metadata, + NULL :: VARIANT AS metadata, 'lido' AS protocol, 'v1' AS version, CONCAT(protocol, '-', version) AS platform, diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql index 73a0d079..71246d29 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql @@ -41,7 +41,7 @@ SELECT OBJECT_CONSTRUCT( 'token0', token0, 'token1', token1 - ) AS metadata, + ) :: VARIANT AS metadata, protocol, version, platform, diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql index ced5eab3..0570f15c 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql @@ -63,7 +63,7 @@ SELECT OBJECT_CONSTRUCT( 'token0', token0, 'token1', token1 - ) AS metadata, + ) :: VARIANT AS metadata, protocol, version, platform, From 62bf8deb41b28d703dc1132d07d7cf607522f9e9 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 18 Nov 2025 13:33:41 -0700 Subject: [PATCH 018/164] eth lido logic --- ...reamline__contract_reads_daily_records.sql | 181 ++++-------------- ...ine__contract_reads_daily_records_test.sql | 69 ------- 2 files changed, 32 insertions(+), 218 deletions(-) delete mode 100644 models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql index b519d313..4d90c023 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql @@ -12,158 +12,41 @@ tags = ['streamline','contract_reads','records','phase_4'] ) }} -WITH lido AS ( - - SELECT - contract_address, - address, - function_name, - function_sig, - input, - metadata, - protocol, - version, - platform - FROM - {{ ref('silver_reads__lido_reads') }} - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) +{% set models = [] %} +{% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} + {% set _ = models.append(ref('silver_reads__lido_reads')) %} {% endif %} -), -uniswap_v2 AS ( - SELECT - contract_address, - address, - function_name, - function_sig, - input, - metadata, - protocol, - version, - platform - FROM {{ ref('silver_reads__uniswap_v2_reads') }} - {% if is_incremental() %} - WHERE modified_timestamp > ( - SELECT MAX(modified_timestamp) - FROM {{ this }} - ) - {% endif %} -), -uniswap_v3 AS ( - SELECT - contract_address, - address, - function_name, - function_sig, - input, - metadata, - protocol, - version, - platform - FROM {{ ref('silver_reads__uniswap_v3_reads') }} - {% if is_incremental() %} - WHERE modified_timestamp > ( - SELECT MAX(modified_timestamp) - FROM {{ this }} - ) - {% endif %} -), -aave_v1 AS ( - SELECT - contract_address, - address, - function_name, - function_sig, - input, - metadata, - protocol, - version, - platform - FROM {{ ref('silver_reads__aave_v1_reads') }} - {% if is_incremental() %} - WHERE modified_timestamp > ( - SELECT MAX(modified_timestamp) - FROM {{ this }} - ) - {% endif %} -), -aave_v2 AS ( - SELECT - contract_address, - address, - function_name, - function_sig, - input, - metadata, - protocol, - version, - platform - FROM {{ ref('silver_reads__aave_v2_reads') }} - {% if is_incremental() %} - WHERE modified_timestamp > ( - SELECT MAX(modified_timestamp) - FROM {{ this }} - ) - {% endif %} -), -aave_v3 AS ( - SELECT - contract_address, - address, - function_name, - function_sig, - input, - metadata, - protocol, - version, - platform - FROM {{ ref('silver_reads__aave_v3_reads') }} - {% if is_incremental() %} - WHERE modified_timestamp > ( - SELECT MAX(modified_timestamp) - FROM {{ this }} - ) - {% endif %} -), +{% set _ = models.append(ref('silver_reads__uniswap_v2_reads')) %} +{% set _ = models.append(ref('silver_reads__uniswap_v3_reads')) %} +{% set _ = models.append(ref('silver_reads__aave_v1_reads')) %} +{% set _ = models.append(ref('silver_reads__aave_v2_reads')) %} +{% set _ = models.append(ref('silver_reads__aave_v3_reads')) %} -all_records AS ( - SELECT - * - FROM - lido - UNION ALL - SELECT - * - FROM - uniswap_v2 - UNION ALL - SELECT - * - FROM - uniswap_v3 - UNION ALL - SELECT - * - FROM - aave_v1 - UNION ALL - SELECT - * - FROM - aave_v2 - UNION ALL - SELECT - * - FROM - aave_v3 +WITH all_records AS ( + {% for model in models %} + SELECT + contract_address, + address, + function_name, + function_sig, + input, + metadata, + protocol, + version, + platform + FROM {{ models[0] }} + {% if not loop.last %} + {% if is_incremental() %} + WHERE modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} + UNION ALL + {% endif %} + {% endfor %} ) + SELECT contract_address, address, diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql deleted file mode 100644 index 5aa0b7f7..00000000 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records_test.sql +++ /dev/null @@ -1,69 +0,0 @@ -{# Get variables #} -{% set vars = return_vars() %} - -{# Log configuration details #} -{{ log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = "incremental", - unique_key = "contract_reads_daily_records_id", - full_refresh = vars.GLOBAL_STREAMLINE_FR_ENABLED, - tags = ['streamline','contract_reads','records','phase_4'] -) }} - -{% set models = [ -(ref('silver_reads__lido_reads')), -(ref('silver_reads__uniswap_v2_reads')), -(ref('silver_reads__uniswap_v3_reads')), -(ref('silver_reads__aave_v1_reads')), -(ref('silver_reads__aave_v2_reads')), -(ref('silver_reads__aave_v3_reads')) - ] -%} - -WITH all_records AS ( - {% for model in models %} - SELECT - contract_address, - address, - function_name, - function_sig, - input, - metadata, - protocol, - version, - platform - FROM {{ models[0] }} - {% if not loop.last %} - {% if is_incremental() %} - WHERE modified_timestamp > ( - SELECT MAX(modified_timestamp) - FROM {{ this }} - ) - {% endif %} - UNION ALL - {% endif %} - {% endfor %} -) - -SELECT - contract_address, - address, - function_name, - function_sig, - input, - metadata, - protocol, - version, - platform, - {{ dbt_utils.generate_surrogate_key( - ['contract_address','address','input','platform'] - ) }} AS contract_reads_daily_records_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - all_records qualify (ROW_NUMBER() over (PARTITION BY contract_reads_daily_records_id -ORDER BY - modified_timestamp DESC)) = 1 From 662cb0213d5f859ce6f36779a0308f7558447c6e Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 18 Nov 2025 15:40:49 -0700 Subject: [PATCH 019/164] model --- .../records/streamline__contract_reads_daily_records.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql index 4d90c023..b4c3f322 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql @@ -4,7 +4,7 @@ {# Log configuration details #} {{ log_model_details() }} -{# Set up dbt configuration #} +-- depends_on: {{ ref('silver_reads__lido_reads') }} {{ config ( materialized = "incremental", unique_key = "contract_reads_daily_records_id", @@ -34,7 +34,7 @@ WITH all_records AS ( protocol, version, platform - FROM {{ models[0] }} + FROM {{ model }} {% if not loop.last %} {% if is_incremental() %} WHERE modified_timestamp > ( From 89e31a8135a7cd0f92755176fcef489d1d7ff2f4 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 20 Nov 2025 15:11:24 -0700 Subject: [PATCH 020/164] versions --- .../streamline/records/aave/silver_reads__aave_v1_reads.sql | 6 ++---- .../streamline/records/aave/silver_reads__aave_v2_reads.sql | 6 ++---- .../streamline/records/aave/silver_reads__aave_v3_reads.sql | 3 +-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql index 36bd09a3..c47f3145 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql @@ -25,8 +25,7 @@ WITH all_tokens AS ( FROM {{ ref('silver_lending__aave_tokens') }} WHERE - protocol = 'aave' - AND version = 'v1' + version = 'v1' {% if is_incremental() %} AND modified_timestamp > ( @@ -52,8 +51,7 @@ FROM {{ ref('silver_lending__aave_ethereum_tokens') }} --relevant for ethereum only WHERE - protocol = 'aave' - AND version = 'v1' + version = 'v1' {% if is_incremental() %} AND modified_timestamp > ( diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql index 03cc9dec..039eab5f 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql @@ -25,8 +25,7 @@ WITH all_tokens AS ( FROM {{ ref('silver_lending__aave_tokens') }} WHERE - protocol = 'aave' - AND version = 'v2' + version = 'v2' {% if is_incremental() %} AND modified_timestamp > ( @@ -52,8 +51,7 @@ FROM {{ ref('silver_lending__aave_ethereum_tokens') }} --relevant for ethereum only WHERE - protocol = 'aave' - AND version IN ('v2','v2.1') + version IN ('v2','v2.1') {% if is_incremental() %} AND modified_timestamp > ( diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql index a2497a98..d9d04b5c 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql @@ -23,8 +23,7 @@ WITH all_tokens AS ( FROM {{ ref('silver_lending__aave_tokens') }} WHERE - protocol = 'aave' - AND version = 'v3' + version = 'v3' {% if is_incremental() %} AND modified_timestamp > ( From c452f3c9d5d27f14958bc8d4a74167e626dbc3c3 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 20 Nov 2025 15:32:45 -0700 Subject: [PATCH 021/164] model name --- ...complete.sql => streamline__contract_reads_daily_complete.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/curated_package/contract_reads/streamline/complete/{streamline__contract_reads_complete.sql => streamline__contract_reads_daily_complete.sql} (100%) diff --git a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql similarity index 100% rename from models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql rename to models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql From fc218c9f148aedc566ce658dd9351e81bdf28909 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 20 Nov 2025 15:39:37 -0700 Subject: [PATCH 022/164] uk --- .../records/uniswap/silver_reads__uniswap_v2_reads.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql index 71246d29..19fd2262 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql @@ -7,7 +7,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - unique_key = 'lido_reads_id', + unique_key = 'uniswap_v2_reads_id', tags = ['silver','contract_reads','curated_daily'] ) }} From 136dbdd607584300bdda5a224bc8f9d21edf186a Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 20 Nov 2025 15:58:37 -0700 Subject: [PATCH 023/164] vars --- macros/global/variables/return_vars.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/macros/global/variables/return_vars.sql b/macros/global/variables/return_vars.sql index dcb72cd8..10025fd2 100644 --- a/macros/global/variables/return_vars.sql +++ b/macros/global/variables/return_vars.sql @@ -280,6 +280,17 @@ {% set ns.CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST = get_var('CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST', []) %} {% set ns.CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST = get_var('CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST', []) %} + {# SL Contract Reads Variables #} + {% set ns.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_SQL_LIMIT = get_var('CURATED_SL_CONTRACT_READS_DAILY_REALTIME_SQL_LIMIT', 12000000) %} + {% set ns.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_PRODUCER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_DAILY_REALTIME_PRODUCER_BATCH_SIZE', 240000) %} + {% set ns.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_WORKER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_DAILY_REALTIME_WORKER_BATCH_SIZE', 60000) %} + {% set ns.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_ASYNC_CONCURRENT_REQUESTS = get_var('CURATED_SL_CONTRACT_READS_DAILY_REALTIME_ASYNC_CONCURRENT_REQUESTS', 10) %} + + {% set ns.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_SQL_LIMIT = get_var('CURATED_SL_CONTRACT_READS_DAILY_HISTORY_SQL_LIMIT', 12000000) %} + {% set ns.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_PRODUCER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_DAILY_HISTORY_PRODUCER_BATCH_SIZE', 600000) %} + {% set ns.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_WORKER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_DAILY_HISTORY_WORKER_BATCH_SIZE', 60000) %} + {% set ns.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_ASYNC_CONCURRENT_REQUESTS = get_var('CURATED_SL_CONTRACT_READS_DAILY_HISTORY_ASYNC_CONCURRENT_REQUESTS', 10) %} + {# SL Stablecoin Reads Variables #} {% set ns.CURATED_SL_STABLECOIN_READS_REALTIME_SQL_LIMIT = get_var('CURATED_SL_STABLECOIN_READS_REALTIME_SQL_LIMIT', 3000) %} {% set ns.CURATED_SL_STABLECOIN_READS_REALTIME_PRODUCER_BATCH_SIZE = get_var('CURATED_SL_STABLECOIN_READS_REALTIME_PRODUCER_BATCH_SIZE', 1500) %} From 8e3ef4b2141ee1e605932ab0d739feb1f9477336 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 20 Nov 2025 16:08:02 -0700 Subject: [PATCH 024/164] cross join --- ...streamline__contract_reads_daily_realtime_requests.sql | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index bb6464a8..a045c5b3 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -33,11 +33,7 @@ to_do AS ( t.platform FROM {{ ref("streamline__contract_reads_daily_records") }} t - INNER JOIN last_x_days d - ON t.block_date = d.block_date - --max daily block_number from 1 day ago, for each contract_address/address pair - WHERE - t.block_date IS NOT NULL + CROSS JOIN last_x_days d EXCEPT SELECT block_number, @@ -86,7 +82,7 @@ SELECT '-', input, '-', - block_number, + block_number ), 'jsonrpc', '2.0', 'method', 'eth_call', From 400baf5fa84f4cd222d8d36f403598bc40968213 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 21 Nov 2025 11:11:06 -0700 Subject: [PATCH 025/164] curve, tc, uni v4 reads --- .../silver_reads__tornado_cash_mixer_seed.sql | 49 +++++++ .../curve/silver_reads__curve_reads.sql | 55 ++++++++ ...reamline__contract_reads_daily_records.sql | 3 + .../silver_reads__tornado_cash_reads.sql | 61 +++++++++ .../silver_reads__uniswap_v4_reads.sql | 125 ++++++++++++++++++ 5 files changed, 293 insertions(+) create mode 100644 data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.sql create mode 100644 models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql create mode 100644 models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql create mode 100644 models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql diff --git a/data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.sql b/data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.sql new file mode 100644 index 00000000..efa4dbdc --- /dev/null +++ b/data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.sql @@ -0,0 +1,49 @@ +chain,token_address,mixer_address +ethereum,null,0x12d66f87a04a9e220743712ce6d9bb1b5616b8fc +ethereum,null,0x47ce0c6ed5b0ce3d3a51fdb1c52dc66a7c3c2936 +ethereum,null,0x910cbd523d972eb0a6f4cae4618ad62622b39dbf +ethereum,null,0xa160cdab225685da1d56aa342ad8841c3b53f291 +ethereum,0x6b175474e89094c44da98b954eedeac495271d0f,0xd4b88df4d29f5cedd6857912842cff3b20c8cfa3 +ethereum,0x6b175474e89094c44da98b954eedeac495271d0f,0xfd8610d20aa15b7b2e3be39b396a1bc3516c7144 +ethereum,0x6b175474e89094c44da98b954eedeac495271d0f,0x07687e702b410fa43f4cb4af7fa097918ffd2730 +ethereum,0x6b175474e89094c44da98b954eedeac495271d0f,0x23773e65ed146a459791799d01336db287f25334 +ethereum,0xdac17f958d2ee523a2206206994597c13d831ec7,0x169ad27a470d064dede56a2d3ff727986b15d52b +ethereum,0xdac17f958d2ee523a2206206994597c13d831ec7,0x0836222f2b2b24a3f36f98668ed8f0b38d1a872f +ethereum,0xdac17f958d2ee523a2206206994597c13d831ec7,0xf67721a2d8f736e75a49fdd7fad2e31d8676542a +ethereum,0xdac17f958d2ee523a2206206994597c13d831ec7,0x9ad122c22b14202b4490edaf288fdb3c7cb3ff5e +ethereum,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,0xd96f2b1c14db8458374d9aca76e26c3d18364307 +ethereum,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,0x4736dcf1b7a3d580672cce6e7c65cd5cc9cfba9d +ethereum,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,0xd691f27f38b395864ea86cfc7253969b409c362d +ethereum,0x2260fac5e5542a773aa44fbcfedf7c193bc2c599,0x178169b423a011fff22b9e3f3abea13414ddd0f1 +ethereum,0x2260fac5e5542a773aa44fbcfedf7c193bc2c599,0x610b717796ad172b316836ac95a2ffad065ceab4 +ethereum,0x2260fac5e5542a773aa44fbcfedf7c193bc2c599,0xbb93e510bbcd0b7beb5a853875f9ec60275cf498 +ethereum,0x5d3a536e4d6dbd6114cc1ead35777bab948e3643,0x22aaa7720ddd5388a3c0a3333430953c68f1849b +ethereum,0x5d3a536e4d6dbd6114cc1ead35777bab948e3643,0x03893a7c7463ae47d46bc7f091665f1893656003 +ethereum,0x5d3a536e4d6dbd6114cc1ead35777bab948e3643,0x2717c5e28cf931547b621a5dddb772ab6a35b701 +ethereum,0x5d3a536e4d6dbd6114cc1ead35777bab948e3643,0xd21be7248e0197ee08e0c20d4a96debdac3d20af +bsc,null,0x84443cfd09a48af6ef360c6976c5392ac5023a1f +bsc,null,0xd47438c816c9e7f2e2888e060936a499af9582b3 +bsc,null,0x330bdfade01ee9bf63c209ee33102dd334618e0a +bsc,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +polygon,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +polygon,null,0xdf231d99ff8b6c6cbf4e9b9a945cbacef9339178 +polygon,null,0xaf4c0b70b2ea9fb7487c7cbb37ada259579fe040 +polygon,null,0xa5c2254e4253490c54cef0a4347fddb8f75a4998 +optimism,null,0x84443cfd09a48af6ef360c6976c5392ac5023a1f +optimism,null,0xd47438c816c9e7f2e2888e060936a499af9582b3 +optimism,null,0x330bdfade01ee9bf63c209ee33102dd334618e0a +optimism,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +arbitrum,null,0x84443cfd09a48af6ef360c6976c5392ac5023a1f +arbitrum,null,0xd47438c816c9e7f2e2888e060936a499af9582b3 +arbitrum,null,0x330bdfade01ee9bf63c209ee33102dd334618e0a +arbitrum,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +gnosis,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +gnosis,null,0xdf231d99ff8b6c6cbf4e9b9a945cbacef9339178 +gnosis,null,0xaf4c0b70b2ea9fb7487c7cbb37ada259579fe040 +gnosis,null,0xa5c2254e4253490c54cef0a4347fddb8f75a4998 +avax,null,0x330bdfade01ee9bf63c209ee33102dd334618e0a +avax,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +avax,null,0xaf8d1839c3c67cf571aa74b5c12398d4901147b3 +ethereumclassic,null,0x2f56d5afc058b8734350b162efee75ee48f034e0 +ethereumclassic,null,0x59fcb629a23e8ed0a60a0188771e221042260118 +ethereumclassic,null,0x784b3a7a7981b959bd8d9d9e73c2013be819fbf2 \ No newline at end of file diff --git a/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql b/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql new file mode 100644 index 00000000..a9130759 --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql @@ -0,0 +1,55 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'curve_reads_id', + tags = ['silver','contract_reads','curated_daily'] +) }} + +WITH liquidity_pools AS ( + + SELECT + DISTINCT pool_address AS address, + token_address AS contract_address, + protocol, + version, + platform + FROM + {{ ref('silver_dex__curve_pools') }} + +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + contract_address, + address, + 'balanceOf' AS function_name, + '0x70a08231' AS function_sig, + CONCAT( + '0x70a08231', + LPAD(SUBSTR(address, 3), 64, '0') + ) AS input, + NULL :: variant AS metadata, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','address','input','platform'] + ) }} AS curve_reads_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + liquidity_pools diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql index b4c3f322..fe589f92 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql @@ -18,9 +18,12 @@ {% endif %} {% set _ = models.append(ref('silver_reads__uniswap_v2_reads')) %} {% set _ = models.append(ref('silver_reads__uniswap_v3_reads')) %} +{% set _ = models.append(ref('silver_reads__uniswap_v4_reads')) %} {% set _ = models.append(ref('silver_reads__aave_v1_reads')) %} {% set _ = models.append(ref('silver_reads__aave_v2_reads')) %} {% set _ = models.append(ref('silver_reads__aave_v3_reads')) %} +{% set _ = models.append(ref('silver_reads__curve_reads')) %} +{% set _ = models.append(ref('silver_reads__tornado_cash_reads')) %} WITH all_records AS ( {% for model in models %} diff --git a/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql b/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql new file mode 100644 index 00000000..6f1b2e45 --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql @@ -0,0 +1,61 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'tornado_cash_reads_id', + tags = ['silver','contract_reads','curated_daily'] +) }} + +WITH mixers AS ( + + SELECT + token_address AS contract_address, + mixer_address AS address, + 'balanceOf' AS function_name, + '0x70a08231' AS function_sig, + CONCAT( + '0x70a08231', + LPAD(SUBSTR(address, 3), 64, '0') + ) AS input, + FROM + {{ ref('silver_reads__tornado_cash_mixer_seed') }} + WHERE + chain = '{{ vars.GLOBAL_PROJECT_NAME }}' + AND contract_address IS NOT NULL --balanceOf calls only apply to erc20 token-mixer pairs. eth_getBalance calls to be handled downstream for null-mixer pairs. + +{% if is_incremental() %} +AND CONCAT(COALESCE(contract_address, 'null'), '-', address) NOT IN ( + SELECT + CONCAT(COALESCE(contract_address, 'null'), '-', address) + FROM + {{ this }} +) +{% endif %} +) +SELECT + contract_address, + NULL AS address, + function_name, + function_sig, + input, + NULL :: variant AS metadata, + 'tornado_cash' AS protocol, + 'v1' AS version, + CONCAT( + protocol, + '-', + version + ) AS platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','address','input','platform'] + ) }} AS tornado_cash_reads_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + mixers diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql new file mode 100644 index 00000000..e822d162 --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql @@ -0,0 +1,125 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'uniswap_v4_reads_id', + tags = ['silver','contract_reads','curated_daily'] +) }} + +WITH liquidity_pools AS ( + + SELECT + token0, + token1, + pool_address AS factory_address, + hook_address, + protocol, + version, + platform + FROM + {{ ref('silver_dex__uniswap_v4_pools') }} + +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} +), +all_balances AS ( + SELECT + token0 AS contract_address, + factory_address AS address, + token0, + token1, + hook_address, + 'factory' AS address_type, + protocol, + version, + platform + FROM + liquidity_pools + UNION + SELECT + token1 AS contract_address, + factory_address AS address, + token0, + token1, + hook_address, + 'factory' AS address_type, + protocol, + version, + platform + FROM + liquidity_pools + UNION + SELECT + token0 AS contract_address, + hook_address AS address, + token0, + token1, + hook_address, + 'hook' AS address_type, + protocol, + version, + platform + FROM + liquidity_pools + WHERE + hook_address IS NOT NULL + AND hook_address <> '0x0000000000000000000000000000000000000000' + UNION + SELECT + token1 AS contract_address, + hook_address AS address, + token0, + token1, + hook_address, + 'hook' AS address_type, + protocol, + version, + platform + FROM + liquidity_pools + WHERE + hook_address IS NOT NULL + AND hook_address <> '0x0000000000000000000000000000000000000000' +) +SELECT + contract_address, + address, + 'balanceOf' AS function_name, + '0x70a08231' AS function_sig, + CONCAT( + '0x70a08231', + LPAD(SUBSTR(address, 3), 64, '0') + ) AS input, + object_construct_keep_null( + 'token0', + token0, + 'token1', + token1, + 'hook_address', + hook_address, + 'address_type', + address_type + ) :: variant AS metadata, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','input','platform'] + ) }} AS uniswap_v4_reads_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + all_balances From fa507c0b6222ac833d10e25ab19802d9effe9236 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 21 Nov 2025 11:38:09 -0700 Subject: [PATCH 026/164] file type --- ...h_mixer_seed.sql => silver_reads__tornado_cash_mixer_seed.csv} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename data/curated/contract_reads/{silver_reads__tornado_cash_mixer_seed.sql => silver_reads__tornado_cash_mixer_seed.csv} (100%) diff --git a/data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.sql b/data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.csv similarity index 100% rename from data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.sql rename to data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.csv From 9d86355d0a2988681024eb247afb4c772be3c526 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 21 Nov 2025 11:45:35 -0700 Subject: [PATCH 027/164] address column --- .../records/tornado_cash/silver_reads__tornado_cash_reads.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql b/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql index 6f1b2e45..e67bdf31 100644 --- a/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql @@ -39,7 +39,7 @@ AND CONCAT(COALESCE(contract_address, 'null'), '-', address) NOT IN ( ) SELECT contract_address, - NULL AS address, + address, function_name, function_sig, input, From add26277804ca9225d92e6268438e973a6aad31e Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 21 Nov 2025 12:18:07 -0700 Subject: [PATCH 028/164] curve ts --- .../streamline/records/curve/silver_reads__curve_reads.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql b/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql index a9130759..9b8150e9 100644 --- a/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql @@ -24,7 +24,7 @@ WITH liquidity_pools AS ( {% if is_incremental() %} WHERE - modified_timestamp > ( + _inserted_timestamp > ( SELECT MAX(modified_timestamp) FROM From 385ddea4588e987d880ac4faeb8cc6ce19d4b989 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 21 Nov 2025 12:29:06 -0700 Subject: [PATCH 029/164] token filter uni v4 --- .../records/uniswap/silver_reads__uniswap_v4_reads.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql index e822d162..ed7ec691 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql @@ -47,6 +47,9 @@ all_balances AS ( platform FROM liquidity_pools + WHERE + token0 IS NOT NULL + AND token0 <> '0x0000000000000000000000000000000000000000' UNION SELECT token1 AS contract_address, @@ -60,6 +63,9 @@ all_balances AS ( platform FROM liquidity_pools + WHERE + token1 IS NOT NULL + AND token1 <> '0x0000000000000000000000000000000000000000' UNION SELECT token0 AS contract_address, From 1ab9fbc1068685bd68013262f5022576e94a53b5 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 21 Nov 2025 12:30:40 -0700 Subject: [PATCH 030/164] comment --- .../records/uniswap/silver_reads__uniswap_v4_reads.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql index ed7ec691..bcdb4a78 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql @@ -49,7 +49,7 @@ all_balances AS ( liquidity_pools WHERE token0 IS NOT NULL - AND token0 <> '0x0000000000000000000000000000000000000000' + AND token0 <> '0x0000000000000000000000000000000000000000' -- Represents native asset. balanceOf calls only apply to erc20 tokens. eth_getBalance calls to be handled downstream for null/native assets. UNION SELECT token1 AS contract_address, From dd42bb1669cdf40aebc4580d8c7e117426315ccd Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 21 Nov 2025 12:38:32 -0700 Subject: [PATCH 031/164] filter --- .../records/uniswap/silver_reads__uniswap_v4_reads.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql index bcdb4a78..dc8b7766 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql @@ -82,6 +82,8 @@ all_balances AS ( WHERE hook_address IS NOT NULL AND hook_address <> '0x0000000000000000000000000000000000000000' + AND token0 IS NOT NULL + AND token0 <> '0x0000000000000000000000000000000000000000' UNION SELECT token1 AS contract_address, @@ -98,6 +100,8 @@ all_balances AS ( WHERE hook_address IS NOT NULL AND hook_address <> '0x0000000000000000000000000000000000000000' + AND token1 IS NOT NULL + AND token1 <> '0x0000000000000000000000000000000000000000' ) SELECT contract_address, From fa5475b9b2bdd9158120d5f32a96d515d67b399c Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 21 Nov 2025 13:38:37 -0700 Subject: [PATCH 032/164] coalesce --- .../streamline__contract_reads_daily_realtime_requests.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index a045c5b3..0f4d60e2 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -78,7 +78,7 @@ SELECT 'id', CONCAT( contract_address, '-', - address, + COALESCE(address,'null'), '-', input, '-', From 0f432a8953201a0ae5264f5bb40141bc567b751d Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 21 Nov 2025 14:24:22 -0700 Subject: [PATCH 033/164] silver columns --- .../silver/silver__contract_reads.sql | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/models/curated_package/contract_reads/silver/silver__contract_reads.sql b/models/curated_package/contract_reads/silver/silver__contract_reads.sql index cf789d22..9b539e7a 100644 --- a/models/curated_package/contract_reads/silver/silver__contract_reads.sql +++ b/models/curated_package/contract_reads/silver/silver__contract_reads.sql @@ -1,9 +1,7 @@ {# Get variables #} {% set vars = return_vars() %} - {# Log configuration details #} {{ log_model_details() }} - -- depends_on: {{ ref('bronze__contract_reads') }} {{ config( materialized = 'incremental', @@ -16,17 +14,20 @@ ) }} SELECT - VALUE :"ADDRESS" :: STRING AS address, - VALUE :"FUNCTION_NAME" :: STRING AS function_name, - VALUE :"FUNCTION_SIG" :: STRING AS function_sig, - VALUE :"INPUT" :: STRING AS input, VALUE :"BLOCK_NUMBER" :: NUMBER AS block_number, ( VALUE :"BLOCK_DATE_UNIX" :: TIMESTAMP ) :: DATE AS block_date, - DATA :result :: STRING AS result_hex, - (VALUE :"METADATA" :: STRING) :: VARIANT AS metadata, + VALUE :"CONTRACT_ADDRESS" :: STRING AS contract_address, + VALUE :"ADDRESS" :: STRING AS address, VALUE :"PLATFORM" :: STRING AS platform, + ( + VALUE :"METADATA" :: STRING + ) :: variant AS metadata, + VALUE :"FUNCTION_NAME" :: STRING AS function_name, + VALUE :"FUNCTION_SIG" :: STRING AS function_sig, + VALUE :"INPUT" :: STRING AS input, + DATA :result :: STRING AS result_hex, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['block_number','contract_address', 'address', 'input', 'platform'] From ae8f352bda120a26b1b50f66830cafa40721de75 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 1 Dec 2025 12:06:50 -0700 Subject: [PATCH 034/164] comment --- .../streamline/records/lido/silver_reads__lido_reads.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql index 3e714c0c..b39d4792 100644 --- a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql @@ -49,5 +49,4 @@ SELECT SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id -FROM pooled_assets ---need to add logic to bypass model / not read if not on ethereum \ No newline at end of file +FROM pooled_assets \ No newline at end of file From e4b419ca440808a0ff6437d857c8e7c4c262c1f8 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 1 Dec 2025 14:26:40 -0700 Subject: [PATCH 035/164] tvl --- ...eamline__contract_reads_daily_complete.sql | 2 + ...contract_reads_daily_realtime_requests.sql | 6 ++ .../defi/tvl/lido/silver__lido_tvl.sql | 55 +++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 models/curated_package/defi/tvl/lido/silver__lido_tvl.sql diff --git a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql index a511a073..21ce6fd0 100644 --- a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql +++ b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql @@ -29,6 +29,8 @@ SELECT ( VALUE :"METADATA" :: STRING ) :: variant AS metadata, + VALUE :"PROTOCOL" :: STRING AS protocol, + VALUE :"VERSION" :: STRING AS version, VALUE :"PLATFORM" :: STRING AS platform, file_name, {{ dbt_utils.generate_surrogate_key( diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index 0f4d60e2..e71e1f14 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -30,6 +30,8 @@ to_do AS ( t.function_sig, t.input, t.metadata, + t.protocol, + t.version, t.platform FROM {{ ref("streamline__contract_reads_daily_records") }} t @@ -44,6 +46,8 @@ to_do AS ( function_sig, input, metadata, + protocol, + version, platform FROM {{ ref("streamline__contract_reads_daily_complete") }} @@ -68,6 +72,8 @@ SELECT function_sig, input, metadata, + protocol, + version, platform, ROUND( block_number, diff --git a/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql b/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql new file mode 100644 index 00000000..d76c8c60 --- /dev/null +++ b/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql @@ -0,0 +1,55 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'lido_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH reads AS ( + + SELECT + block_number, + block_date, + contract_address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + protocol, + version, + platform, + _inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} + WHERE + platform = 'lido-v1' + AND amount_raw IS NOT NULL + ) + SELECT + block_number, + block_date, + contract_address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','platform'] + ) }} AS lido_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + reads From f8fe361f96a4432ae2aec2f617a83260562a16f3 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 1 Dec 2025 14:56:58 -0700 Subject: [PATCH 036/164] protocol column --- .../contract_reads/silver/silver__contract_reads.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/curated_package/contract_reads/silver/silver__contract_reads.sql b/models/curated_package/contract_reads/silver/silver__contract_reads.sql index 9b539e7a..994e9cf2 100644 --- a/models/curated_package/contract_reads/silver/silver__contract_reads.sql +++ b/models/curated_package/contract_reads/silver/silver__contract_reads.sql @@ -20,6 +20,8 @@ SELECT ) :: DATE AS block_date, VALUE :"CONTRACT_ADDRESS" :: STRING AS contract_address, VALUE :"ADDRESS" :: STRING AS address, + VALUE :"PROTOCOL" :: STRING AS protocol, + VALUE :"VERSION" :: STRING AS version, VALUE :"PLATFORM" :: STRING AS platform, ( VALUE :"METADATA" :: STRING From 58269efb1c89e1eefd2cb263076fdcbe4c899ee8 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 1 Dec 2025 15:42:33 -0700 Subject: [PATCH 037/164] tc --- .../defi/tvl/lido/silver__lido_tvl.sql | 4 +- .../tornado_cash/silver__tornado_cash_tvl.sql | 57 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql diff --git a/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql b/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql index d76c8c60..076e4b1a 100644 --- a/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql +++ b/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql @@ -17,6 +17,7 @@ WITH reads AS ( block_number, block_date, contract_address, + address, result_hex AS amount_hex, IFNULL( CASE @@ -40,13 +41,14 @@ WITH reads AS ( block_number, block_date, contract_address, + address, amount_hex, amount_raw, protocol, version, platform, {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address','platform'] + ['block_date','contract_address','address','platform'] ) }} AS lido_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql new file mode 100644 index 00000000..ae127211 --- /dev/null +++ b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql @@ -0,0 +1,57 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'tornado_cash_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH reads AS ( + + SELECT + block_number, + block_date, + contract_address, + address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + protocol, + version, + platform, + _inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} + WHERE + platform = 'tornado_cash-v1' + AND amount_raw IS NOT NULL + ) + SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS tornado_cash_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + reads From 849648104b61bfceb676b425ea84da7555c60f6f Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 2 Dec 2025 10:46:46 -0700 Subject: [PATCH 038/164] uni v2 --- .../defi/tvl/lido/silver__lido_tvl.sql | 43 +++++--- .../tornado_cash/silver__tornado_cash_tvl.sql | 43 +++++--- .../tvl/uniswap/silver__uniswap_v2_tvl.sql | 100 ++++++++++++++++++ 3 files changed, 152 insertions(+), 34 deletions(-) create mode 100644 models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql diff --git a/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql b/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql index 076e4b1a..b4d1b657 100644 --- a/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql +++ b/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql @@ -36,22 +36,31 @@ WITH reads AS ( WHERE platform = 'lido-v1' AND amount_raw IS NOT NULL - ) + +{% if is_incremental() %} +AND modified_timestamp >= ( SELECT - block_number, - block_date, - contract_address, - address, - amount_hex, - amount_raw, - protocol, - version, - platform, - {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address','address','platform'] - ) }} AS lido_tvl_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id + MAX(modified_timestamp) FROM - reads + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','platform'] + ) }} AS lido_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql index ae127211..ebca365e 100644 --- a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql +++ b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql @@ -36,22 +36,31 @@ WITH reads AS ( WHERE platform = 'tornado_cash-v1' AND amount_raw IS NOT NULL - ) + +{% if is_incremental() %} +AND modified_timestamp >= ( SELECT - block_number, - block_date, - contract_address, - address, - amount_hex, - amount_raw, - protocol, - version, - platform, - {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address','address','platform'] - ) }} AS tornado_cash_tvl_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id + MAX(modified_timestamp) FROM - reads + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS tornado_cash_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql new file mode 100644 index 00000000..354ff785 --- /dev/null +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql @@ -0,0 +1,100 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'uniswap_v2_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH reads AS ( + + SELECT + r.block_number, + r.block_date, + r.contract_address, + r.address, + regexp_substr_all(SUBSTR(result_hex, 3, len(result_hex)), '.{64}') AS segmented_data, + segmented_data [0] :: STRING AS reserve_0_hex, + segmented_data [1] :: STRING AS reserve_1_hex, + segmented_data [2] :: STRING AS block_timestamp_last_hex, + IFNULL( + CASE + WHEN LENGTH(reserve_0_hex) <= 4300 + AND reserve_0_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(reserve_0_hex) AS FLOAT)END, + CASE + WHEN reserve_0_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(reserve_0_hex, '0')) AS FLOAT) + END + ) AS reserve_0_raw, + IFNULL( + CASE + WHEN LENGTH(reserve_1_hex) <= 4300 + AND reserve_1_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(reserve_1_hex) AS FLOAT)END, + CASE + WHEN reserve_1_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(reserve_1_hex, '0')) AS FLOAT) + END + ) AS reserve_1_raw, + IFNULL( + CASE + WHEN LENGTH(block_timestamp_last_hex) <= 4300 + AND block_timestamp_last_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(block_timestamp_last_hex) AS TIMESTAMP)END, + CASE + WHEN block_timestamp_last_hex IS NOT NULL THEN TRY_CAST( + utils.udf_hex_to_int(RTRIM(block_timestamp_last_hex, '0')) AS TIMESTAMP + ) + END + ) AS block_timestamp_last_raw, + p.token0 AS token_0_address, + p.token1 AS token_1_address, + r.protocol, + r.version, + r.platform, + r._inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} + r + INNER JOIN {{ ref('silver_dex__paircreated_evt_v2_pools') }} + p + ON r.contract_address = p.pool_address + WHERE + reserve_0_raw IS NOT NULL + AND reserve_1_raw IS NOT NULL + AND block_timestamp_last_raw IS NOT NULL + +{% if is_incremental() %} +AND r.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + reserve_0_hex, + reserve_1_hex, + block_timestamp_last_hex, + reserve_0_raw, + reserve_1_raw, + block_timestamp_last_raw, + token_0_address, + token_1_address, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','platform'] + ) }} AS uniswap_v2_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads From f9a44faa60ecb9d4ded1752f30c1b2ee22f67df9 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 2 Dec 2025 13:41:10 -0700 Subject: [PATCH 039/164] metadata str --- .../streamline__contract_reads_daily_realtime_requests.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index e71e1f14..57f1ec73 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -71,7 +71,7 @@ SELECT function_name, function_sig, input, - metadata, + metadata :: STRING AS metadata_str, protocol, version, platform, From 631e33535675d3c6d38e7a734f87b54c067ca7dc Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 2 Dec 2025 13:50:38 -0700 Subject: [PATCH 040/164] parse json metadata --- .../contract_reads/silver/silver__contract_reads.sql | 4 ++-- .../complete/streamline__contract_reads_daily_complete.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/models/curated_package/contract_reads/silver/silver__contract_reads.sql b/models/curated_package/contract_reads/silver/silver__contract_reads.sql index 994e9cf2..1c9d3a45 100644 --- a/models/curated_package/contract_reads/silver/silver__contract_reads.sql +++ b/models/curated_package/contract_reads/silver/silver__contract_reads.sql @@ -23,8 +23,8 @@ SELECT VALUE :"PROTOCOL" :: STRING AS protocol, VALUE :"VERSION" :: STRING AS version, VALUE :"PLATFORM" :: STRING AS platform, - ( - VALUE :"METADATA" :: STRING + PARSE_JSON( + VALUE :"METADATA_STR" :: STRING ) :: variant AS metadata, VALUE :"FUNCTION_NAME" :: STRING AS function_name, VALUE :"FUNCTION_SIG" :: STRING AS function_sig, diff --git a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql index 21ce6fd0..bb4bd469 100644 --- a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql +++ b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql @@ -26,8 +26,8 @@ SELECT VALUE :"FUNCTION_NAME" :: STRING AS function_name, VALUE :"FUNCTION_SIG" :: STRING AS function_sig, VALUE :"INPUT" :: STRING AS input, - ( - VALUE :"METADATA" :: STRING + PARSE_JSON( + VALUE :"METADATA_STR" :: STRING ) :: variant AS metadata, VALUE :"PROTOCOL" :: STRING AS protocol, VALUE :"VERSION" :: STRING AS version, From 926e36e1d648653146d4c1ac46e9893fab7b5a3f Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 2 Dec 2025 14:14:13 -0700 Subject: [PATCH 041/164] aave uni curve tvl --- .../defi/tvl/aave/silver__aave_v1_tvl.sql | 68 +++++++++++++++++ .../defi/tvl/aave/silver__aave_v2_tvl.sql | 68 +++++++++++++++++ .../defi/tvl/aave/silver__aave_v3_tvl.sql | 68 +++++++++++++++++ .../defi/tvl/curve/silver__curve_v1_tvl.sql | 66 +++++++++++++++++ .../tornado_cash/silver__tornado_cash_tvl.sql | 1 + .../tvl/uniswap/silver__uniswap_v2_tvl.sql | 32 ++++---- .../tvl/uniswap/silver__uniswap_v3_tvl.sql | 73 ++++++++++++++++++ .../tvl/uniswap/silver__uniswap_v4_tvl.sql | 74 +++++++++++++++++++ 8 files changed, 434 insertions(+), 16 deletions(-) create mode 100644 models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql create mode 100644 models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql create mode 100644 models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql create mode 100644 models/curated_package/defi/tvl/curve/silver__curve_v1_tvl.sql create mode 100644 models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql create mode 100644 models/curated_package/defi/tvl/uniswap/silver__uniswap_v4_tvl.sql diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql b/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql new file mode 100644 index 00000000..00cde87f --- /dev/null +++ b/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql @@ -0,0 +1,68 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'aave_v1_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} +WITH reads AS ( + + SELECT + C.block_number, + C.block_date, + C.contract_address, + C.address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + C.protocol, + C.version, + C.platform, + C._inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} C + LEFT JOIN {{ ref('silver_reads__aave_v1_reads') }} + r + ON C.contract_address = r.contract_address + WHERE + r.contract_address IS NOT NULL + AND amount_raw IS NOT NULL + +{% if is_incremental() %} +AND C.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS aave_v1_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql b/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql new file mode 100644 index 00000000..7a4b2522 --- /dev/null +++ b/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql @@ -0,0 +1,68 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'aave_v2_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} +WITH reads AS ( + + SELECT + C.block_number, + C.block_date, + C.contract_address, + C.address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + C.protocol, + C.version, + C.platform, + C._inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} C + LEFT JOIN {{ ref('silver_reads__aave_v2_reads') }} + r + ON C.contract_address = r.contract_address + WHERE + r.contract_address IS NOT NULL + AND amount_raw IS NOT NULL + +{% if is_incremental() %} +AND C.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','platform'] + ) }} AS aave_v2_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql b/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql new file mode 100644 index 00000000..4ca74b80 --- /dev/null +++ b/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql @@ -0,0 +1,68 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'aave_v3_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} +WITH reads AS ( + + SELECT + C.block_number, + C.block_date, + C.contract_address, + C.address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + C.protocol, + C.version, + C.platform, + C._inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} C + LEFT JOIN {{ ref('silver_reads__aave_v3_reads') }} + r + ON C.contract_address = r.contract_address + WHERE + r.contract_address IS NOT NULL + AND amount_raw IS NOT NULL + +{% if is_incremental() %} +AND C.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','platform'] + ) }} AS aave_v3_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads diff --git a/models/curated_package/defi/tvl/curve/silver__curve_v1_tvl.sql b/models/curated_package/defi/tvl/curve/silver__curve_v1_tvl.sql new file mode 100644 index 00000000..e1545126 --- /dev/null +++ b/models/curated_package/defi/tvl/curve/silver__curve_v1_tvl.sql @@ -0,0 +1,66 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'curve_v1_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH reads AS ( + + SELECT + block_number, + block_date, + contract_address, + address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + protocol, + version, + platform, + _inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} + WHERE + platform = 'curve-v1' + AND amount_raw IS NOT NULL + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS curve_v1_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql index ebca365e..c49bbca3 100644 --- a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql +++ b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql @@ -64,3 +64,4 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM reads + --need to join balances diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql index 354ff785..3fb1394f 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql @@ -14,10 +14,10 @@ WITH reads AS ( SELECT - r.block_number, - r.block_date, - r.contract_address, - r.address, + C.block_number, + block_date, + C.contract_address, + C.address, regexp_substr_all(SUBSTR(result_hex, 3, len(result_hex)), '.{64}') AS segmented_data, segmented_data [0] :: STRING AS reserve_0_hex, segmented_data [1] :: STRING AS reserve_1_hex, @@ -48,25 +48,25 @@ WITH reads AS ( ) END ) AS block_timestamp_last_raw, - p.token0 AS token_0_address, - p.token1 AS token_1_address, - r.protocol, - r.version, - r.platform, - r._inserted_timestamp + metadata :token0 :: STRING AS token_0_address, + metadata :token1 :: STRING AS token_1_address, + C.protocol, + C.version, + C.platform, + C._inserted_timestamp FROM - {{ ref('silver__contract_reads') }} + {{ ref('silver__contract_reads') }} C + LEFT JOIN {{ ref('silver_reads__uniswap_v2_reads') }} r - INNER JOIN {{ ref('silver_dex__paircreated_evt_v2_pools') }} - p - ON r.contract_address = p.pool_address + ON C.contract_address = r.contract_address WHERE - reserve_0_raw IS NOT NULL + r.contract_address IS NOT NULL + AND reserve_0_raw IS NOT NULL AND reserve_1_raw IS NOT NULL AND block_timestamp_last_raw IS NOT NULL {% if is_incremental() %} -AND r.modified_timestamp >= ( +AND C.modified_timestamp >= ( SELECT MAX(modified_timestamp) FROM diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql new file mode 100644 index 00000000..d2f41dad --- /dev/null +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql @@ -0,0 +1,73 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'uniswap_v3_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH reads AS ( + + SELECT + C.block_number, + block_date, + C.contract_address, + C.address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + metadata :token0 :: STRING AS token_0_address, + metadata :token1 :: STRING AS token_1_address, + C.protocol, + C.version, + C.platform, + C._inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} C + LEFT JOIN {{ ref('silver_reads__uniswap_v3_reads') }} + r + ON C.contract_address = r.contract_address + WHERE + r.contract_address IS NOT NULL + AND amount_raw IS NOT NULL + +{% if is_incremental() %} +AND C.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + token_0_address, + token_1_address, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS uniswap_v3_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v4_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v4_tvl.sql new file mode 100644 index 00000000..524ce36f --- /dev/null +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v4_tvl.sql @@ -0,0 +1,74 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'uniswap_v4_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH reads AS ( + + SELECT + block_number, + block_date, + contract_address, + address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + metadata :address_type :: STRING AS address_type, + metadata :hook_address :: STRING AS hook_address, + metadata :token0 :: STRING AS token_0_address, + metadata :token1 :: STRING AS token_1_address, + protocol, + version, + platform, + _inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} + WHERE + platform = 'uniswap-v4' + AND amount_raw IS NOT NULL + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + address_type, + hook_address, + token_0_address, + token_1_address, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS uniswap_v4_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads From 0ac4b98d4c79c64b913188bab6bbe7dddb0eeb6b Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 2 Dec 2025 14:26:24 -0700 Subject: [PATCH 042/164] uni v1 --- .../tvl/uniswap/silver__uniswap_v1_tvl.sql | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql new file mode 100644 index 00000000..aff79197 --- /dev/null +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql @@ -0,0 +1,61 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'uniswap_v1_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH balances AS ( + + SELECT + b.block_number, + b.block_date, + b.address AS contract_address, + balance_hex, + balance_raw, + p.protocol, + p.version, + p.platform + FROM + {{ ref('balances__ez_balances_native_daily') }} + b + LEFT JOIN {{ ref('silver_dex__uniswap_v1_pools') }} + p + ON b.address = p.pool_address + WHERE + p.address IS NOT NULL + AND balance_raw IS NOT NULL + +{% if is_incremental() %} +AND b.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + NULL AS address, + balance_hex, + balance_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','platform'] + ) }} AS uniswap_v1_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + balances From 0cf469cdc560db3906c7a4344f4d66572b91505c Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 2 Dec 2025 14:33:44 -0700 Subject: [PATCH 043/164] tc tvl --- .../tornado_cash/silver__tornado_cash_tvl.sql | 65 ++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql index c49bbca3..a15828fb 100644 --- a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql +++ b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql @@ -45,6 +45,68 @@ AND modified_timestamp >= ( {{ this }} ) {% endif %} +), +balances AS ( + SELECT + b.block_number, + b.block_date, + b.address AS contract_address, + NULL AS address, + balance_hex AS amount_hex, + balance_raw AS amount_raw, + 'tornado_cash' AS protocol, + 'v1' AS version, + CONCAT( + protocol, + '-', + version + ) AS platform, + b.modified_timestamp AS _inserted_timestamp + FROM + {{ ref('balances__ez_balances_native_daily') }} + b + LEFT JOIN {{ ref('silver_reads__tornado_cash_mixer_seed') }} + t + ON b.address = t.mixer_address + WHERE + t.mixer_address IS NOT NULL + AND balance_raw IS NOT NULL + +{% if is_incremental() %} +AND b.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +FINAL AS ( + SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform + FROM + reads + UNION ALL + SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform + FROM + balances ) SELECT block_number, @@ -63,5 +125,4 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads - --need to join balances + FINAL From b4ebe62bf77aa9d65976aaf3dd7457322946dbf2 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 2 Dec 2025 14:47:59 -0700 Subject: [PATCH 044/164] alias --- .../defi/tvl/uniswap/silver__uniswap_v1_tvl.sql | 2 +- .../defi/tvl/uniswap/silver__uniswap_v2_tvl.sql | 4 ++-- .../defi/tvl/uniswap/silver__uniswap_v3_tvl.sql | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql index aff79197..5e05bb5b 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql @@ -29,7 +29,7 @@ WITH balances AS ( p ON b.address = p.pool_address WHERE - p.address IS NOT NULL + p.pool_address IS NOT NULL AND balance_raw IS NOT NULL {% if is_incremental() %} diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql index 3fb1394f..3e9f36f1 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql @@ -48,8 +48,8 @@ WITH reads AS ( ) END ) AS block_timestamp_last_raw, - metadata :token0 :: STRING AS token_0_address, - metadata :token1 :: STRING AS token_1_address, + C.metadata :token0 :: STRING AS token_0_address, + C.metadata :token1 :: STRING AS token_1_address, C.protocol, C.version, C.platform, diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql index d2f41dad..ea390b26 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql @@ -27,8 +27,8 @@ WITH reads AS ( WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) END ) AS amount_raw, - metadata :token0 :: STRING AS token_0_address, - metadata :token1 :: STRING AS token_1_address, + C.metadata :token0 :: STRING AS token_0_address, + C.metadata :token1 :: STRING AS token_1_address, C.protocol, C.version, C.platform, From ca6240bc529ca5edd0cc96fa4da354c6243a8f5b Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 11:32:49 -0700 Subject: [PATCH 045/164] complete tvl --- .../defi/tvl/silver__complete_tvl.sql | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 models/curated_package/defi/tvl/silver__complete_tvl.sql diff --git a/models/curated_package/defi/tvl/silver__complete_tvl.sql b/models/curated_package/defi/tvl/silver__complete_tvl.sql new file mode 100644 index 00000000..3a9bba53 --- /dev/null +++ b/models/curated_package/defi/tvl/silver__complete_tvl.sql @@ -0,0 +1,196 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_tvl_id', + tags = ['silver','defi','tvl','complete','curated_daily'] +) }} + +WITH contracts AS ( + + SELECT + address AS contract_address, + symbol AS token_symbol, + decimals AS token_decimals, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__dim_contracts') }} + UNION ALL + SELECT + '0x0000000000000000000000000000000000000000' AS contract_address, + '{{ vars.GLOBAL_NATIVE_ASSET_SYMBOL }}' AS token_symbol, + decimals AS token_decimals, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__dim_contracts') }} + WHERE + address = '{{ vars.GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS }}' +), +prices AS ( + SELECT + token_address, + price, + HOUR, + is_verified, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('price__ez_prices_hourly') }} + UNION ALL + SELECT + '0x0000000000000000000000000000000000000000' AS token_address, + price, + HOUR, + is_verified, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('price__ez_prices_hourly') }} + WHERE + token_address = '{{ vars.GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS }}' +), +aave_v1 AS ( + +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform +FROM + {{ ref('silver__aave_v1_tvl') }} +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} +), +aave_v2 AS ( +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform +FROM + {{ ref('silver__aave_v2_tvl') }} +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} +), +aave_v3 AS ( +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform +FROM + {{ ref('silver__aave_v3_tvl') }} +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} +), +all_tvl AS ( + SELECT + * + FROM + aave_v1 + UNION ALL + SELECT + * + FROM + aave_v2 + UNION ALL + SELECT + * + FROM + aave_v3 +), +final AS ( + SELECT + block_number, + block_date, + contract_address, + address, + c1.token_decimals AS decimals, + c1.token_symbol AS symbol, + amount_hex, + amount_raw, + utils.udf_decimal_adjust( + amount_raw, + decimals + ) AS amount_precise, + amount_precise :: FLOAT AS amount, + ROUND(amount * p1.price, 2) AS amount_usd, + protocol, + version, + platform + FROM + all_tvl a + LEFT JOIN contracts + c1 + ON a.contract_address = c1.contract_address + LEFT JOIN prices + p1 + ON a.contract_address = p1.token_address + AND DATEADD( + 'hour', + 23, + a.block_date + ) = p1.hour +) +SELECT + block_number, + block_date, + contract_address, + address, + decimals, + symbol, + amount_hex, + amount_raw, + amount_precise, + amount, + amount_usd, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS complete_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + final \ No newline at end of file From 722ad4b88610a2a7a9beb171da6f00c71d283858 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 11:36:56 -0700 Subject: [PATCH 046/164] alias --- .../defi/tvl/silver__complete_tvl.sql | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/models/curated_package/defi/tvl/silver__complete_tvl.sql b/models/curated_package/defi/tvl/silver__complete_tvl.sql index 3a9bba53..dd7196fc 100644 --- a/models/curated_package/defi/tvl/silver__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver__complete_tvl.sql @@ -52,8 +52,7 @@ prices AS ( WHERE token_address = '{{ vars.GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS }}' ), -aave_v1 AS ( - +aave_v1 AS ( SELECT block_number, block_date, @@ -140,10 +139,10 @@ all_tvl AS ( ), final AS ( SELECT - block_number, - block_date, - contract_address, - address, + a.block_number, + a.block_date, + a.contract_address, + a.address, c1.token_decimals AS decimals, c1.token_symbol AS symbol, amount_hex, @@ -154,9 +153,9 @@ final AS ( ) AS amount_precise, amount_precise :: FLOAT AS amount, ROUND(amount * p1.price, 2) AS amount_usd, - protocol, - version, - platform + a.protocol, + a.version, + a.platform FROM all_tvl a LEFT JOIN contracts From 07fe4b312420d6d45706fc6c2b1812a79b61f6c5 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 12:41:55 -0700 Subject: [PATCH 047/164] join key --- models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql | 3 ++- models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql | 3 ++- models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql | 3 ++- .../defi/tvl/uniswap/silver__uniswap_v1_tvl.sql | 3 ++- .../defi/tvl/uniswap/silver__uniswap_v2_tvl.sql | 3 ++- .../defi/tvl/uniswap/silver__uniswap_v3_tvl.sql | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql b/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql index 00cde87f..53a51e85 100644 --- a/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql +++ b/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql @@ -35,8 +35,9 @@ WITH reads AS ( LEFT JOIN {{ ref('silver_reads__aave_v1_reads') }} r ON C.contract_address = r.contract_address + AND C.platform = r.platform WHERE - r.contract_address IS NOT NULL + r.platform IS NOT NULL AND amount_raw IS NOT NULL {% if is_incremental() %} diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql b/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql index 7a4b2522..d42db63f 100644 --- a/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql +++ b/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql @@ -35,8 +35,9 @@ WITH reads AS ( LEFT JOIN {{ ref('silver_reads__aave_v2_reads') }} r ON C.contract_address = r.contract_address + AND C.platform = r.platform WHERE - r.contract_address IS NOT NULL + r.platform IS NOT NULL AND amount_raw IS NOT NULL {% if is_incremental() %} diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql b/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql index 4ca74b80..33b92b98 100644 --- a/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql +++ b/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql @@ -35,8 +35,9 @@ WITH reads AS ( LEFT JOIN {{ ref('silver_reads__aave_v3_reads') }} r ON C.contract_address = r.contract_address + AND C.platform = r.platform WHERE - r.contract_address IS NOT NULL + r.platform IS NOT NULL AND amount_raw IS NOT NULL {% if is_incremental() %} diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql index 5e05bb5b..3c6e06f7 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql @@ -28,8 +28,9 @@ WITH balances AS ( LEFT JOIN {{ ref('silver_dex__uniswap_v1_pools') }} p ON b.address = p.pool_address + AND b.platform = p.platform WHERE - p.pool_address IS NOT NULL + p.platform IS NOT NULL AND balance_raw IS NOT NULL {% if is_incremental() %} diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql index 3e9f36f1..294cdd1b 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql @@ -59,8 +59,9 @@ WITH reads AS ( LEFT JOIN {{ ref('silver_reads__uniswap_v2_reads') }} r ON C.contract_address = r.contract_address + AND C.platform = r.platform WHERE - r.contract_address IS NOT NULL + r.platform IS NOT NULL AND reserve_0_raw IS NOT NULL AND reserve_1_raw IS NOT NULL AND block_timestamp_last_raw IS NOT NULL diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql index ea390b26..301fb860 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql @@ -38,8 +38,9 @@ WITH reads AS ( LEFT JOIN {{ ref('silver_reads__uniswap_v3_reads') }} r ON C.contract_address = r.contract_address + AND C.platform = r.platform WHERE - r.contract_address IS NOT NULL + r.platform IS NOT NULL AND amount_raw IS NOT NULL {% if is_incremental() %} From 1a0407b8b0718f30c373acabb79a28a04462f02b Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 12:47:18 -0700 Subject: [PATCH 048/164] revert uni v1 --- .../defi/tvl/uniswap/silver__uniswap_v1_tvl.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql index 3c6e06f7..5e05bb5b 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql @@ -28,9 +28,8 @@ WITH balances AS ( LEFT JOIN {{ ref('silver_dex__uniswap_v1_pools') }} p ON b.address = p.pool_address - AND b.platform = p.platform WHERE - p.platform IS NOT NULL + p.pool_address IS NOT NULL AND balance_raw IS NOT NULL {% if is_incremental() %} From 356ba53716b0153515ba34315b18eb9b6fa61399 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 12:54:57 -0700 Subject: [PATCH 049/164] fix dupes --- .../defi/tvl/aave/silver__aave_v1_tvl.sql | 36 +++++++++---------- .../defi/tvl/aave/silver__aave_v2_tvl.sql | 36 +++++++++---------- .../defi/tvl/aave/silver__aave_v3_tvl.sql | 36 +++++++++---------- .../defi/tvl/curve/silver__curve_v1_tvl.sql | 2 +- .../defi/tvl/lido/silver__lido_tvl.sql | 2 +- .../tornado_cash/silver__tornado_cash_tvl.sql | 2 +- .../tvl/uniswap/silver__uniswap_v1_tvl.sql | 2 +- .../tvl/uniswap/silver__uniswap_v2_tvl.sql | 33 +++++++++-------- .../tvl/uniswap/silver__uniswap_v3_tvl.sql | 33 +++++++++-------- .../tvl/uniswap/silver__uniswap_v4_tvl.sql | 2 +- 10 files changed, 91 insertions(+), 93 deletions(-) diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql b/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql index 53a51e85..ed0eaff7 100644 --- a/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql +++ b/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql @@ -1,22 +1,21 @@ {# Get variables #} {% set vars = return_vars() %} - {# Log configuration details #} {{ log_model_details() }} - {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'aave_v1_tvl_id', tags = ['silver','defi','tvl','curated_daily'] ) }} + WITH reads AS ( SELECT - C.block_number, - C.block_date, - C.contract_address, - C.address, + block_number, + block_date, + contract_address, + address, result_hex AS amount_hex, IFNULL( CASE @@ -26,22 +25,23 @@ WITH reads AS ( WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) END ) AS amount_raw, - C.protocol, - C.version, - C.platform, - C._inserted_timestamp + protocol, + version, + platform, + _inserted_timestamp FROM - {{ ref('silver__contract_reads') }} C - LEFT JOIN {{ ref('silver_reads__aave_v1_reads') }} - r - ON C.contract_address = r.contract_address - AND C.platform = r.platform + {{ ref('silver__contract_reads') }} WHERE - r.platform IS NOT NULL - AND amount_raw IS NOT NULL + amount_raw IS NOT NULL + AND platform IN ( + SELECT + DISTINCT platform + FROM + {{ ref('silver_reads__aave_v1_reads') }} + ) {% if is_incremental() %} -AND C.modified_timestamp >= ( +AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql b/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql index d42db63f..395afdd7 100644 --- a/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql +++ b/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql @@ -1,22 +1,21 @@ {# Get variables #} {% set vars = return_vars() %} - {# Log configuration details #} {{ log_model_details() }} - {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'aave_v2_tvl_id', tags = ['silver','defi','tvl','curated_daily'] ) }} + WITH reads AS ( SELECT - C.block_number, - C.block_date, - C.contract_address, - C.address, + block_number, + block_date, + contract_address, + address, result_hex AS amount_hex, IFNULL( CASE @@ -26,22 +25,23 @@ WITH reads AS ( WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) END ) AS amount_raw, - C.protocol, - C.version, - C.platform, - C._inserted_timestamp + protocol, + version, + platform, + _inserted_timestamp FROM - {{ ref('silver__contract_reads') }} C - LEFT JOIN {{ ref('silver_reads__aave_v2_reads') }} - r - ON C.contract_address = r.contract_address - AND C.platform = r.platform + {{ ref('silver__contract_reads') }} WHERE - r.platform IS NOT NULL - AND amount_raw IS NOT NULL + amount_raw IS NOT NULL + AND platform IN ( + SELECT + DISTINCT platform + FROM + {{ ref('silver_reads__aave_v2_reads') }} + ) {% if is_incremental() %} -AND C.modified_timestamp >= ( +AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql b/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql index 33b92b98..f2d266d9 100644 --- a/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql +++ b/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql @@ -1,22 +1,21 @@ {# Get variables #} {% set vars = return_vars() %} - {# Log configuration details #} {{ log_model_details() }} - {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'aave_v3_tvl_id', tags = ['silver','defi','tvl','curated_daily'] ) }} + WITH reads AS ( SELECT - C.block_number, - C.block_date, - C.contract_address, - C.address, + block_number, + block_date, + contract_address, + address, result_hex AS amount_hex, IFNULL( CASE @@ -26,22 +25,23 @@ WITH reads AS ( WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) END ) AS amount_raw, - C.protocol, - C.version, - C.platform, - C._inserted_timestamp + protocol, + version, + platform, + _inserted_timestamp FROM - {{ ref('silver__contract_reads') }} C - LEFT JOIN {{ ref('silver_reads__aave_v3_reads') }} - r - ON C.contract_address = r.contract_address - AND C.platform = r.platform + {{ ref('silver__contract_reads') }} WHERE - r.platform IS NOT NULL - AND amount_raw IS NOT NULL + amount_raw IS NOT NULL + AND platform IN ( + SELECT + DISTINCT platform + FROM + {{ ref('silver_reads__aave_v3_reads') }} + ) {% if is_incremental() %} -AND C.modified_timestamp >= ( +AND modified_timestamp >= ( SELECT MAX(modified_timestamp) FROM diff --git a/models/curated_package/defi/tvl/curve/silver__curve_v1_tvl.sql b/models/curated_package/defi/tvl/curve/silver__curve_v1_tvl.sql index e1545126..08177289 100644 --- a/models/curated_package/defi/tvl/curve/silver__curve_v1_tvl.sql +++ b/models/curated_package/defi/tvl/curve/silver__curve_v1_tvl.sql @@ -38,7 +38,7 @@ WITH reads AS ( AND amount_raw IS NOT NULL {% if is_incremental() %} -AND modified_timestamp >= ( +AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM diff --git a/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql b/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql index b4d1b657..5d4bd748 100644 --- a/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql +++ b/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql @@ -38,7 +38,7 @@ WITH reads AS ( AND amount_raw IS NOT NULL {% if is_incremental() %} -AND modified_timestamp >= ( +AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql index a15828fb..d067f0a1 100644 --- a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql +++ b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql @@ -73,7 +73,7 @@ balances AS ( AND balance_raw IS NOT NULL {% if is_incremental() %} -AND b.modified_timestamp >= ( +AND b.modified_timestamp > ( SELECT MAX(modified_timestamp) FROM diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql index 5e05bb5b..f77508d0 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql @@ -33,7 +33,7 @@ WITH balances AS ( AND balance_raw IS NOT NULL {% if is_incremental() %} -AND b.modified_timestamp >= ( +AND b.modified_timestamp > ( SELECT MAX(modified_timestamp) FROM diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql index 294cdd1b..ca9a981a 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql @@ -14,10 +14,10 @@ WITH reads AS ( SELECT - C.block_number, + block_number, block_date, - C.contract_address, - C.address, + contract_address, + address, regexp_substr_all(SUBSTR(result_hex, 3, len(result_hex)), '.{64}') AS segmented_data, segmented_data [0] :: STRING AS reserve_0_hex, segmented_data [1] :: STRING AS reserve_1_hex, @@ -48,26 +48,25 @@ WITH reads AS ( ) END ) AS block_timestamp_last_raw, - C.metadata :token0 :: STRING AS token_0_address, - C.metadata :token1 :: STRING AS token_1_address, - C.protocol, - C.version, - C.platform, - C._inserted_timestamp + metadata :token0 :: STRING AS token_0_address, + metadata :token1 :: STRING AS token_1_address, + protocol, + version, + platform, + _inserted_timestamp FROM - {{ ref('silver__contract_reads') }} C - LEFT JOIN {{ ref('silver_reads__uniswap_v2_reads') }} - r - ON C.contract_address = r.contract_address - AND C.platform = r.platform + {{ ref('silver__contract_reads') }} WHERE - r.platform IS NOT NULL - AND reserve_0_raw IS NOT NULL + reserve_0_raw IS NOT NULL AND reserve_1_raw IS NOT NULL AND block_timestamp_last_raw IS NOT NULL + AND platform IN ( + SELECT DISTINCT platform + FROM {{ ref('silver_reads__uniswap_v2_reads') }} + ) {% if is_incremental() %} -AND C.modified_timestamp >= ( +AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql index 301fb860..333f609f 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql @@ -14,10 +14,10 @@ WITH reads AS ( SELECT - C.block_number, + block_number, block_date, - C.contract_address, - C.address, + contract_address, + address, result_hex AS amount_hex, IFNULL( CASE @@ -27,24 +27,23 @@ WITH reads AS ( WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) END ) AS amount_raw, - C.metadata :token0 :: STRING AS token_0_address, - C.metadata :token1 :: STRING AS token_1_address, - C.protocol, - C.version, - C.platform, - C._inserted_timestamp + metadata :token0 :: STRING AS token_0_address, + metadata :token1 :: STRING AS token_1_address, + protocol, + version, + platform, + _inserted_timestamp FROM - {{ ref('silver__contract_reads') }} C - LEFT JOIN {{ ref('silver_reads__uniswap_v3_reads') }} - r - ON C.contract_address = r.contract_address - AND C.platform = r.platform + {{ ref('silver__contract_reads') }} WHERE - r.platform IS NOT NULL - AND amount_raw IS NOT NULL + amount_raw IS NOT NULL + AND platform IN ( + SELECT DISTINCT platform + FROM {{ ref('silver_reads__uniswap_v3_reads') }} + ) {% if is_incremental() %} -AND C.modified_timestamp >= ( +AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v4_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v4_tvl.sql index 524ce36f..33f88450 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v4_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v4_tvl.sql @@ -42,7 +42,7 @@ WITH reads AS ( AND amount_raw IS NOT NULL {% if is_incremental() %} -AND modified_timestamp >= ( +AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM From 6c43049adec2722a512d735baed871c867821cb9 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 13:28:38 -0700 Subject: [PATCH 050/164] tvl by address --- .../defi/tvl/silver__complete_tvl.sql | 173 +++++------------- .../tvl/uniswap/silver__uniswap_v2_tvl.sql | 41 +++-- 2 files changed, 72 insertions(+), 142 deletions(-) diff --git a/models/curated_package/defi/tvl/silver__complete_tvl.sql b/models/curated_package/defi/tvl/silver__complete_tvl.sql index dd7196fc..f39bc4e2 100644 --- a/models/curated_package/defi/tvl/silver__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver__complete_tvl.sql @@ -11,131 +11,42 @@ tags = ['silver','defi','tvl','complete','curated_daily'] ) }} -WITH contracts AS ( +{% set models = [] %} +{% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} + {% set _ = models.append(ref('silver__lido_tvl')) %} +{% endif %} +{% set _ = models.append(ref('silver__uniswap_v2_tvl')) %} +{% set _ = models.append(ref('silver__uniswap_v3_tvl')) %} +{% set _ = models.append(ref('silver__uniswap_v4_tvl')) %} +{% set _ = models.append(ref('silver__aave_v1_tvl')) %} +{% set _ = models.append(ref('silver__aave_v2_tvl')) %} +{% set _ = models.append(ref('silver__aave_v3_tvl')) %} +{% set _ = models.append(ref('silver__curve_tvl')) %} +{% set _ = models.append(ref('silver__tornado_cash_tvl')) %} - SELECT - address AS contract_address, - symbol AS token_symbol, - decimals AS token_decimals, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('core__dim_contracts') }} - UNION ALL - SELECT - '0x0000000000000000000000000000000000000000' AS contract_address, - '{{ vars.GLOBAL_NATIVE_ASSET_SYMBOL }}' AS token_symbol, - decimals AS token_decimals, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('core__dim_contracts') }} - WHERE - address = '{{ vars.GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS }}' -), -prices AS ( - SELECT - token_address, - price, - HOUR, - is_verified, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('price__ez_prices_hourly') }} - UNION ALL - SELECT - '0x0000000000000000000000000000000000000000' AS token_address, - price, - HOUR, - is_verified, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('price__ez_prices_hourly') }} - WHERE - token_address = '{{ vars.GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS }}' -), -aave_v1 AS ( -SELECT - block_number, - block_date, - contract_address, - address, - amount_hex, - amount_raw, - protocol, - version, - platform -FROM - {{ ref('silver__aave_v1_tvl') }} -{% if is_incremental() %} -WHERE - modified_timestamp > ( +WITH all_tvl AS ( + {% for model in models %} SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -aave_v2 AS ( -SELECT - block_number, - block_date, - contract_address, - address, - amount_hex, - amount_raw, - protocol, - version, - platform -FROM - {{ ref('silver__aave_v2_tvl') }} -{% if is_incremental() %} -WHERE - modified_timestamp > ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -aave_v3 AS ( -SELECT - block_number, - block_date, - contract_address, - address, - amount_hex, - amount_raw, - protocol, - version, - platform -FROM - {{ ref('silver__aave_v3_tvl') }} -{% if is_incremental() %} -WHERE - modified_timestamp > ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -all_tvl AS ( - SELECT - * - FROM - aave_v1 - UNION ALL - SELECT - * - FROM - aave_v2 - UNION ALL - SELECT - * - FROM - aave_v3 + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform + FROM {{ model }} + {% if not loop.last %} + {% if is_incremental() %} + WHERE modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} + UNION ALL + {% endif %} + {% endfor %} ), final AS ( SELECT @@ -143,13 +54,13 @@ final AS ( a.block_date, a.contract_address, a.address, - c1.token_decimals AS decimals, - c1.token_symbol AS symbol, - amount_hex, - amount_raw, + c1.decimals, + c1.symbol, + a.amount_hex, + a.amount_raw, utils.udf_decimal_adjust( - amount_raw, - decimals + a.amount_raw, + c1.decimals ) AS amount_precise, amount_precise :: FLOAT AS amount, ROUND(amount * p1.price, 2) AS amount_usd, @@ -158,10 +69,10 @@ final AS ( a.platform FROM all_tvl a - LEFT JOIN contracts + LEFT JOIN {{ ref('core__dim_contracts') }} c1 ON a.contract_address = c1.contract_address - LEFT JOIN prices + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p1 ON a.contract_address = p1.token_address AND DATEADD( diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql index ca9a981a..4e2aa2c0 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql @@ -16,8 +16,8 @@ WITH reads AS ( SELECT block_number, block_date, - contract_address, - address, + contract_address AS pool_address, + address, --NULL regexp_substr_all(SUBSTR(result_hex, 3, len(result_hex)), '.{64}') AS segmented_data, segmented_data [0] :: STRING AS reserve_0_hex, segmented_data [1] :: STRING AS reserve_1_hex, @@ -73,20 +73,39 @@ AND modified_timestamp > ( {{ this }} ) {% endif %} +), +all_reads AS ( +SELECT + block_number, + block_date, + token_0_address AS contract_address, + pool_address AS address, + reserve_0_hex AS amount_hex, + reserve_0_raw AS amount_raw, + protocol, + version, + platform +FROM reads +UNION ALL +SELECT + block_number, + block_date, + token_1_address AS contract_address, + pool_address AS address, + reserve_1_hex AS amount_hex, + reserve_1_raw AS amount_raw, + protocol, + version, + platform +FROM reads ) SELECT block_number, block_date, contract_address, address, - reserve_0_hex, - reserve_1_hex, - block_timestamp_last_hex, - reserve_0_raw, - reserve_1_raw, - block_timestamp_last_raw, - token_0_address, - token_1_address, + amount_hex, + amount_raw, protocol, version, platform, @@ -97,4 +116,4 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads + all_reads From d58bb5cbd2acec0bafb42ccfaafdddfd704370d0 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 13:32:49 -0700 Subject: [PATCH 051/164] versions --- .../lido/{silver__lido_tvl.sql => silver__lido_v1_tvl.sql} | 0 models/curated_package/defi/tvl/silver__complete_tvl.sql | 6 +++--- ...tornado_cash_tvl.sql => silver__tornado_cash_v1_tvl.sql} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename models/curated_package/defi/tvl/lido/{silver__lido_tvl.sql => silver__lido_v1_tvl.sql} (100%) rename models/curated_package/defi/tvl/tornado_cash/{silver__tornado_cash_tvl.sql => silver__tornado_cash_v1_tvl.sql} (100%) diff --git a/models/curated_package/defi/tvl/lido/silver__lido_tvl.sql b/models/curated_package/defi/tvl/lido/silver__lido_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/lido/silver__lido_tvl.sql rename to models/curated_package/defi/tvl/lido/silver__lido_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/silver__complete_tvl.sql b/models/curated_package/defi/tvl/silver__complete_tvl.sql index f39bc4e2..9bbae31d 100644 --- a/models/curated_package/defi/tvl/silver__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver__complete_tvl.sql @@ -13,7 +13,7 @@ {% set models = [] %} {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} - {% set _ = models.append(ref('silver__lido_tvl')) %} + {% set _ = models.append(ref('silver__lido_v1_tvl')) %} {% endif %} {% set _ = models.append(ref('silver__uniswap_v2_tvl')) %} {% set _ = models.append(ref('silver__uniswap_v3_tvl')) %} @@ -21,8 +21,8 @@ {% set _ = models.append(ref('silver__aave_v1_tvl')) %} {% set _ = models.append(ref('silver__aave_v2_tvl')) %} {% set _ = models.append(ref('silver__aave_v3_tvl')) %} -{% set _ = models.append(ref('silver__curve_tvl')) %} -{% set _ = models.append(ref('silver__tornado_cash_tvl')) %} +{% set _ = models.append(ref('silver__curve_v1_tvl')) %} +{% set _ = models.append(ref('silver__tornado_cash_v1_tvl')) %} WITH all_tvl AS ( {% for model in models %} diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_tvl.sql rename to models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql From 088d32a505862c9c5545b24e869b89005bf178e2 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 13:37:12 -0700 Subject: [PATCH 052/164] depends --- models/curated_package/defi/tvl/silver__complete_tvl.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/curated_package/defi/tvl/silver__complete_tvl.sql b/models/curated_package/defi/tvl/silver__complete_tvl.sql index 9bbae31d..c6286be9 100644 --- a/models/curated_package/defi/tvl/silver__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver__complete_tvl.sql @@ -4,6 +4,7 @@ {# Log configuration details #} {{ log_model_details() }} +-- depends_on: {{ ref('silver__lido_v1_tvl') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', From f44b6677ae97f54e328b10f7c34b73ff7b585930 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 13:38:23 -0700 Subject: [PATCH 053/164] remove if --- .../curated_package/defi/tvl/silver__complete_tvl.sql | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/models/curated_package/defi/tvl/silver__complete_tvl.sql b/models/curated_package/defi/tvl/silver__complete_tvl.sql index c6286be9..a1ec03ec 100644 --- a/models/curated_package/defi/tvl/silver__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver__complete_tvl.sql @@ -13,17 +13,15 @@ ) }} {% set models = [] %} -{% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} - {% set _ = models.append(ref('silver__lido_v1_tvl')) %} -{% endif %} -{% set _ = models.append(ref('silver__uniswap_v2_tvl')) %} -{% set _ = models.append(ref('silver__uniswap_v3_tvl')) %} -{% set _ = models.append(ref('silver__uniswap_v4_tvl')) %} {% set _ = models.append(ref('silver__aave_v1_tvl')) %} {% set _ = models.append(ref('silver__aave_v2_tvl')) %} {% set _ = models.append(ref('silver__aave_v3_tvl')) %} {% set _ = models.append(ref('silver__curve_v1_tvl')) %} +{% set _ = models.append(ref('silver__lido_v1_tvl')) %} {% set _ = models.append(ref('silver__tornado_cash_v1_tvl')) %} +{% set _ = models.append(ref('silver__uniswap_v2_tvl')) %} +{% set _ = models.append(ref('silver__uniswap_v3_tvl')) %} +{% set _ = models.append(ref('silver__uniswap_v4_tvl')) %} WITH all_tvl AS ( {% for model in models %} From fa202d14e7906da97ea0f50e9f143ecef92dcbb3 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 13:38:36 -0700 Subject: [PATCH 054/164] remove depends --- models/curated_package/defi/tvl/silver__complete_tvl.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/models/curated_package/defi/tvl/silver__complete_tvl.sql b/models/curated_package/defi/tvl/silver__complete_tvl.sql index a1ec03ec..3d60e7a8 100644 --- a/models/curated_package/defi/tvl/silver__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver__complete_tvl.sql @@ -4,7 +4,6 @@ {# Log configuration details #} {{ log_model_details() }} --- depends_on: {{ ref('silver__lido_v1_tvl') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', From ddba9a4fc192906f1f47fb6f1cae8f89582d9f26 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 13:43:49 -0700 Subject: [PATCH 055/164] column name --- models/curated_package/defi/tvl/silver__complete_tvl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/defi/tvl/silver__complete_tvl.sql b/models/curated_package/defi/tvl/silver__complete_tvl.sql index 3d60e7a8..739e06c9 100644 --- a/models/curated_package/defi/tvl/silver__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver__complete_tvl.sql @@ -69,7 +69,7 @@ final AS ( all_tvl a LEFT JOIN {{ ref('core__dim_contracts') }} c1 - ON a.contract_address = c1.contract_address + ON a.contract_address = c1.address LEFT JOIN {{ ref('price__ez_prices_hourly') }} p1 ON a.contract_address = p1.token_address From 127b8900687b7f9420548ad74abb9ca8cf9c27f9 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 14:04:53 -0700 Subject: [PATCH 056/164] chain filter --- .../defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql index d067f0a1..826970ab 100644 --- a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql +++ b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql @@ -71,6 +71,7 @@ balances AS ( WHERE t.mixer_address IS NOT NULL AND balance_raw IS NOT NULL + AND t.chain = '{{ vars.GLOBAL_PROJECT_NAME }}' {% if is_incremental() %} AND b.modified_timestamp > ( From 8f26b59cfaeb2ece20bc6b64ee08dc35756d8d7a Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 14:27:05 -0700 Subject: [PATCH 057/164] tornado cash --- .../defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql index 826970ab..3ce76ca8 100644 --- a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql +++ b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql @@ -70,6 +70,7 @@ balances AS ( ON b.address = t.mixer_address WHERE t.mixer_address IS NOT NULL + AND t.token_address IS NULL AND balance_raw IS NOT NULL AND t.chain = '{{ vars.GLOBAL_PROJECT_NAME }}' From b419c2f30d52732d3a9079347e2292c3b83c70ef Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 14:32:14 -0700 Subject: [PATCH 058/164] 0x00 --- .../silver_reads__tornado_cash_mixer_seed.csv | 60 +++++++++---------- .../defi/tvl/silver__complete_tvl.sql | 53 ++++++++++++++-- .../silver__tornado_cash_v1_tvl.sql | 4 +- 3 files changed, 79 insertions(+), 38 deletions(-) diff --git a/data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.csv b/data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.csv index efa4dbdc..1e4373e2 100644 --- a/data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.csv +++ b/data/curated/contract_reads/silver_reads__tornado_cash_mixer_seed.csv @@ -1,8 +1,8 @@ chain,token_address,mixer_address -ethereum,null,0x12d66f87a04a9e220743712ce6d9bb1b5616b8fc -ethereum,null,0x47ce0c6ed5b0ce3d3a51fdb1c52dc66a7c3c2936 -ethereum,null,0x910cbd523d972eb0a6f4cae4618ad62622b39dbf -ethereum,null,0xa160cdab225685da1d56aa342ad8841c3b53f291 +ethereum,0x0000000000000000000000000000000000000000,0x12d66f87a04a9e220743712ce6d9bb1b5616b8fc +ethereum,0x0000000000000000000000000000000000000000,0x47ce0c6ed5b0ce3d3a51fdb1c52dc66a7c3c2936 +ethereum,0x0000000000000000000000000000000000000000,0x910cbd523d972eb0a6f4cae4618ad62622b39dbf +ethereum,0x0000000000000000000000000000000000000000,0xa160cdab225685da1d56aa342ad8841c3b53f291 ethereum,0x6b175474e89094c44da98b954eedeac495271d0f,0xd4b88df4d29f5cedd6857912842cff3b20c8cfa3 ethereum,0x6b175474e89094c44da98b954eedeac495271d0f,0xfd8610d20aa15b7b2e3be39b396a1bc3516c7144 ethereum,0x6b175474e89094c44da98b954eedeac495271d0f,0x07687e702b410fa43f4cb4af7fa097918ffd2730 @@ -21,29 +21,29 @@ ethereum,0x5d3a536e4d6dbd6114cc1ead35777bab948e3643,0x22aaa7720ddd5388a3c0a33334 ethereum,0x5d3a536e4d6dbd6114cc1ead35777bab948e3643,0x03893a7c7463ae47d46bc7f091665f1893656003 ethereum,0x5d3a536e4d6dbd6114cc1ead35777bab948e3643,0x2717c5e28cf931547b621a5dddb772ab6a35b701 ethereum,0x5d3a536e4d6dbd6114cc1ead35777bab948e3643,0xd21be7248e0197ee08e0c20d4a96debdac3d20af -bsc,null,0x84443cfd09a48af6ef360c6976c5392ac5023a1f -bsc,null,0xd47438c816c9e7f2e2888e060936a499af9582b3 -bsc,null,0x330bdfade01ee9bf63c209ee33102dd334618e0a -bsc,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd -polygon,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd -polygon,null,0xdf231d99ff8b6c6cbf4e9b9a945cbacef9339178 -polygon,null,0xaf4c0b70b2ea9fb7487c7cbb37ada259579fe040 -polygon,null,0xa5c2254e4253490c54cef0a4347fddb8f75a4998 -optimism,null,0x84443cfd09a48af6ef360c6976c5392ac5023a1f -optimism,null,0xd47438c816c9e7f2e2888e060936a499af9582b3 -optimism,null,0x330bdfade01ee9bf63c209ee33102dd334618e0a -optimism,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd -arbitrum,null,0x84443cfd09a48af6ef360c6976c5392ac5023a1f -arbitrum,null,0xd47438c816c9e7f2e2888e060936a499af9582b3 -arbitrum,null,0x330bdfade01ee9bf63c209ee33102dd334618e0a -arbitrum,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd -gnosis,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd -gnosis,null,0xdf231d99ff8b6c6cbf4e9b9a945cbacef9339178 -gnosis,null,0xaf4c0b70b2ea9fb7487c7cbb37ada259579fe040 -gnosis,null,0xa5c2254e4253490c54cef0a4347fddb8f75a4998 -avax,null,0x330bdfade01ee9bf63c209ee33102dd334618e0a -avax,null,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd -avax,null,0xaf8d1839c3c67cf571aa74b5c12398d4901147b3 -ethereumclassic,null,0x2f56d5afc058b8734350b162efee75ee48f034e0 -ethereumclassic,null,0x59fcb629a23e8ed0a60a0188771e221042260118 -ethereumclassic,null,0x784b3a7a7981b959bd8d9d9e73c2013be819fbf2 \ No newline at end of file +bsc,0x0000000000000000000000000000000000000000,0x84443cfd09a48af6ef360c6976c5392ac5023a1f +bsc,0x0000000000000000000000000000000000000000,0xd47438c816c9e7f2e2888e060936a499af9582b3 +bsc,0x0000000000000000000000000000000000000000,0x330bdfade01ee9bf63c209ee33102dd334618e0a +bsc,0x0000000000000000000000000000000000000000,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +polygon,0x0000000000000000000000000000000000000000,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +polygon,0x0000000000000000000000000000000000000000,0xdf231d99ff8b6c6cbf4e9b9a945cbacef9339178 +polygon,0x0000000000000000000000000000000000000000,0xaf4c0b70b2ea9fb7487c7cbb37ada259579fe040 +polygon,0x0000000000000000000000000000000000000000,0xa5c2254e4253490c54cef0a4347fddb8f75a4998 +optimism,0x0000000000000000000000000000000000000000,0x84443cfd09a48af6ef360c6976c5392ac5023a1f +optimism,0x0000000000000000000000000000000000000000,0xd47438c816c9e7f2e2888e060936a499af9582b3 +optimism,0x0000000000000000000000000000000000000000,0x330bdfade01ee9bf63c209ee33102dd334618e0a +optimism,0x0000000000000000000000000000000000000000,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +arbitrum,0x0000000000000000000000000000000000000000,0x84443cfd09a48af6ef360c6976c5392ac5023a1f +arbitrum,0x0000000000000000000000000000000000000000,0xd47438c816c9e7f2e2888e060936a499af9582b3 +arbitrum,0x0000000000000000000000000000000000000000,0x330bdfade01ee9bf63c209ee33102dd334618e0a +arbitrum,0x0000000000000000000000000000000000000000,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +gnosis,0x0000000000000000000000000000000000000000,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +gnosis,0x0000000000000000000000000000000000000000,0xdf231d99ff8b6c6cbf4e9b9a945cbacef9339178 +gnosis,0x0000000000000000000000000000000000000000,0xaf4c0b70b2ea9fb7487c7cbb37ada259579fe040 +gnosis,0x0000000000000000000000000000000000000000,0xa5c2254e4253490c54cef0a4347fddb8f75a4998 +avax,0x0000000000000000000000000000000000000000,0x330bdfade01ee9bf63c209ee33102dd334618e0a +avax,0x0000000000000000000000000000000000000000,0x1e34a77868e19a6647b1f2f47b51ed72dede95dd +avax,0x0000000000000000000000000000000000000000,0xaf8d1839c3c67cf571aa74b5c12398d4901147b3 +ethereumclassic,0x0000000000000000000000000000000000000000,0x2f56d5afc058b8734350b162efee75ee48f034e0 +ethereumclassic,0x0000000000000000000000000000000000000000,0x59fcb629a23e8ed0a60a0188771e221042260118 +ethereumclassic,0x0000000000000000000000000000000000000000,0x784b3a7a7981b959bd8d9d9e73c2013be819fbf2 \ No newline at end of file diff --git a/models/curated_package/defi/tvl/silver__complete_tvl.sql b/models/curated_package/defi/tvl/silver__complete_tvl.sql index 739e06c9..d92b5399 100644 --- a/models/curated_package/defi/tvl/silver__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver__complete_tvl.sql @@ -46,19 +46,60 @@ WITH all_tvl AS ( {% endif %} {% endfor %} ), +contracts AS ( + SELECT + address AS contract_address, + symbol AS token_symbol, + decimals AS token_decimals, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__dim_contracts') }} + UNION ALL + SELECT + '0x0000000000000000000000000000000000000000' AS contract_address, + '{{ vars.GLOBAL_NATIVE_ASSET_SYMBOL }}' AS token_symbol, + decimals AS token_decimals, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__dim_contracts') }} + WHERE + address = '{{ vars.GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS }}' +), +prices AS ( + SELECT + token_address, + price, + HOUR, + is_verified, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('price__ez_prices_hourly') }} + UNION ALL + SELECT + '0x0000000000000000000000000000000000000000' AS token_address, + price, + HOUR, + is_verified, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('price__ez_prices_hourly') }} + WHERE + token_address = '{{ vars.GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS }}' +), final AS ( SELECT a.block_number, a.block_date, a.contract_address, a.address, - c1.decimals, - c1.symbol, + c1.token_decimals AS decimals, + c1.token_symbol AS symbol, + p1.is_verified, a.amount_hex, a.amount_raw, utils.udf_decimal_adjust( a.amount_raw, - c1.decimals + c1.token_decimals ) AS amount_precise, amount_precise :: FLOAT AS amount, ROUND(amount * p1.price, 2) AS amount_usd, @@ -67,10 +108,10 @@ final AS ( a.platform FROM all_tvl a - LEFT JOIN {{ ref('core__dim_contracts') }} + LEFT JOIN contracts c1 - ON a.contract_address = c1.address - LEFT JOIN {{ ref('price__ez_prices_hourly') }} + ON a.contract_address = c1.contract_address + LEFT JOIN prices p1 ON a.contract_address = p1.token_address AND DATEADD( diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql index 3ce76ca8..6510ae33 100644 --- a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql +++ b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql @@ -70,9 +70,9 @@ balances AS ( ON b.address = t.mixer_address WHERE t.mixer_address IS NOT NULL - AND t.token_address IS NULL - AND balance_raw IS NOT NULL + AND t.token_address = '0x0000000000000000000000000000000000000000' AND t.chain = '{{ vars.GLOBAL_PROJECT_NAME }}' + AND balance_raw IS NOT NULL {% if is_incremental() %} AND b.modified_timestamp > ( From 2bc3237865a2e6aa468f91bca49ff5413ee139cd Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 14:54:58 -0700 Subject: [PATCH 059/164] tc --- .../defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql index 6510ae33..d29af251 100644 --- a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql +++ b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql @@ -50,8 +50,8 @@ balances AS ( SELECT b.block_number, b.block_date, - b.address AS contract_address, - NULL AS address, + t.contract_address, + b.address AS address, balance_hex AS amount_hex, balance_raw AS amount_raw, 'tornado_cash' AS protocol, From fa6000c0b8ef6bde25e35a613a6ba861a691831f Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 14:59:08 -0700 Subject: [PATCH 060/164] column name --- .../defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql index d29af251..de0f309f 100644 --- a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql +++ b/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql @@ -50,7 +50,7 @@ balances AS ( SELECT b.block_number, b.block_date, - t.contract_address, + t.token_address AS contract_address, b.address AS address, balance_hex AS amount_hex, balance_raw AS amount_raw, From 44f9c326ffc86db0fcbc0c16da5f2275c089d9e8 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 14:59:58 -0700 Subject: [PATCH 061/164] address column --- .../defi/tvl/uniswap/silver__uniswap_v1_tvl.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql index f77508d0..6fdcc192 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql @@ -17,6 +17,7 @@ WITH balances AS ( b.block_number, b.block_date, b.address AS contract_address, + p.pool_address AS address, balance_hex, balance_raw, p.protocol, @@ -45,7 +46,7 @@ SELECT block_number, block_date, contract_address, - NULL AS address, + address, balance_hex, balance_raw, protocol, From 234abe8e4a066a0962e5c8cee08fe3027ec45a57 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 15:02:07 -0700 Subject: [PATCH 062/164] uni v1 --- .../defi/tvl/uniswap/silver__uniswap_v1_tvl.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql index 6fdcc192..436dd011 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql @@ -16,8 +16,8 @@ WITH balances AS ( SELECT b.block_number, b.block_date, - b.address AS contract_address, - p.pool_address AS address, + '0x0000000000000000000000000000000000000000' AS contract_address, + b.address, balance_hex, balance_raw, p.protocol, From 869daaf14a895938e21a6897982250cd4181c0eb Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 15:03:15 -0700 Subject: [PATCH 063/164] comment --- .../curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql index 436dd011..cb698281 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql @@ -17,6 +17,7 @@ WITH balances AS ( b.block_number, b.block_date, '0x0000000000000000000000000000000000000000' AS contract_address, + --every pool is token-ETH only, no token-token pairs so TVL = 2x ETH value (pools are always 50/50 and 0x000 represents native asset) b.address, balance_hex, balance_raw, From ef941163662229101eb35a101881b8a10d5e8c0d Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 15:10:18 -0700 Subject: [PATCH 064/164] uni v1 --- models/curated_package/defi/tvl/silver__complete_tvl.sql | 1 + .../defi/tvl/uniswap/silver__uniswap_v1_tvl.sql | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/models/curated_package/defi/tvl/silver__complete_tvl.sql b/models/curated_package/defi/tvl/silver__complete_tvl.sql index d92b5399..8b290c67 100644 --- a/models/curated_package/defi/tvl/silver__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver__complete_tvl.sql @@ -18,6 +18,7 @@ {% set _ = models.append(ref('silver__curve_v1_tvl')) %} {% set _ = models.append(ref('silver__lido_v1_tvl')) %} {% set _ = models.append(ref('silver__tornado_cash_v1_tvl')) %} +{% set _ = models.append(ref('silver__uniswap_v1_tvl')) %} {% set _ = models.append(ref('silver__uniswap_v2_tvl')) %} {% set _ = models.append(ref('silver__uniswap_v3_tvl')) %} {% set _ = models.append(ref('silver__uniswap_v4_tvl')) %} diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql index cb698281..e922a6e0 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql @@ -16,8 +16,8 @@ WITH balances AS ( SELECT b.block_number, b.block_date, - '0x0000000000000000000000000000000000000000' AS contract_address, - --every pool is token-ETH only, no token-token pairs so TVL = 2x ETH value (pools are always 50/50 and 0x000 represents native asset) + token1 AS contract_address, + --every pool is token-ETH only, no token-token pairs so TVL = 2x ETH value (pools are always 50/50 and 0x000 e.g. token1, represents native asset) b.address, balance_hex, balance_raw, From f8b5386793493f40e5b9bc8f85a38a6786ea2e0d Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 15:14:37 -0700 Subject: [PATCH 065/164] column name --- .../defi/tvl/uniswap/silver__uniswap_v1_tvl.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql index e922a6e0..70d85854 100644 --- a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql @@ -48,8 +48,8 @@ SELECT block_date, contract_address, address, - balance_hex, - balance_raw, + balance_hex AS amount_hex, + balance_raw AS amount_raw, protocol, version, platform, From 5d08e75e130f783706c722d314fcb67b53dcf1c7 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 15:51:24 -0700 Subject: [PATCH 066/164] schema name --- .../aave/{silver__aave_v1_tvl.sql => silver_tvl__aave_v1_tvl.sql} | 0 .../aave/{silver__aave_v2_tvl.sql => silver_tvl__aave_v2_tvl.sql} | 0 .../aave/{silver__aave_v3_tvl.sql => silver_tvl__aave_v3_tvl.sql} | 0 .../{silver__curve_v1_tvl.sql => silver_tvl__curve_v1_tvl.sql} | 0 .../lido/{silver__lido_v1_tvl.sql => silver_tvl__lido_v1_tvl.sql} | 0 .../{silver__complete_tvl.sql => silver_tvl__complete_tvl.sql} | 0 ...ornado_cash_v1_tvl.sql => silver_tvl__tornado_cash_v1_tvl.sql} | 0 ...{silver__uniswap_v1_tvl.sql => silver_tvl__uniswap_v1_tvl.sql} | 0 ...{silver__uniswap_v2_tvl.sql => silver_tvl__uniswap_v2_tvl.sql} | 0 ...{silver__uniswap_v3_tvl.sql => silver_tvl__uniswap_v3_tvl.sql} | 0 ...{silver__uniswap_v4_tvl.sql => silver_tvl__uniswap_v4_tvl.sql} | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename models/curated_package/defi/tvl/aave/{silver__aave_v1_tvl.sql => silver_tvl__aave_v1_tvl.sql} (100%) rename models/curated_package/defi/tvl/aave/{silver__aave_v2_tvl.sql => silver_tvl__aave_v2_tvl.sql} (100%) rename models/curated_package/defi/tvl/aave/{silver__aave_v3_tvl.sql => silver_tvl__aave_v3_tvl.sql} (100%) rename models/curated_package/defi/tvl/curve/{silver__curve_v1_tvl.sql => silver_tvl__curve_v1_tvl.sql} (100%) rename models/curated_package/defi/tvl/lido/{silver__lido_v1_tvl.sql => silver_tvl__lido_v1_tvl.sql} (100%) rename models/curated_package/defi/tvl/{silver__complete_tvl.sql => silver_tvl__complete_tvl.sql} (100%) rename models/curated_package/defi/tvl/tornado_cash/{silver__tornado_cash_v1_tvl.sql => silver_tvl__tornado_cash_v1_tvl.sql} (100%) rename models/curated_package/defi/tvl/uniswap/{silver__uniswap_v1_tvl.sql => silver_tvl__uniswap_v1_tvl.sql} (100%) rename models/curated_package/defi/tvl/uniswap/{silver__uniswap_v2_tvl.sql => silver_tvl__uniswap_v2_tvl.sql} (100%) rename models/curated_package/defi/tvl/uniswap/{silver__uniswap_v3_tvl.sql => silver_tvl__uniswap_v3_tvl.sql} (100%) rename models/curated_package/defi/tvl/uniswap/{silver__uniswap_v4_tvl.sql => silver_tvl__uniswap_v4_tvl.sql} (100%) diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql b/models/curated_package/defi/tvl/aave/silver_tvl__aave_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/aave/silver__aave_v1_tvl.sql rename to models/curated_package/defi/tvl/aave/silver_tvl__aave_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql b/models/curated_package/defi/tvl/aave/silver_tvl__aave_v2_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/aave/silver__aave_v2_tvl.sql rename to models/curated_package/defi/tvl/aave/silver_tvl__aave_v2_tvl.sql diff --git a/models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql b/models/curated_package/defi/tvl/aave/silver_tvl__aave_v3_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/aave/silver__aave_v3_tvl.sql rename to models/curated_package/defi/tvl/aave/silver_tvl__aave_v3_tvl.sql diff --git a/models/curated_package/defi/tvl/curve/silver__curve_v1_tvl.sql b/models/curated_package/defi/tvl/curve/silver_tvl__curve_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/curve/silver__curve_v1_tvl.sql rename to models/curated_package/defi/tvl/curve/silver_tvl__curve_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/lido/silver__lido_v1_tvl.sql b/models/curated_package/defi/tvl/lido/silver_tvl__lido_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/lido/silver__lido_v1_tvl.sql rename to models/curated_package/defi/tvl/lido/silver_tvl__lido_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/silver__complete_tvl.sql b/models/curated_package/defi/tvl/silver_tvl__complete_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/silver__complete_tvl.sql rename to models/curated_package/defi/tvl/silver_tvl__complete_tvl.sql diff --git a/models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql b/models/curated_package/defi/tvl/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/tornado_cash/silver__tornado_cash_v1_tvl.sql rename to models/curated_package/defi/tvl/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/uniswap/silver__uniswap_v1_tvl.sql rename to models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v2_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/uniswap/silver__uniswap_v2_tvl.sql rename to models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v2_tvl.sql diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v3_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/uniswap/silver__uniswap_v3_tvl.sql rename to models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v3_tvl.sql diff --git a/models/curated_package/defi/tvl/uniswap/silver__uniswap_v4_tvl.sql b/models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v4_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/uniswap/silver__uniswap_v4_tvl.sql rename to models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v4_tvl.sql From 2721c5940acdc32baab3fd9785cce93cfd132137 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 16:08:52 -0700 Subject: [PATCH 067/164] new folders and gold --- .../gold/defi__ez_stablecoins_supply.sql | 2 +- .../defi/tvl/gold/defi__ez_protocol_tvl.sql | 76 ++++++ .../aave/silver_tvl__aave_v1_tvl.sql | 0 .../aave/silver_tvl__aave_v2_tvl.sql | 0 .../aave/silver_tvl__aave_v3_tvl.sql | 0 .../curve/silver_tvl__curve_v1_tvl.sql | 0 .../lido/silver_tvl__lido_v1_tvl.sql | 0 .../tvl/silver/silver_tvl__complete_tvl.sql | 217 ++++++++++++++++++ .../silver_tvl__tornado_cash_v1_tvl.sql | 0 .../uniswap/silver_tvl__uniswap_v1_tvl.sql | 0 .../uniswap/silver_tvl__uniswap_v2_tvl.sql | 0 .../uniswap/silver_tvl__uniswap_v3_tvl.sql | 0 .../uniswap/silver_tvl__uniswap_v4_tvl.sql | 0 .../defi/tvl/silver_tvl__complete_tvl.sql | 146 ------------ 14 files changed, 294 insertions(+), 147 deletions(-) create mode 100644 models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql rename models/curated_package/defi/tvl/{ => silver}/aave/silver_tvl__aave_v1_tvl.sql (100%) rename models/curated_package/defi/tvl/{ => silver}/aave/silver_tvl__aave_v2_tvl.sql (100%) rename models/curated_package/defi/tvl/{ => silver}/aave/silver_tvl__aave_v3_tvl.sql (100%) rename models/curated_package/defi/tvl/{ => silver}/curve/silver_tvl__curve_v1_tvl.sql (100%) rename models/curated_package/defi/tvl/{ => silver}/lido/silver_tvl__lido_v1_tvl.sql (100%) create mode 100644 models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql rename models/curated_package/defi/tvl/{ => silver}/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql (100%) rename models/curated_package/defi/tvl/{ => silver}/uniswap/silver_tvl__uniswap_v1_tvl.sql (100%) rename models/curated_package/defi/tvl/{ => silver}/uniswap/silver_tvl__uniswap_v2_tvl.sql (100%) rename models/curated_package/defi/tvl/{ => silver}/uniswap/silver_tvl__uniswap_v3_tvl.sql (100%) rename models/curated_package/defi/tvl/{ => silver}/uniswap/silver_tvl__uniswap_v4_tvl.sql (100%) delete mode 100644 models/curated_package/defi/tvl/silver_tvl__complete_tvl.sql diff --git a/models/curated_package/defi/stablecoins/gold/defi__ez_stablecoins_supply.sql b/models/curated_package/defi/stablecoins/gold/defi__ez_stablecoins_supply.sql index c5068c59..61f2d63c 100644 --- a/models/curated_package/defi/stablecoins/gold/defi__ez_stablecoins_supply.sql +++ b/models/curated_package/defi/stablecoins/gold/defi__ez_stablecoins_supply.sql @@ -8,7 +8,7 @@ materialized = 'view', persist_docs ={ "relation": true, "columns": true }, - tags = ['gold','defi','stablecoins','heal','curated_daily'] + tags = ['gold','defi','stablecoins','curated_daily'] ) }} SELECT diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql new file mode 100644 index 00000000..5ab5230a --- /dev/null +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql @@ -0,0 +1,76 @@ +{# Get variables #} +{% set vars = return_vars() %} +{# Log configuration details #} +{{ log_model_details() }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'ez_protocol_tvl_id', + persist_docs ={ "relation": true, + "columns": true }, + tags = ['gold','defi','tvl','curated_daily'] +) }} + +WITH complete_tvl AS ( + + SELECT + block_number, + block_date, + contract_address, + address, + decimals, + symbol, + amount_hex, + amount_raw, + amount_precise, + amount, + amount_usd, + protocol, + version, + platform + FROM + {{ ref('silver_tvl__complete_tvl') }} + +{% if is_incremental() %} +WHERE + CONCAT( + block_date, + '-', + platform + ) IN ( + SELECT + DISTINCT CONCAT( + block_date, + '-', + platform + ) + FROM + {{ ref('silver_tvl__complete_tvl') }} + WHERE + modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + ) +{% endif %} +) +SELECT + block_date, + SUM(COALESCE(amount_usd, 0)) AS tvl_usd, + protocol, + version, + platform, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_date','platform'] + ) }} AS ez_protocol_tvl_id +FROM + complete_tvl +GROUP BY + block_date, + protocol, + version, + platform diff --git a/models/curated_package/defi/tvl/aave/silver_tvl__aave_v1_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/aave/silver_tvl__aave_v1_tvl.sql rename to models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/aave/silver_tvl__aave_v2_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/aave/silver_tvl__aave_v2_tvl.sql rename to models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql diff --git a/models/curated_package/defi/tvl/aave/silver_tvl__aave_v3_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/aave/silver_tvl__aave_v3_tvl.sql rename to models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql diff --git a/models/curated_package/defi/tvl/curve/silver_tvl__curve_v1_tvl.sql b/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/curve/silver_tvl__curve_v1_tvl.sql rename to models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/lido/silver_tvl__lido_v1_tvl.sql b/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/lido/silver_tvl__lido_v1_tvl.sql rename to models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql new file mode 100644 index 00000000..2f891841 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -0,0 +1,217 @@ +{# Get variables #} +{% set vars = return_vars() %} +{# Log configuration details #} +{{ log_model_details() }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_tvl_id', + tags = ['silver','defi','tvl','complete','heal','curated_daily'] +) }} + +{% set models = [] %} +{% set _ = models.append(ref('silver__aave_v1_tvl')) %} +{% set _ = models.append(ref('silver__aave_v2_tvl')) %} +{% set _ = models.append(ref('silver__aave_v3_tvl')) %} +{% set _ = models.append(ref('silver__curve_v1_tvl')) %} +{% set _ = models.append(ref('silver__lido_v1_tvl')) %} +{% set _ = models.append(ref('silver__tornado_cash_v1_tvl')) %} +{% set _ = models.append(ref('silver__uniswap_v1_tvl')) %} +{% set _ = models.append(ref('silver__uniswap_v2_tvl')) %} +{% set _ = models.append(ref('silver__uniswap_v3_tvl')) %} +{% set _ = models.append(ref('silver__uniswap_v4_tvl')) %} + +WITH all_tvl AS ( + {% for model in models %} + SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform + FROM {{ model }} + {% if not loop.last %} + {% if is_incremental() %} + WHERE modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} + UNION ALL + {% endif %} + {% endfor %} +), +contracts AS ( + SELECT + address AS contract_address, + symbol AS token_symbol, + decimals AS token_decimals, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__dim_contracts') }} + UNION ALL + SELECT + '0x0000000000000000000000000000000000000000' AS contract_address, + '{{ vars.GLOBAL_NATIVE_ASSET_SYMBOL }}' AS token_symbol, + decimals AS token_decimals, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__dim_contracts') }} + WHERE + address = '{{ vars.GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS }}' +), +prices AS ( + SELECT + token_address, + price, + HOUR, + is_verified, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('price__ez_prices_hourly') }} + UNION ALL + SELECT + '0x0000000000000000000000000000000000000000' AS token_address, + price, + HOUR, + is_verified, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('price__ez_prices_hourly') }} + WHERE + token_address = '{{ vars.GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS }}' +), +complete_tvl AS ( + SELECT + A.block_number, + A.block_date, + A.contract_address, + A.address, + c1.token_decimals AS decimals, + c1.token_symbol AS symbol, + p1.is_verified, + A.amount_hex, + A.amount_raw, + utils.udf_decimal_adjust( + A.amount_raw, + c1.token_decimals + ) AS amount_precise, + amount_precise :: FLOAT AS amount, + ROUND( + amount * p1.price, + 2 + ) AS amount_usd, + A.protocol, + A.version, + A.platform + FROM + all_tvl A + LEFT JOIN contracts c1 + ON A.contract_address = c1.contract_address + LEFT JOIN prices p1 + ON A.contract_address = p1.token_address + AND DATEADD( + 'hour', + 23, + A.block_date + ) = p1.hour +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + t.block_number, + t.block_date, + t.contract_address, + t.address, + c1.token_decimals AS decimals_heal, + c1.token_symbol AS symbol_heal, + p1.is_verified AS is_verified_heal, + t.amount_hex, + t.amount_raw, + utils.udf_decimal_adjust( + t.amount_raw, + c1.token_decimals + ) AS amount_precise_heal, + amount_precise_heal :: FLOAT AS amount_heal, + ROUND( + amount_heal * p1.price, + 2 + ) AS amount_usd_heal, + t.protocol, + t.version, + t.platform + FROM + {{ this }} + t + LEFT JOIN contracts c1 + ON A.contract_address = c1.contract_address + LEFT JOIN prices p1 + ON A.contract_address = p1.token_address + AND DATEADD( + 'hour', + 23, + A.block_date + ) = p1.hour +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_tvl + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + block_number, + block_date, + contract_address, + address, + decimals_heal AS decimals, + symbol_heal AS symbol, + is_verified_heal AS is_verified, + amount_hex, + amount_raw, + amount_precise_heal AS amount_precise, + amount_heal AS amount, + amount_usd_heal AS amount_usd, + protocol, + version, + platform +FROM + heal_model +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + decimals, + symbol, + amount_hex, + amount_raw, + amount_precise, + amount, + amount_usd, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS complete_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL diff --git a/models/curated_package/defi/tvl/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql b/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql rename to models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v1_tvl.sql rename to models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql diff --git a/models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v2_tvl.sql rename to models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql diff --git a/models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v3_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v3_tvl.sql rename to models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql diff --git a/models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v4_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql similarity index 100% rename from models/curated_package/defi/tvl/uniswap/silver_tvl__uniswap_v4_tvl.sql rename to models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql diff --git a/models/curated_package/defi/tvl/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver_tvl__complete_tvl.sql deleted file mode 100644 index 8b290c67..00000000 --- a/models/curated_package/defi/tvl/silver_tvl__complete_tvl.sql +++ /dev/null @@ -1,146 +0,0 @@ -{# Get variables #} -{% set vars = return_vars() %} - -{# Log configuration details #} -{{ log_model_details() }} - -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'complete_tvl_id', - tags = ['silver','defi','tvl','complete','curated_daily'] -) }} - -{% set models = [] %} -{% set _ = models.append(ref('silver__aave_v1_tvl')) %} -{% set _ = models.append(ref('silver__aave_v2_tvl')) %} -{% set _ = models.append(ref('silver__aave_v3_tvl')) %} -{% set _ = models.append(ref('silver__curve_v1_tvl')) %} -{% set _ = models.append(ref('silver__lido_v1_tvl')) %} -{% set _ = models.append(ref('silver__tornado_cash_v1_tvl')) %} -{% set _ = models.append(ref('silver__uniswap_v1_tvl')) %} -{% set _ = models.append(ref('silver__uniswap_v2_tvl')) %} -{% set _ = models.append(ref('silver__uniswap_v3_tvl')) %} -{% set _ = models.append(ref('silver__uniswap_v4_tvl')) %} - -WITH all_tvl AS ( - {% for model in models %} - SELECT - block_number, - block_date, - contract_address, - address, - amount_hex, - amount_raw, - protocol, - version, - platform - FROM {{ model }} - {% if not loop.last %} - {% if is_incremental() %} - WHERE modified_timestamp > ( - SELECT MAX(modified_timestamp) - FROM {{ this }} - ) - {% endif %} - UNION ALL - {% endif %} - {% endfor %} -), -contracts AS ( - SELECT - address AS contract_address, - symbol AS token_symbol, - decimals AS token_decimals, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('core__dim_contracts') }} - UNION ALL - SELECT - '0x0000000000000000000000000000000000000000' AS contract_address, - '{{ vars.GLOBAL_NATIVE_ASSET_SYMBOL }}' AS token_symbol, - decimals AS token_decimals, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('core__dim_contracts') }} - WHERE - address = '{{ vars.GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS }}' -), -prices AS ( - SELECT - token_address, - price, - HOUR, - is_verified, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('price__ez_prices_hourly') }} - UNION ALL - SELECT - '0x0000000000000000000000000000000000000000' AS token_address, - price, - HOUR, - is_verified, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('price__ez_prices_hourly') }} - WHERE - token_address = '{{ vars.GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS }}' -), -final AS ( - SELECT - a.block_number, - a.block_date, - a.contract_address, - a.address, - c1.token_decimals AS decimals, - c1.token_symbol AS symbol, - p1.is_verified, - a.amount_hex, - a.amount_raw, - utils.udf_decimal_adjust( - a.amount_raw, - c1.token_decimals - ) AS amount_precise, - amount_precise :: FLOAT AS amount, - ROUND(amount * p1.price, 2) AS amount_usd, - a.protocol, - a.version, - a.platform - FROM - all_tvl a - LEFT JOIN contracts - c1 - ON a.contract_address = c1.contract_address - LEFT JOIN prices - p1 - ON a.contract_address = p1.token_address - AND DATEADD( - 'hour', - 23, - a.block_date - ) = p1.hour -) -SELECT - block_number, - block_date, - contract_address, - address, - decimals, - symbol, - amount_hex, - amount_raw, - amount_precise, - amount, - amount_usd, - protocol, - version, - platform, - {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address','address','platform'] - ) }} AS complete_tvl_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - final \ No newline at end of file From 84e9d0cf22500c1fef8f41dbbfa2ad25f771bd01 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 3 Dec 2025 17:52:55 -0700 Subject: [PATCH 068/164] refs --- .../tvl/silver/silver_tvl__complete_tvl.sql | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index 2f891841..488b2992 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -10,16 +10,16 @@ ) }} {% set models = [] %} -{% set _ = models.append(ref('silver__aave_v1_tvl')) %} -{% set _ = models.append(ref('silver__aave_v2_tvl')) %} -{% set _ = models.append(ref('silver__aave_v3_tvl')) %} -{% set _ = models.append(ref('silver__curve_v1_tvl')) %} -{% set _ = models.append(ref('silver__lido_v1_tvl')) %} -{% set _ = models.append(ref('silver__tornado_cash_v1_tvl')) %} -{% set _ = models.append(ref('silver__uniswap_v1_tvl')) %} -{% set _ = models.append(ref('silver__uniswap_v2_tvl')) %} -{% set _ = models.append(ref('silver__uniswap_v3_tvl')) %} -{% set _ = models.append(ref('silver__uniswap_v4_tvl')) %} +{% set _ = models.append(ref('silver_tvl__aave_v1_tvl')) %} +{% set _ = models.append(ref('silver_tvl__aave_v2_tvl')) %} +{% set _ = models.append(ref('silver_tvl__aave_v3_tvl')) %} +{% set _ = models.append(ref('silver_tvl__curve_v1_tvl')) %} +{% set _ = models.append(ref('silver_tvl__lido_v1_tvl')) %} +{% set _ = models.append(ref('silver_tvl__tornado_cash_v1_tvl')) %} +{% set _ = models.append(ref('silver_tvl__uniswap_v1_tvl')) %} +{% set _ = models.append(ref('silver_tvl__uniswap_v2_tvl')) %} +{% set _ = models.append(ref('silver_tvl__uniswap_v3_tvl')) %} +{% set _ = models.append(ref('silver_tvl__uniswap_v4_tvl')) %} WITH all_tvl AS ( {% for model in models %} From cf4d32c3bcca65013bcd0136d6625c6921c2a6f0 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 4 Dec 2025 09:16:38 -0700 Subject: [PATCH 069/164] uni v1 swaps --- ...lver_dex__complete_dex_liquidity_pools.sql | 43 ++++ .../silver/silver_dex__complete_dex_swaps.sql | 42 ++++ .../uniswap/silver_dex__uniswap_v1_swaps.sql | 204 ++++++++++++++++++ 3 files changed, 289 insertions(+) create mode 100644 models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql diff --git a/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pools.sql b/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pools.sql index 94640939..26b242d8 100644 --- a/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pools.sql +++ b/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pools.sql @@ -867,6 +867,44 @@ WHERE ) {% endif %} ), +uniswap_v1 AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + pool_address, + '0x' AS pool_id, + NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, + token0, + token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, + platform, + protocol, + version, + type, + _log_id AS _id, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('silver_dex__uniswap_v1_pools') }} + +{% if is_incremental() and 'uniswap_v1' not in vars.CURATED_FR_MODELS %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '{{ vars.CURATED_COMPLETE_LOOKBACK_HOURS }}' + FROM + {{ this }} + ) +{% endif %} +), uniswap_v4 AS ( SELECT block_number, @@ -1234,6 +1272,11 @@ all_pools AS ( FROM pancakeswap_v3 UNION ALL + SELECT + * + FROM + uniswap_v1 + UNION ALL SELECT * FROM diff --git a/models/curated_package/defi/dex/silver/silver_dex__complete_dex_swaps.sql b/models/curated_package/defi/dex/silver/silver_dex__complete_dex_swaps.sql index 52086f24..77734507 100644 --- a/models/curated_package/defi/dex/silver/silver_dex__complete_dex_swaps.sql +++ b/models/curated_package/defi/dex/silver/silver_dex__complete_dex_swaps.sql @@ -1206,6 +1206,43 @@ WHERE ) {% endif %} ), +uniswap_v1 AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + '0x' AS pool_id, + event_name, + amount_in_unadj, + amount_out_unadj, + token_in, + token_out, + sender, + tx_to, + event_index, + platform, + protocol, + version, + type, + _log_id, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('silver_dex__uniswap_v1_swaps') }} + +{% if is_incremental() and 'uniswap_v1' not in vars.CURATED_FR_MODELS %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '{{ vars.CURATED_COMPLETE_LOOKBACK_HOURS }}' + FROM + {{ this }} + ) +{% endif %} +), uniswap_v4 AS ( SELECT block_number, @@ -1695,6 +1732,11 @@ all_dex AS ( FROM dackie UNION ALL + SELECT + * + FROM + uniswap_v1 + UNION ALL SELECT * FROM diff --git a/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql new file mode 100644 index 00000000..ac0922fb --- /dev/null +++ b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql @@ -0,0 +1,204 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'block_number', + cluster_by = ['block_timestamp::DATE'], + tags = ['silver_dex','defi','dex','curated'] +) }} + +WITH swaps AS ( + + SELECT + l.block_number, + l.block_timestamp, + l.tx_hash, + l.event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + l.contract_address, + CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS buyer_address, + utils.udf_hex_to_int( + topics [2] :: STRING + ) :: FLOAT AS sold_amount, + utils.udf_hex_to_int( + topics [3] :: STRING + ) :: FLOAT AS bought_amount, + p.token0, + p.token1, + p.platform, + p.protocol, + p.version, + p.type, + CASE + WHEN topics [0] :: STRING = '0xcd60aa75dea3072fbc07ae6d7d856b5dc5f4eee88854f5b4abf7b680ef8bc50f' THEN 'TokenPurchase' + ELSE 'EthPurchase' + END AS event_name, + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp + FROM + {{ ref('core__fact_event_logs') }} + l + INNER JOIN {{ ref('silver_dex__uniswap_v1_pools') }} + p + ON p.pool_address = l.contract_address + WHERE + topics [0] :: STRING IN ( + '0xcd60aa75dea3072fbc07ae6d7d856b5dc5f4eee88854f5b4abf7b680ef8bc50f', + --TokenPurchase + '0x7f4091b46c33e918a0f3aa42307641d17bb67029427a5369e54b353984238705' + ) --EthPurchase + AND tx_succeeded + +{% if is_incremental() %} +AND l.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) - INTERVAL '{{ vars.CURATED_LOOKBACK_HOURS }}' + FROM + {{ this }} +) +AND l.modified_timestamp >= SYSDATE() - INTERVAL '{{ vars.CURATED_LOOKBACK_DAYS }}' +{% endif %} +), +transfers AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + contract_address, + CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS from_address, + CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS to_address, + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + utils.udf_hex_to_int( + segmented_data [0] :: STRING + ) :: FLOAT AS amount + FROM + {{ ref('core__fact_event_logs') }} + WHERE + topics [0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' --Transfer + AND tx_hash IN ( + SELECT + tx_hash + FROM + swaps + ) + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) - INTERVAL '{{ vars.CURATED_LOOKBACK_HOURS }}' + FROM + {{ this }} +) +AND modified_timestamp >= SYSDATE() - INTERVAL '{{ vars.CURATED_LOOKBACK_DAYS }}' +{% endif %} +), +eth_purchase AS ( + SELECT + s.block_number, + s.block_timestamp, + s.tx_hash, + s.event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + s.contract_address, + buyer_address AS sender, + to_address AS tx_to, + sold_amount AS amount_in_unadj, + bought_amount AS amount_out_unadj, + t.contract_address AS token_in, + '0x0000000000000000000000000000000000000000' AS token_out, + --native ETH, for pricing purposes + platform, + protocol, + version, + TYPE, + event_name, + _log_id, + modified_timestamp + FROM + swaps s + INNER JOIN transfers t + ON s.tx_hash = t.tx_hash + AND s.buyer_address = t.from_address + AND s.sold_amount = t.amount + AND s.event_name = 'EthPurchase' +), +token_purchase AS ( + SELECT + s.block_number, + s.block_timestamp, + s.tx_hash, + s.event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + s.contract_address, + buyer_address AS sender, + to_address AS tx_to, + sold_amount AS amount_in_unadj, + bought_amount AS amount_out_unadj, + '0x0000000000000000000000000000000000000000' AS token_in, + t.contract_address AS token_out, + platform, + protocol, + version, + TYPE, + event_name, + _log_id, + modified_timestamp + FROM + swaps s + INNER JOIN transfers t + ON s.tx_hash = t.tx_hash + AND s.buyer_address = t.to_address + AND s.bought_amount = t.amount + AND s.event_name = 'TokenPurchase' +), +all_swaps AS ( + SELECT + * + FROM + eth_purchase + UNION ALL + SELECT + * + FROM + token_purchase +) +SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + sender, + tx_to, + amount_in_unadj, + amount_out_unadj, + token_in, + token_out, + platform, + protocol, + version, + TYPE, + event_name, + _log_id, + modified_timestamp +FROM + all_swaps From 8ece06962f9b902f1f4a2b3b03b98007f549ec12 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 4 Dec 2025 09:43:11 -0700 Subject: [PATCH 070/164] uni v1 lp actions --- ...x__complete_dex_liquidity_pool_actions.sql | 51 +++++++ .../silver_dex__uniswap_v1_pool_actions.sql | 110 ++++++++++++++ .../uniswap/silver_dex__uniswap_v1_swaps.sql | 136 +++--------------- 3 files changed, 178 insertions(+), 119 deletions(-) create mode 100644 models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pool_actions.sql diff --git a/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pool_actions.sql b/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pool_actions.sql index bd673df5..820e0bbc 100644 --- a/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pool_actions.sql +++ b/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pool_actions.sql @@ -792,6 +792,52 @@ WHERE ) {% endif %} ), +uniswap_v1 AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + event_name, + liquidity_provider, + sender, + receiver, + pool_address, + token0, + token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, + amount0_unadj, + amount1_unadj, + NULL AS amount2_unadj, + NULL AS amount3_unadj, + NULL AS amount4_unadj, + NULL AS amount5_unadj, + NULL AS amount6_unadj, + NULL AS amount7_unadj, + platform, + protocol, + version, + TYPE, + _log_id AS _id, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('silver_dex__uniswap_v1_pool_actions') }} + +{% if is_incremental() and 'uniswap_v1' not in vars.curated_fr_models %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '{{ vars.CURATED_COMPLETE_LOOKBACK_HOURS }}' + FROM + {{ this }} + ) +{% endif %} +), velodrome_v1 AS ( SELECT block_number, @@ -1067,6 +1113,11 @@ all_pools AS ( FROM zyberswap_v2 UNION ALL + SELECT + * + FROM + uniswap_v1 + UNION ALL SELECT * FROM diff --git a/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pool_actions.sql b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pool_actions.sql new file mode 100644 index 00000000..bdb6be07 --- /dev/null +++ b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pool_actions.sql @@ -0,0 +1,110 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'block_number', + cluster_by = ['block_timestamp::DATE'], + tags = ['silver_dex','defi','dex','lp_actions','curated'] +) }} + +WITH evt AS ( + SELECT + l.block_number, + l.block_timestamp, + l.tx_hash, + l.event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + l.contract_address AS pool_address, + token0, + token1, + topic_0, + topic_1, + topic_2, + topic_3, + data, + CASE + WHEN topic_0 = '0x06239653922ac7bea6aa2b19dc486b9361821d37712eb796adfd38d81de278ca' THEN 'AddLiquidity' + ELSE 'RemoveLiquidity' + END AS event_name, + CONCAT('0x', SUBSTR(topic_1, 27, 40)) AS provider_address, + utils.udf_hex_to_int( + topic_2 + ) :: FLOAT AS eth_amount, + utils.udf_hex_to_int( + topic_3 + ) :: FLOAT AS token_amount, + provider_address AS liquidity_provider, + provider_address AS sender, + CASE + WHEN event_name = 'AddLiquidity' THEN pool_address + ELSE provider_address + END AS receiver, + p.protocol, + p.version, + p.type, + CONCAT( + p.protocol, + '-', + p.version + ) AS platform, + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp + FROM + {{ ref('core__fact_event_logs') }} + l + INNER JOIN {{ref('silver_dex__uniswap_v1_pools')}} p + ON l.contract_address = p.pool_address + WHERE + topic_0 IN ('0x06239653922ac7bea6aa2b19dc486b9361821d37712eb796adfd38d81de278ca', --AddLiquidity + '0x0fbf06c058b90cb038a618f8c2acbf6145f8b3570fd1fa56abb8f0f3f05b36e8' --RemoveLiquidity + ) + AND tx_succeeded + +{% if is_incremental() %} +AND l.modified_timestamp >= ( + SELECT + MAX(modified_timestamp) - INTERVAL '{{ vars.CURATED_LOOKBACK_HOURS }}' + FROM + {{ this }} +) +AND l.modified_timestamp >= SYSDATE() - INTERVAL '{{ vars.CURATED_LOOKBACK_DAYS }}' +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + event_name, + pool_address, + token0, + token1, + liquidity_provider, + sender, + receiver, + token_amount AS amount0_unadj, + eth_amount AS amount1_unadj, + protocol, + version, + type, + platform, + _log_id, + modified_timestamp +FROM + evt qualify(ROW_NUMBER() over (PARTITION BY _log_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql index ac0922fb..5346cf8b 100644 --- a/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql +++ b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql @@ -23,6 +23,10 @@ WITH swaps AS ( origin_from_address, origin_to_address, l.contract_address, + CASE + WHEN topics [0] :: STRING = '0xcd60aa75dea3072fbc07ae6d7d856b5dc5f4eee88854f5b4abf7b680ef8bc50f' THEN 'TokenPurchase' + ELSE 'EthPurchase' + END AS event_name, CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS buyer_address, utils.udf_hex_to_int( topics [2] :: STRING @@ -30,16 +34,18 @@ WITH swaps AS ( utils.udf_hex_to_int( topics [3] :: STRING ) :: FLOAT AS bought_amount, - p.token0, - p.token1, + CASE + WHEN event_name = 'EthPurchase' THEN p.token0 + ELSE p.token1 + END AS token_in, + CASE + WHEN event_name = 'EthPurchase' THEN p.token1 + ELSE p.token0 + END AS token_out, p.platform, p.protocol, p.version, p.type, - CASE - WHEN topics [0] :: STRING = '0xcd60aa75dea3072fbc07ae6d7d856b5dc5f4eee88854f5b4abf7b680ef8bc50f' THEN 'TokenPurchase' - ELSE 'EthPurchase' - END AS event_name, CONCAT( l.tx_hash :: STRING, '-', @@ -69,114 +75,6 @@ AND l.modified_timestamp >= ( ) AND l.modified_timestamp >= SYSDATE() - INTERVAL '{{ vars.CURATED_LOOKBACK_DAYS }}' {% endif %} -), -transfers AS ( - SELECT - block_number, - block_timestamp, - tx_hash, - event_index, - contract_address, - CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS from_address, - CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS to_address, - regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) :: FLOAT AS amount - FROM - {{ ref('core__fact_event_logs') }} - WHERE - topics [0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' --Transfer - AND tx_hash IN ( - SELECT - tx_hash - FROM - swaps - ) - -{% if is_incremental() %} -AND modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - INTERVAL '{{ vars.CURATED_LOOKBACK_HOURS }}' - FROM - {{ this }} -) -AND modified_timestamp >= SYSDATE() - INTERVAL '{{ vars.CURATED_LOOKBACK_DAYS }}' -{% endif %} -), -eth_purchase AS ( - SELECT - s.block_number, - s.block_timestamp, - s.tx_hash, - s.event_index, - origin_function_signature, - origin_from_address, - origin_to_address, - s.contract_address, - buyer_address AS sender, - to_address AS tx_to, - sold_amount AS amount_in_unadj, - bought_amount AS amount_out_unadj, - t.contract_address AS token_in, - '0x0000000000000000000000000000000000000000' AS token_out, - --native ETH, for pricing purposes - platform, - protocol, - version, - TYPE, - event_name, - _log_id, - modified_timestamp - FROM - swaps s - INNER JOIN transfers t - ON s.tx_hash = t.tx_hash - AND s.buyer_address = t.from_address - AND s.sold_amount = t.amount - AND s.event_name = 'EthPurchase' -), -token_purchase AS ( - SELECT - s.block_number, - s.block_timestamp, - s.tx_hash, - s.event_index, - origin_function_signature, - origin_from_address, - origin_to_address, - s.contract_address, - buyer_address AS sender, - to_address AS tx_to, - sold_amount AS amount_in_unadj, - bought_amount AS amount_out_unadj, - '0x0000000000000000000000000000000000000000' AS token_in, - t.contract_address AS token_out, - platform, - protocol, - version, - TYPE, - event_name, - _log_id, - modified_timestamp - FROM - swaps s - INNER JOIN transfers t - ON s.tx_hash = t.tx_hash - AND s.buyer_address = t.to_address - AND s.bought_amount = t.amount - AND s.event_name = 'TokenPurchase' -), -all_swaps AS ( - SELECT - * - FROM - eth_purchase - UNION ALL - SELECT - * - FROM - token_purchase ) SELECT block_number, @@ -187,10 +85,10 @@ SELECT origin_from_address, origin_to_address, contract_address, - sender, - tx_to, - amount_in_unadj, - amount_out_unadj, + buyer_address AS sender, + sender AS tx_to, + sold_amount AS amount_in_unadj, + bought_amount AS amount_out_unadj, token_in, token_out, platform, @@ -201,4 +99,4 @@ SELECT _log_id, modified_timestamp FROM - all_swaps + swaps From 75febe81875493a637d5be74266970cbe97dca19 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 4 Dec 2025 10:00:37 -0700 Subject: [PATCH 071/164] event column --- .../defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql index 5346cf8b..6651b8a4 100644 --- a/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql +++ b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql @@ -24,8 +24,8 @@ WITH swaps AS ( origin_to_address, l.contract_address, CASE - WHEN topics [0] :: STRING = '0xcd60aa75dea3072fbc07ae6d7d856b5dc5f4eee88854f5b4abf7b680ef8bc50f' THEN 'TokenPurchase' - ELSE 'EthPurchase' + WHEN topics [0] :: STRING = '0x7f4091b46c33e918a0f3aa42307641d17bb67029427a5369e54b353984238705' THEN 'EthPurchase' + ELSE 'TokenPurchase' END AS event_name, CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS buyer_address, utils.udf_hex_to_int( From 05eac3b7a9ef236a2d18053bb53dd6c4f43dd872 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 4 Dec 2025 10:04:01 -0700 Subject: [PATCH 072/164] event name --- .../defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql index 6651b8a4..9a13a036 100644 --- a/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql +++ b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_swaps.sql @@ -35,11 +35,11 @@ WITH swaps AS ( topics [3] :: STRING ) :: FLOAT AS bought_amount, CASE - WHEN event_name = 'EthPurchase' THEN p.token0 + WHEN topics [0] :: STRING = '0x7f4091b46c33e918a0f3aa42307641d17bb67029427a5369e54b353984238705' THEN p.token0 ELSE p.token1 END AS token_in, CASE - WHEN event_name = 'EthPurchase' THEN p.token1 + WHEN topics [0] :: STRING = '0x7f4091b46c33e918a0f3aa42307641d17bb67029427a5369e54b353984238705' THEN p.token1 ELSE p.token0 END AS token_out, p.platform, From 241aea2d31852e4f99702ae6cae8b812b3729f63 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 4 Dec 2025 10:22:20 -0700 Subject: [PATCH 073/164] topic change --- .../dex/silver/uniswap/silver_dex__uniswap_v1_pool_actions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pool_actions.sql b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pool_actions.sql index bdb6be07..be0857c9 100644 --- a/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pool_actions.sql +++ b/models/curated_package/defi/dex/silver/uniswap/silver_dex__uniswap_v1_pool_actions.sql @@ -43,7 +43,7 @@ WITH evt AS ( provider_address AS liquidity_provider, provider_address AS sender, CASE - WHEN event_name = 'AddLiquidity' THEN pool_address + WHEN topic_0 = '0x06239653922ac7bea6aa2b19dc486b9361821d37712eb796adfd38d81de278ca' THEN pool_address ELSE provider_address END AS receiver, p.protocol, From 8de859b55c9f9ddc1a40e3748b0a3567d8ebdbdb Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 4 Dec 2025 11:47:25 -0700 Subject: [PATCH 074/164] curated recency exclusion for uni v1 --- macros/global/variables/project_vars/ethereum_vars.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/global/variables/project_vars/ethereum_vars.sql b/macros/global/variables/project_vars/ethereum_vars.sql index 90ee66f1..a22e9c5d 100644 --- a/macros/global/variables/project_vars/ethereum_vars.sql +++ b/macros/global/variables/project_vars/ethereum_vars.sql @@ -26,8 +26,8 @@ 'MAIN_OBSERV_EXCLUSION_LIST_ENABLED': true, 'MAIN_SL_TOKEN_READS_BRONZE_TABLE_ENABLED': true, 'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['silo-v1','aave-v1','flux-v1','aave-v2','aave-v2.1','sturdy-v1','silo-v2'], - 'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['hashflow-v1','pancakeswap-v2','synthetix-v1'], - 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['kyberswap-v2'], + 'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['hashflow-v1','pancakeswap-v2','synthetix-v1','uniswap-v1'], + 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['kyberswap-v2','uniswap-v1'], 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['ronin_axie_bridge-v1','multichain-v7','symbiosis-v1','axie_infinity-v2','across-v1','near_rainbow_bridge-v1','hop-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'uniswap': { From 9be943c5260d47de8d1a29414e6e279decddd146 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 4 Dec 2025 11:59:05 -0700 Subject: [PATCH 075/164] temp reads model with notes --- .../silver_reads__stablecoins_reads.sql | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql diff --git a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql new file mode 100644 index 00000000..5e54fca6 --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql @@ -0,0 +1,62 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +-- depends_on: {{ ref('price__ez_asset_metadata') }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'stablecoins_reads_id', + post_hook = '{{ unverify_stablecoins() }}', + tags = ['silver','contract_reads','heal','curated_daily'] +) }} +--make sure tags are accurate for workflows +--make sure complete model includes existing stablecoin records +--rename external table to contract_reads (instead of stablecoin_reads) and when complete runs for the first time +--it should include existing stablecoin records +--refactor/remove stablecoins reads models, update downstream refs/logic, update workflows accordingly +--outline deployment steps for contract reads pipeline, tvl models, uniswap v1, stablecoins reads migration + +WITH verified_stablecoins AS ( + + SELECT + contract_address + FROM + {{ ref('defi__dim_stablecoins') }} + WHERE + is_verified + AND contract_address IS NOT NULL + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + contract_address, + NULL AS address, + 'totalSupply' AS function_name, + '0x18160ddd' AS function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input, + NULL :: VARIANT AS metadata, + 'stablecoins' AS protocol, + 'v1' AS version, + CONCAT('stablecoins','-','v1') AS platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','input','platform'] + ) }} AS stablecoins_reads_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + verified_stablecoins From b40c287e1e1026779a9ca8e454123377a554f95a Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 4 Dec 2025 13:31:40 -0700 Subject: [PATCH 076/164] table ref --- .../records/streamline__contract_reads_daily_records.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql index fe589f92..90274c06 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql @@ -16,6 +16,7 @@ {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} {% set _ = models.append(ref('silver_reads__lido_reads')) %} {% endif %} +{% set _ = models.append(ref('silver_reads__stablecoins_reads')) %} {% set _ = models.append(ref('silver_reads__uniswap_v2_reads')) %} {% set _ = models.append(ref('silver_reads__uniswap_v3_reads')) %} {% set _ = models.append(ref('silver_reads__uniswap_v4_reads')) %} From d71c511b5270527fb62f74cc76299c0e55ca2498 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 9 Dec 2025 11:49:35 -0700 Subject: [PATCH 077/164] integrate stablecoin reads into generic pipeline, add heal logic --- macros/global/variables/return_vars.sql | 13 +- ...reamline__contract_reads_daily_history.sql | 39 +++++ ..._contract_reads_daily_history_requests.sql | 106 +++++++++++ .../silver_reads__stablecoins_reads.sql | 8 +- .../reads/bronze/bronze__stablecoin_reads.sql | 16 -- .../bronze/bronze__stablecoin_reads_fr.sql | 16 -- .../reads/silver/silver__stablecoin_reads.sql | 109 ------------ .../streamline__stablecoin_reads_complete.sql | 49 ------ ...reamline__contract_reads_daily_history.sql | 141 --------------- ...eamline__contract_reads_daily_realtime.sql | 137 --------------- ...lecoins__stablecoins_reads_totalsupply.sql | 165 ++++++++++++++++++ ...ecoins__stablecoins_reads_totalsupply.yml} | 4 +- .../silver_stablecoins__supply_complete.sql | 18 +- 13 files changed, 321 insertions(+), 500 deletions(-) create mode 100644 models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history.sql create mode 100644 models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql delete mode 100644 models/curated_package/defi/stablecoins/reads/bronze/bronze__stablecoin_reads.sql delete mode 100644 models/curated_package/defi/stablecoins/reads/bronze/bronze__stablecoin_reads_fr.sql delete mode 100644 models/curated_package/defi/stablecoins/reads/silver/silver__stablecoin_reads.sql delete mode 100644 models/curated_package/defi/stablecoins/reads/streamline/complete/streamline__stablecoin_reads_complete.sql delete mode 100644 models/curated_package/defi/stablecoins/reads/streamline/history/streamline__contract_reads_daily_history.sql delete mode 100644 models/curated_package/defi/stablecoins/reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql create mode 100644 models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql rename models/curated_package/defi/stablecoins/{reads/silver/silver__stablecoin_reads.yml => silver/silver_stablecoins__stablecoins_reads_totalsupply.yml} (51%) diff --git a/macros/global/variables/return_vars.sql b/macros/global/variables/return_vars.sql index d65e7656..f7c30a19 100644 --- a/macros/global/variables/return_vars.sql +++ b/macros/global/variables/return_vars.sql @@ -282,6 +282,8 @@ {% set ns.CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST = get_var('CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST', []) %} {# SL Contract Reads Variables #} + {% set ns.CURATED_SL_CONTRACT_READS_START_DATE = get_var('CURATED_SL_CONTRACT_READS_START_DATE', '2025-06-10') %} + {% set ns.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_SQL_LIMIT = get_var('CURATED_SL_CONTRACT_READS_DAILY_REALTIME_SQL_LIMIT', 12000000) %} {% set ns.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_PRODUCER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_DAILY_REALTIME_PRODUCER_BATCH_SIZE', 240000) %} {% set ns.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_WORKER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_DAILY_REALTIME_WORKER_BATCH_SIZE', 60000) %} @@ -292,17 +294,6 @@ {% set ns.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_WORKER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_DAILY_HISTORY_WORKER_BATCH_SIZE', 60000) %} {% set ns.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_ASYNC_CONCURRENT_REQUESTS = get_var('CURATED_SL_CONTRACT_READS_DAILY_HISTORY_ASYNC_CONCURRENT_REQUESTS', 10) %} - {# SL Stablecoin Reads Variables #} - {% set ns.CURATED_SL_STABLECOIN_READS_REALTIME_SQL_LIMIT = get_var('CURATED_SL_STABLECOIN_READS_REALTIME_SQL_LIMIT', 3000) %} - {% set ns.CURATED_SL_STABLECOIN_READS_REALTIME_PRODUCER_BATCH_SIZE = get_var('CURATED_SL_STABLECOIN_READS_REALTIME_PRODUCER_BATCH_SIZE', 1500) %} - {% set ns.CURATED_SL_STABLECOIN_READS_REALTIME_WORKER_BATCH_SIZE = get_var('CURATED_SL_STABLECOIN_READS_REALTIME_WORKER_BATCH_SIZE', 500) %} - {% set ns.CURATED_SL_STABLECOIN_READS_REALTIME_ASYNC_CONCURRENT_REQUESTS = get_var('CURATED_SL_STABLECOIN_READS_REALTIME_ASYNC_CONCURRENT_REQUESTS', 5) %} - - {% set ns.CURATED_SL_STABLECOIN_READS_HISTORY_SQL_LIMIT = get_var('CURATED_SL_STABLECOIN_READS_HISTORY_SQL_LIMIT', 20000) %} - {% set ns.CURATED_SL_STABLECOIN_READS_HISTORY_PRODUCER_BATCH_SIZE = get_var('CURATED_SL_STABLECOIN_READS_HISTORY_PRODUCER_BATCH_SIZE', 1500) %} - {% set ns.CURATED_SL_STABLECOIN_READS_HISTORY_WORKER_BATCH_SIZE = get_var('CURATED_SL_STABLECOIN_READS_HISTORY_WORKER_BATCH_SIZE', 500) %} - {% set ns.CURATED_SL_STABLECOIN_READS_HISTORY_ASYNC_CONCURRENT_REQUESTS = get_var('CURATED_SL_STABLECOIN_READS_HISTORY_ASYNC_CONCURRENT_REQUESTS', 5) %} - {# Curated Bridge Variables #} {% set ns.CURATED_DEFI_BRIDGE_CONTRACT_MAPPING = get_var('CURATED_DEFI_BRIDGE_CONTRACT_MAPPING', {}) %} {% set ns.CURATED_DEFI_BRIDGE_ALLBRIDGE_SOURCE_CHAIN = get_var('CURATED_DEFI_BRIDGE_ALLBRIDGE_SOURCE_CHAIN', '') %} diff --git a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history.sql b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history.sql new file mode 100644 index 00000000..ef708adc --- /dev/null +++ b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history.sql @@ -0,0 +1,39 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +{{ config ( + materialized = "view", + tags = ['streamline','contract_reads','history','phase_4'] +) }} + +SELECT + * +FROM + {{ ref('streamline__contract_reads_daily_history_requests') }} + +{# Streamline Function Call #} +{% if execute %} + {% set params = { + "external_table": 'contract_reads', + "sql_limit": vars.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_SQL_LIMIT, + "producer_batch_size": vars.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_PRODUCER_BATCH_SIZE, + "worker_batch_size": vars.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_WORKER_BATCH_SIZE, + "async_concurrent_requests": vars.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_ASYNC_CONCURRENT_REQUESTS, + "sql_source": 'contract_reads_daily_history' + } %} + + {% set function_call_sql %} + {{ fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = this.schema ~ "." ~ this.identifier, + params = params + ) }} + {% endset %} + + {% do run_query(function_call_sql) %} + {{ log("Streamline function call: " ~ function_call_sql, info=true) }} +{% endif %} \ No newline at end of file diff --git a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql new file mode 100644 index 00000000..710c498e --- /dev/null +++ b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql @@ -0,0 +1,106 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{# Set up dbt configuration #} +{{ config ( + materialized = "table", + tags = ['streamline','contract_reads','history','phase_4'] +) }} + +WITH last_x_days AS ( + + SELECT + block_number, + block_date + FROM + {{ ref("_max_block_by_date") }} + WHERE block_date >= ('{{ vars.CURATED_SL_CONTRACT_READS_START_DATE }}' :: TIMESTAMP) :: DATE +), +to_do AS ( + SELECT + DISTINCT + d.block_number, + d.block_date, + t.contract_address, + t.address, + t.function_name, + t.function_sig, + t.input, + t.metadata, + t.protocol, + t.version, + t.platform + FROM + {{ ref("streamline__contract_reads_daily_records") }} t + CROSS JOIN last_x_days d + EXCEPT + SELECT + block_number, + block_date, + contract_address, + address, + function_name, + function_sig, + input, + metadata, + protocol, + version, + platform + FROM + {{ ref("streamline__contract_reads_daily_complete") }} + WHERE + block_date IS NOT NULL +) +SELECT + block_number, + DATE_PART('EPOCH_SECONDS', block_date) :: INT AS block_date_unix, + contract_address, + address, + function_name, + function_sig, + input, + metadata :: STRING AS metadata_str, + protocol, + version, + platform, + ROUND( + block_number, + -3 + ) AS partition_key, + OBJECT_CONSTRUCT( + 'data', OBJECT_CONSTRUCT( + 'id', CONCAT( + contract_address, + '-', + COALESCE(address,'null'), + '-', + input, + '-', + block_number + ), + 'jsonrpc', '2.0', + 'method', 'eth_call', + 'params', ARRAY_CONSTRUCT( + OBJECT_CONSTRUCT( + 'to', contract_address, + 'data', input + ), + utils.udf_int_to_hex(block_number) + ) + ), + 'headers', OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'x-fsc-livequery', 'true' + ), + 'method', 'POST', + 'secret_name', '{{ vars.GLOBAL_NODE_VAULT_PATH }}', + 'url', '{{ vars.GLOBAL_NODE_URL }}' + ) AS request +FROM + to_do +ORDER BY partition_key DESC, block_number DESC + +LIMIT {{ vars.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_SQL_LIMIT }} \ No newline at end of file diff --git a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql index 5e54fca6..2360a7e3 100644 --- a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql @@ -10,14 +10,8 @@ incremental_strategy = 'delete+insert', unique_key = 'stablecoins_reads_id', post_hook = '{{ unverify_stablecoins() }}', - tags = ['silver','contract_reads','heal','curated_daily'] + tags = ['silver','contract_reads','curated_daily'] ) }} ---make sure tags are accurate for workflows ---make sure complete model includes existing stablecoin records ---rename external table to contract_reads (instead of stablecoin_reads) and when complete runs for the first time ---it should include existing stablecoin records ---refactor/remove stablecoins reads models, update downstream refs/logic, update workflows accordingly ---outline deployment steps for contract reads pipeline, tvl models, uniswap v1, stablecoins reads migration WITH verified_stablecoins AS ( diff --git a/models/curated_package/defi/stablecoins/reads/bronze/bronze__stablecoin_reads.sql b/models/curated_package/defi/stablecoins/reads/bronze/bronze__stablecoin_reads.sql deleted file mode 100644 index a7d706da..00000000 --- a/models/curated_package/defi/stablecoins/reads/bronze/bronze__stablecoin_reads.sql +++ /dev/null @@ -1,16 +0,0 @@ -{# Log configuration details #} -{{ log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','stablecoin_reads','phase_4'] -) }} - -{# Main query starts here #} -{{ streamline_external_table_query( - source_name = 'contract_reads', - block_number = false, - contract_address = true, - data_not_null = false -) }} \ No newline at end of file diff --git a/models/curated_package/defi/stablecoins/reads/bronze/bronze__stablecoin_reads_fr.sql b/models/curated_package/defi/stablecoins/reads/bronze/bronze__stablecoin_reads_fr.sql deleted file mode 100644 index 488a663c..00000000 --- a/models/curated_package/defi/stablecoins/reads/bronze/bronze__stablecoin_reads_fr.sql +++ /dev/null @@ -1,16 +0,0 @@ -{# Log configuration details #} -{{ log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'view', - tags = ['bronze','stablecoin_reads','phase_4'] -) }} - -{# Main query starts here #} -{{ streamline_external_table_query_fr( - source_name = 'contract_reads', - block_number = false, - contract_address = true, - data_not_null = false -) }} \ No newline at end of file diff --git a/models/curated_package/defi/stablecoins/reads/silver/silver__stablecoin_reads.sql b/models/curated_package/defi/stablecoins/reads/silver/silver__stablecoin_reads.sql deleted file mode 100644 index e4b5b115..00000000 --- a/models/curated_package/defi/stablecoins/reads/silver/silver__stablecoin_reads.sql +++ /dev/null @@ -1,109 +0,0 @@ -{# Get variables #} -{% set vars = return_vars() %} - -{# Log configuration details #} -{{ log_model_details() }} - --- depends_on: {{ ref('bronze__stablecoin_reads') }} --- depends_on: {{ ref('price__ez_asset_metadata') }} -{{ config( - materialized = 'incremental', - unique_key = 'stablecoin_reads_id', - cluster_by = ['block_date'], - incremental_predicates = ["dynamic_range", "block_number"], - merge_exclude_columns = ["inserted_timestamp"], - post_hook = "{{ unverify_stablecoins() }}", - full_refresh = vars.GLOBAL_SILVER_FR_ENABLED, - tags = ['silver','defi','stablecoins','curated_daily'] -) }} - -WITH stablecoin_reads AS ( - - SELECT - VALUE :"BLOCK_NUMBER" :: NUMBER AS block_number, - ( - VALUE :"BLOCK_DATE_UNIX" :: TIMESTAMP - ) :: DATE AS block_date, - contract_address, - PARSE_JSON( - VALUE :"METADATA_STR" :: STRING - ) :: variant AS metadata, - DATA :result :: STRING AS result_hex, - _inserted_timestamp - FROM - -{% if is_incremental() %} -{{ ref('bronze__stablecoin_reads') }} -WHERE - _inserted_timestamp >= ( - SELECT - COALESCE(MAX(_inserted_timestamp), '1970-01-01') - FROM - {{ this }}) - AND DATA :result :: STRING <> '0x' - {% else %} - {{ ref('bronze__stablecoin_reads_fr') }} - WHERE - DATA :result :: STRING <> '0x' - {% endif %} - - qualify(ROW_NUMBER() over (PARTITION BY block_number, contract_address - ORDER BY - _inserted_timestamp DESC)) = 1 - ), -results AS ( - SELECT - block_number, - block_date, - contract_address, - metadata, - IFF( - C.decimals IS NULL, - 18, - C.decimals - ) AS decimals_adj, - result_hex AS amount_hex, - IFNULL( - CASE - WHEN LENGTH(amount_hex) <= 4300 - AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, - CASE - WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) - END - ) AS amount_raw, - IFF( - decimals_adj IS NULL, - NULL, - utils.udf_decimal_adjust( - amount_raw, - decimals_adj - ) - ) AS amount_precise, - amount_precise :: FLOAT AS amount, - _inserted_timestamp - FROM - stablecoin_reads s - LEFT JOIN {{ ref('core__dim_contracts') }} C - ON s.contract_address = C.address - WHERE - amount_raw IS NOT NULL - ) -SELECT - block_number, - block_date, - contract_address, - metadata, - decimals_adj AS decimals, - amount_hex, - amount_raw, - amount_precise, - amount, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address'] - ) }} AS stablecoin_reads_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - results diff --git a/models/curated_package/defi/stablecoins/reads/streamline/complete/streamline__stablecoin_reads_complete.sql b/models/curated_package/defi/stablecoins/reads/streamline/complete/streamline__stablecoin_reads_complete.sql deleted file mode 100644 index 1fea3ff1..00000000 --- a/models/curated_package/defi/stablecoins/reads/streamline/complete/streamline__stablecoin_reads_complete.sql +++ /dev/null @@ -1,49 +0,0 @@ -{# Get variables #} -{% set vars = return_vars() %} - -{# Log configuration details #} -{{ log_model_details() }} - --- depends on: {{ ref('bronze__stablecoin_reads') }} - -{# Set up dbt configuration #} -{{ config ( - materialized = 'incremental', - unique_key = 'stablecoin_reads_complete_id', - incremental_predicates = ['dynamic_range', 'partition_key'], - full_refresh = vars.GLOBAL_STREAMLINE_FR_ENABLED, - tags = ['streamline','stablecoin_reads','complete','phase_4'] -) }} - -SELECT - partition_key, - contract_address, - VALUE :"BLOCK_NUMBER" :: NUMBER AS block_number, - ( - VALUE :"BLOCK_DATE_UNIX" :: TIMESTAMP - ) :: DATE AS block_date, - file_name, - {{ dbt_utils.generate_surrogate_key( - ['contract_address', 'block_number'] - ) }} AS stablecoin_reads_complete_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - _inserted_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - -{% if is_incremental() %} -{{ ref('bronze__stablecoin_reads') }} -WHERE - _inserted_timestamp >= ( - SELECT - COALESCE (MAX(_inserted_timestamp), '1970-01-01' :: TIMESTAMP) - FROM - {{ this }}) - {% else %} - {{ ref('bronze__stablecoin_reads_fr') }} - {% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY stablecoin_reads_complete_id -ORDER BY - _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/curated_package/defi/stablecoins/reads/streamline/history/streamline__contract_reads_daily_history.sql b/models/curated_package/defi/stablecoins/reads/streamline/history/streamline__contract_reads_daily_history.sql deleted file mode 100644 index b17e0f11..00000000 --- a/models/curated_package/defi/stablecoins/reads/streamline/history/streamline__contract_reads_daily_history.sql +++ /dev/null @@ -1,141 +0,0 @@ -{# Get variables #} -{% set vars = return_vars() %} - -{# Log configuration details #} -{{ log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = "view", - tags = ['streamline','stablecoin_reads','history','phase_4'] -) }} - -WITH verified_stablecoins AS ( - SELECT - contract_address, - OBJECT_CONSTRUCT( - 'symbol', symbol, - 'name', name, - 'label', label, - 'decimals', decimals, - 'is_verified', is_verified - ) :: VARIANT AS metadata - FROM - {{ ref('defi__dim_stablecoins') }} - WHERE - is_verified - AND contract_address IS NOT NULL -), -max_blocks AS ( - SELECT - block_number, - block_date - FROM - {{ ref("_max_block_by_date") }} - WHERE - block_date >= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE -), -base AS ( - SELECT - s.contract_address, - s.metadata, - m.block_number, - m.block_date - FROM - verified_stablecoins s - CROSS JOIN max_blocks m - LEFT JOIN {{ ref('streamline__stablecoin_reads_complete') }} c - ON s.contract_address = c.contract_address - AND m.block_number = c.block_number - WHERE - c.contract_address IS NULL - AND m.block_date < ( - SELECT MAX(block_date) - FROM max_blocks - ) -), -function_sigs AS ( - SELECT - '0x18160ddd' AS function_sig, - 'totalSupply' AS function_name -), -ready_reads AS ( - SELECT - contract_address, - block_number, - block_date, - function_sig, - RPAD( - function_sig, - 64, - '0' - ) AS input, - metadata - FROM - base - JOIN function_sigs - ON 1 = 1 -) -SELECT - contract_address, - block_number, - DATE_PART('EPOCH_SECONDS', block_date) :: INT AS block_date_unix, - ROUND(block_number,-3) AS partition_key, - function_sig, - input, - metadata :: STRING AS metadata_str, - live.udf_api( - 'POST', - '{{ vars.GLOBAL_NODE_URL }}', - OBJECT_CONSTRUCT( - 'Content-Type', 'application/json', - 'fsc-quantum-state', 'streamline' - ), - OBJECT_CONSTRUCT( - 'method', 'eth_call', - 'jsonrpc', '2.0', - 'params', [{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(block_number)], - 'id', concat_ws( - '-', - contract_address, - input, - block_number - ) - ), - '{{ vars.GLOBAL_NODE_VAULT_PATH }}' - ) AS request -FROM - ready_reads -WHERE - EXISTS ( - SELECT - 1 - FROM - ready_reads - LIMIT - 1 - ) - -{# Streamline Function Call #} -{% if execute %} - {% set params = { - "external_table": 'contract_reads', - "sql_limit": vars.CURATED_SL_STABLECOIN_READS_HISTORY_SQL_LIMIT, - "producer_batch_size": vars.CURATED_SL_STABLECOIN_READS_HISTORY_PRODUCER_BATCH_SIZE, - "worker_batch_size": vars.CURATED_SL_STABLECOIN_READS_HISTORY_WORKER_BATCH_SIZE, - "async_concurrent_requests": vars.CURATED_SL_STABLECOIN_READS_HISTORY_ASYNC_CONCURRENT_REQUESTS, - "sql_source": 'contract_reads_daily_history' - } %} - - {% set function_call_sql %} - {{ fsc_utils.if_data_call_function_v2( - func = 'streamline.udf_bulk_rest_api_v2', - target = this.schema ~ "." ~ this.identifier, - params = params - ) }} - {% endset %} - - {% do run_query(function_call_sql) %} - {{ log("Streamline function call: " ~ function_call_sql, info=true) }} -{% endif %} - diff --git a/models/curated_package/defi/stablecoins/reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql b/models/curated_package/defi/stablecoins/reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql deleted file mode 100644 index 0e8fd500..00000000 --- a/models/curated_package/defi/stablecoins/reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql +++ /dev/null @@ -1,137 +0,0 @@ -{# Get variables #} -{% set vars = return_vars() %} - -{# Log configuration details #} -{{ log_model_details() }} - -{# Set up dbt configuration #} -{{ config ( - materialized = "view", - tags = ['streamline','stablecoin_reads','realtime','phase_4'] -) }} - -WITH verified_stablecoins AS ( - SELECT - contract_address, - OBJECT_CONSTRUCT( - 'symbol', symbol, - 'name', name, - 'label', label, - 'decimals', decimals, - 'is_verified', is_verified - ) :: VARIANT AS metadata - FROM - {{ ref('defi__dim_stablecoins') }} - WHERE - is_verified - AND contract_address IS NOT NULL -), -max_blocks AS ( - SELECT - block_number, - block_date - FROM - {{ ref("_max_block_by_date") }} - WHERE - block_date >= DATEADD('day',-4,SYSDATE()) --last 3 max block_number by date -), -base AS ( - SELECT - s.contract_address, - s.metadata, - m.block_number, - m.block_date - FROM - verified_stablecoins s - CROSS JOIN max_blocks m - LEFT JOIN {{ ref('streamline__stablecoin_reads_complete') }} c - ON s.contract_address = c.contract_address - AND m.block_number = c.block_number - WHERE - c.contract_address IS NULL -), -function_sigs AS ( - SELECT - '0x18160ddd' AS function_sig, - 'totalSupply' AS function_name -), -ready_reads AS ( - SELECT - contract_address, - block_number, - block_date, - function_sig, - RPAD( - function_sig, - 64, - '0' - ) AS input, - metadata - FROM - base - JOIN function_sigs - ON 1 = 1 -) -SELECT - contract_address, - block_number, - DATE_PART('EPOCH_SECONDS', block_date) :: INT AS block_date_unix, - ROUND(block_number,-3) AS partition_key, - function_sig, - input, - metadata :: STRING AS metadata_str, - live.udf_api( - 'POST', - '{{ vars.GLOBAL_NODE_URL }}', - OBJECT_CONSTRUCT( - 'Content-Type', 'application/json', - 'fsc-quantum-state', 'streamline' - ), - OBJECT_CONSTRUCT( - 'method', 'eth_call', - 'jsonrpc', '2.0', - 'params', [{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(block_number)], - 'id', concat_ws( - '-', - contract_address, - input, - block_number - ) - ), - '{{ vars.GLOBAL_NODE_VAULT_PATH }}' - ) AS request -FROM - ready_reads -WHERE - EXISTS ( - SELECT - 1 - FROM - ready_reads - LIMIT - 1 - ) - -{# Streamline Function Call #} -{% if execute %} - {% set params = { - "external_table" :"contract_reads", - "sql_limit" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_SQL_LIMIT, - "producer_batch_size" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_PRODUCER_BATCH_SIZE, - "worker_batch_size" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_WORKER_BATCH_SIZE, - "async_concurrent_requests" : vars.CURATED_SL_STABLECOIN_READS_REALTIME_ASYNC_CONCURRENT_REQUESTS, - "sql_source" : 'contract_reads_daily_realtime' - } %} - - {% set function_call_sql %} - {{ fsc_utils.if_data_call_function_v2( - func = 'streamline.udf_bulk_rest_api_v2', - target = this.schema ~ "." ~ this.identifier, - params = params - ) }} - {% endset %} - - {% do run_query(function_call_sql) %} - {{ log("Streamline function call: " ~ function_call_sql, info=true) }} -{% endif %} - diff --git a/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql new file mode 100644 index 00000000..d33cad87 --- /dev/null +++ b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql @@ -0,0 +1,165 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +-- depends_on: {{ ref('bronze__contract_reads') }} +-- depends_on: {{ ref('price__ez_asset_metadata') }} +{{ config( + materialized = 'incremental', + unique_key = 'stablecoins_reads_totalsupply_id', + cluster_by = ['block_date'], + incremental_predicates = ["dynamic_range", "block_number"], + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "{{ unverify_stablecoins() }}", + full_refresh = vars.GLOBAL_SILVER_FR_ENABLED, + tags = ['silver','defi','stablecoins','curated_daily'] +) }} + +WITH stablecoin_reads AS ( + + SELECT + VALUE :"BLOCK_NUMBER" :: NUMBER AS block_number, + ( + VALUE :"BLOCK_DATE_UNIX" :: TIMESTAMP + ) :: DATE AS block_date, + contract_address, + PARSE_JSON( + VALUE :"METADATA_STR" :: STRING + ) :: variant AS metadata, + DATA :result :: STRING AS result_hex, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze__contract_reads') }} +WHERE + _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1970-01-01') + FROM + {{ this }}) + AND DATA :result :: STRING <> '0x' + {% else %} + {{ ref('bronze__contract_reads_fr') }} + WHERE + DATA :result :: STRING <> '0x' + {% endif %} + + qualify(ROW_NUMBER() over (PARTITION BY block_number, contract_address + ORDER BY + _inserted_timestamp DESC)) = 1 + ), + results AS ( + SELECT + block_number, + block_date, + contract_address, + metadata, + metadata :decimals AS decimals, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + IFF( + decimals IS NULL, + NULL, + utils.udf_decimal_adjust( + amount_raw, + decimals + ) + ) AS amount_precise, + amount_precise :: FLOAT AS amount, + _inserted_timestamp + FROM + stablecoin_reads + WHERE + amount_raw IS NOT NULL + ), + +{% if is_incremental() %} +heal_model AS ( + SELECT + t.block_number, + t.block_date, + t.contract_address, + t.metadata, + d.decimals AS decimals_heal, + t.amount_hex, + t.amount_raw, + IFF( + decimals_heal IS NULL, + NULL, + utils.udf_decimal_adjust( + t.amount_raw, + decimals_heal + ) + ) AS amount_precise_heal, + amount_precise_heal :: FLOAT AS amount_heal, + t._inserted_timestamp + FROM + {{ this }} + t + INNER JOIN {{ ref('defi__dim_stablecoins') }} + d + ON t.contract_address = d.contract_address + WHERE + t.decimals IS NULL + AND d.decimals IS NOT NULL +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + results + +{% if is_incremental() %} +UNION ALL +SELECT + block_number, + block_date, + contract_address, + metadata, + decimals_heal AS decimals, + amount_hex, + amount_raw, + amount_precise_heal AS amount_precise, + amount_heal AS amount, + _inserted_timestamp +FROM + heal_model +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + metadata, + decimals, + amount_hex, + amount_raw, + amount_precise, + amount, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address'] + ) }} AS stablecoins_reads_totalsupply_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL + +{% if is_incremental() %} +qualify(ROW_NUMBER() over (PARTITION BY stablecoins_reads_totalsupply_id +ORDER BY + modified_timestamp DESC)) = 1 +{% endif %} diff --git a/models/curated_package/defi/stablecoins/reads/silver/silver__stablecoin_reads.yml b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.yml similarity index 51% rename from models/curated_package/defi/stablecoins/reads/silver/silver__stablecoin_reads.yml rename to models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.yml index cd629ad2..a0bb6669 100644 --- a/models/curated_package/defi/stablecoins/reads/silver/silver__stablecoin_reads.yml +++ b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: silver__stablecoin_reads + - name: silver_stablecoins__stablecoins_reads_totalsupply tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - stablecoin_reads_id \ No newline at end of file + - stablecoins_reads_totalsupply_id \ No newline at end of file diff --git a/models/curated_package/defi/stablecoins/silver/silver_stablecoins__supply_complete.sql b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__supply_complete.sql index 158aa88b..a9dfad8c 100644 --- a/models/curated_package/defi/stablecoins/silver/silver_stablecoins__supply_complete.sql +++ b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__supply_complete.sql @@ -10,7 +10,7 @@ unique_key = ["stablecoins_supply_complete_id"], cluster_by = ['block_date'], post_hook = '{{ unverify_stablecoins() }}', - tags = ['silver','defi','stablecoins','heal','curated_daily'] + tags = ['silver','defi','stablecoins','curated_daily'] ) }} WITH @@ -23,7 +23,7 @@ max_ts AS ( incremental_dates AS ( -- Get all distinct dates that have been updated in any source table SELECT DISTINCT block_date - FROM {{ ref('silver__stablecoin_reads') }} + FROM {{ ref('silver_stablecoins__stablecoins_reads_totalsupply') }} WHERE modified_timestamp > (SELECT max_modified_timestamp FROM max_ts) UNION SELECT DISTINCT block_date @@ -54,7 +54,7 @@ total_supply AS ( metadata :label :: STRING AS label, metadata :decimals :: INTEGER AS decimals FROM - {{ ref('silver__stablecoin_reads') }} + {{ ref('silver_stablecoins__stablecoins_reads_totalsupply') }} {% if is_incremental() %} WHERE @@ -310,9 +310,7 @@ all_supply AS ( AND s.contract_address = h.contract_address ), -{% if is_incremental() and var( - 'HEAL_MODEL' -) %} +{% if is_incremental() %} heal_model AS ( SELECT t.block_date, @@ -367,9 +365,7 @@ FINAL AS ( FROM all_supply -{% if is_incremental() and var( - 'HEAL_MODEL' -) %} +{% if is_incremental() %} UNION ALL SELECT block_date, @@ -417,9 +413,7 @@ SELECT FROM FINAL -{% if is_incremental() and var( - 'HEAL_MODEL' -) %} +{% if is_incremental() %} qualify(ROW_NUMBER() over (PARTITION BY stablecoins_supply_complete_id ORDER BY modified_timestamp DESC)) = 1 From 9b12c8aabc01104a50b8fa5ff0533606cf9f7bd5 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 9 Dec 2025 11:53:02 -0700 Subject: [PATCH 078/164] tags --- .../streamline/records/aave/silver_reads__aave_v1_reads.sql | 2 +- .../streamline/records/aave/silver_reads__aave_v2_reads.sql | 2 +- .../streamline/records/aave/silver_reads__aave_v3_reads.sql | 2 +- .../streamline/records/curve/silver_reads__curve_reads.sql | 2 +- .../streamline/records/lido/silver_reads__lido_reads.sql | 2 +- .../records/stablecoins/silver_reads__stablecoins_reads.sql | 2 +- .../records/tornado_cash/silver_reads__tornado_cash_reads.sql | 2 +- .../records/uniswap/silver_reads__uniswap_v2_reads.sql | 2 +- .../records/uniswap/silver_reads__uniswap_v3_reads.sql | 2 +- .../records/uniswap/silver_reads__uniswap_v4_reads.sql | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql index c47f3145..861a1ca5 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'aave_v1_reads_id', - tags = ['silver','contract_reads','curated_daily'] + tags = ['silver','contract_reads'] ) }} WITH all_tokens AS ( diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql index 039eab5f..a94795f0 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'aave_v2_reads_id', - tags = ['silver','contract_reads','curated_daily'] + tags = ['silver','contract_reads'] ) }} WITH all_tokens AS ( diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql index d9d04b5c..03684d09 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql @@ -6,7 +6,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'aave_v3_reads_id', - tags = ['silver','contract_reads','curated_daily'] + tags = ['silver','contract_reads'] ) }} WITH all_tokens AS ( diff --git a/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql b/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql index 9b8150e9..61a7e49e 100644 --- a/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'curve_reads_id', - tags = ['silver','contract_reads','curated_daily'] + tags = ['silver','contract_reads'] ) }} WITH liquidity_pools AS ( diff --git a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql index b39d4792..95504dca 100644 --- a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'lido_reads_id', - tags = ['silver','contract_reads','curated_daily'] + tags = ['silver','contract_reads'] ) }} WITH pooled_assets AS ( diff --git a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql index 2360a7e3..5411a265 100644 --- a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql @@ -10,7 +10,7 @@ incremental_strategy = 'delete+insert', unique_key = 'stablecoins_reads_id', post_hook = '{{ unverify_stablecoins() }}', - tags = ['silver','contract_reads','curated_daily'] + tags = ['silver','contract_reads'] ) }} WITH verified_stablecoins AS ( diff --git a/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql b/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql index e67bdf31..46499b22 100644 --- a/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'tornado_cash_reads_id', - tags = ['silver','contract_reads','curated_daily'] + tags = ['silver','contract_reads'] ) }} WITH mixers AS ( diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql index 19fd2262..3f6983b5 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'uniswap_v2_reads_id', - tags = ['silver','contract_reads','curated_daily'] + tags = ['silver','contract_reads'] ) }} WITH liquidity_pools AS ( diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql index 0570f15c..797801d2 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'uniswap_v3_reads_id', - tags = ['silver','contract_reads','curated_daily'] + tags = ['silver','contract_reads'] ) }} WITH liquidity_pools AS ( diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql index dc8b7766..5dad6ca1 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'uniswap_v4_reads_id', - tags = ['silver','contract_reads','curated_daily'] + tags = ['silver','contract_reads'] ) }} WITH liquidity_pools AS ( From a5039ebc9988ffad149e35b774beca8086d233ad Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 9 Dec 2025 12:03:17 -0700 Subject: [PATCH 079/164] misc tags and refs to silver --- .../balances__ez_balances_erc20_daily.sql | 2 +- .../silver/silver__balances_erc20_daily.sql | 2 +- .../balances__ez_balances_native_daily.sql | 2 +- .../silver/silver__balances_native_daily.sql | 2 +- .../silver_reads__stablecoins_reads.sql | 2 +- ...lecoins__stablecoins_reads_totalsupply.sql | 96 ++++++++----------- .../silver_stablecoins__supply_complete.sql | 2 +- 7 files changed, 48 insertions(+), 60 deletions(-) diff --git a/models/balances_package/erc20/gold/balances__ez_balances_erc20_daily.sql b/models/balances_package/erc20/gold/balances__ez_balances_erc20_daily.sql index 6a8628cd..484fdadb 100644 --- a/models/balances_package/erc20/gold/balances__ez_balances_erc20_daily.sql +++ b/models/balances_package/erc20/gold/balances__ez_balances_erc20_daily.sql @@ -18,7 +18,7 @@ ], merge_exclude_columns = ["inserted_timestamp"], full_refresh = vars.GLOBAL_GOLD_FR_ENABLED, - tags = ['gold','balances','erc20','heal','phase_4'] + tags = ['gold','balances','erc20','heal','curated_daily','phase_4'] ) }} WITH balances AS ( diff --git a/models/balances_package/erc20/silver/silver__balances_erc20_daily.sql b/models/balances_package/erc20/silver/silver__balances_erc20_daily.sql index 1fd09784..789cc925 100644 --- a/models/balances_package/erc20/silver/silver__balances_erc20_daily.sql +++ b/models/balances_package/erc20/silver/silver__balances_erc20_daily.sql @@ -15,7 +15,7 @@ merge_exclude_columns = ["inserted_timestamp"], post_hook = "{{ unverify_balances() }}", full_refresh = vars.GLOBAL_SILVER_FR_ENABLED, - tags = ['silver','balances','erc20','heal','phase_4'] + tags = ['silver','balances','erc20','heal','curated_daily','phase_4'] ) }} SELECT diff --git a/models/balances_package/native/gold/balances__ez_balances_native_daily.sql b/models/balances_package/native/gold/balances__ez_balances_native_daily.sql index be50569d..1eab308c 100644 --- a/models/balances_package/native/gold/balances__ez_balances_native_daily.sql +++ b/models/balances_package/native/gold/balances__ez_balances_native_daily.sql @@ -13,7 +13,7 @@ post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(address)", merge_exclude_columns = ["inserted_timestamp"], full_refresh = vars.GLOBAL_GOLD_FR_ENABLED, - tags = ['gold','balances','native','phase_4'] + tags = ['gold','balances','native','curated_daily','phase_4'] ) }} WITH balances AS ( diff --git a/models/balances_package/native/silver/silver__balances_native_daily.sql b/models/balances_package/native/silver/silver__balances_native_daily.sql index 687a0763..ad31e507 100644 --- a/models/balances_package/native/silver/silver__balances_native_daily.sql +++ b/models/balances_package/native/silver/silver__balances_native_daily.sql @@ -12,7 +12,7 @@ incremental_predicates = ["dynamic_range", "block_number"], merge_exclude_columns = ["inserted_timestamp"], full_refresh = vars.GLOBAL_SILVER_FR_ENABLED, - tags = ['silver','balances','native','phase_4'] + tags = ['silver','balances','native','curated_daily','phase_4'] ) }} SELECT diff --git a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql index 5411a265..7a98d7c5 100644 --- a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql @@ -10,7 +10,7 @@ incremental_strategy = 'delete+insert', unique_key = 'stablecoins_reads_id', post_hook = '{{ unverify_stablecoins() }}', - tags = ['silver','contract_reads'] + tags = ['silver','contract_reads','heal'] ) }} WITH verified_stablecoins AS ( diff --git a/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql index d33cad87..97e99d3c 100644 --- a/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql +++ b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql @@ -14,74 +14,62 @@ merge_exclude_columns = ["inserted_timestamp"], post_hook = "{{ unverify_stablecoins() }}", full_refresh = vars.GLOBAL_SILVER_FR_ENABLED, - tags = ['silver','defi','stablecoins','curated_daily'] + tags = ['silver','defi','stablecoins','heal','curated_daily'] ) }} WITH stablecoin_reads AS ( SELECT - VALUE :"BLOCK_NUMBER" :: NUMBER AS block_number, - ( - VALUE :"BLOCK_DATE_UNIX" :: TIMESTAMP - ) :: DATE AS block_date, + block_number, + block_date, contract_address, - PARSE_JSON( - VALUE :"METADATA_STR" :: STRING - ) :: variant AS metadata, - DATA :result :: STRING AS result_hex, + metadata, + result_hex, _inserted_timestamp FROM + {{ ref('silver__contract_reads') }} {% if is_incremental() %} -{{ ref('bronze__contract_reads') }} WHERE - _inserted_timestamp >= ( + modified_timestamp > ( SELECT - COALESCE(MAX(_inserted_timestamp), '1970-01-01') + MAX(modified_timestamp) FROM - {{ this }}) - AND DATA :result :: STRING <> '0x' - {% else %} - {{ ref('bronze__contract_reads_fr') }} - WHERE - DATA :result :: STRING <> '0x' - {% endif %} - - qualify(ROW_NUMBER() over (PARTITION BY block_number, contract_address - ORDER BY - _inserted_timestamp DESC)) = 1 - ), - results AS ( - SELECT - block_number, - block_date, - contract_address, - metadata, - metadata :decimals AS decimals, - result_hex AS amount_hex, - IFNULL( + {{ this }} + ) +{% endif %} +), +results AS ( + SELECT + block_number, + block_date, + contract_address, + metadata, + metadata :decimals AS decimals, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, CASE - WHEN LENGTH(amount_hex) <= 4300 - AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, - CASE - WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) - END - ) AS amount_raw, - IFF( - decimals IS NULL, - NULL, - utils.udf_decimal_adjust( - amount_raw, - decimals - ) - ) AS amount_precise, - amount_precise :: FLOAT AS amount, - _inserted_timestamp - FROM - stablecoin_reads - WHERE - amount_raw IS NOT NULL - ), + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + IFF( + decimals IS NULL, + NULL, + utils.udf_decimal_adjust( + amount_raw, + decimals + ) + ) AS amount_precise, + amount_precise :: FLOAT AS amount, + _inserted_timestamp + FROM + stablecoin_reads + WHERE + amount_raw IS NOT NULL + ), {% if is_incremental() %} heal_model AS ( diff --git a/models/curated_package/defi/stablecoins/silver/silver_stablecoins__supply_complete.sql b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__supply_complete.sql index a9dfad8c..815679bd 100644 --- a/models/curated_package/defi/stablecoins/silver/silver_stablecoins__supply_complete.sql +++ b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__supply_complete.sql @@ -10,7 +10,7 @@ unique_key = ["stablecoins_supply_complete_id"], cluster_by = ['block_date'], post_hook = '{{ unverify_stablecoins() }}', - tags = ['silver','defi','stablecoins','curated_daily'] + tags = ['silver','defi','stablecoins','heal','curated_daily'] ) }} WITH From af81723414b3f887389cee7dd878d9e036cb1432 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 9 Dec 2025 13:52:34 -0700 Subject: [PATCH 080/164] depends --- .../silver_stablecoins__stablecoins_reads_totalsupply.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql index 97e99d3c..5dce13c5 100644 --- a/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql +++ b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql @@ -4,8 +4,8 @@ {# Log configuration details #} {{ log_model_details() }} --- depends_on: {{ ref('bronze__contract_reads') }} -- depends_on: {{ ref('price__ez_asset_metadata') }} +-- depends_on: {{ ref('defi__dim_stablecoins') }} {{ config( materialized = 'incremental', unique_key = 'stablecoins_reads_totalsupply_id', From 0fab6902d1da68b4d55a288f2b0c172bab722e6d Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 9 Dec 2025 13:56:42 -0700 Subject: [PATCH 081/164] stablecoin metadata --- .../stablecoins/silver_reads__stablecoins_reads.sql | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql index 7a98d7c5..dcd76f20 100644 --- a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql @@ -16,7 +16,14 @@ WITH verified_stablecoins AS ( SELECT - contract_address + contract_address, + OBJECT_CONSTRUCT( + 'symbol', symbol, + 'name', name, + 'label', label, + 'decimals', decimals, + 'is_verified', is_verified + ) :: VARIANT AS metadata FROM {{ ref('defi__dim_stablecoins') }} WHERE @@ -42,7 +49,7 @@ SELECT 64, '0' ) AS input, - NULL :: VARIANT AS metadata, + metadata, 'stablecoins' AS protocol, 'v1' AS version, CONCAT('stablecoins','-','v1') AS platform, From 8b225f3a4f16b25f41130a15fe82942e2a5206f1 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 9 Dec 2025 14:45:32 -0700 Subject: [PATCH 082/164] stablecoin reads filter --- ...tablecoins__stablecoins_reads_totalsupply.sql | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql index 5dce13c5..134c5b44 100644 --- a/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql +++ b/models/curated_package/defi/stablecoins/silver/silver_stablecoins__stablecoins_reads_totalsupply.sql @@ -28,15 +28,17 @@ WITH stablecoin_reads AS ( _inserted_timestamp FROM {{ ref('silver__contract_reads') }} + WHERE + protocol = 'stablecoins' + AND function_name = 'totalSupply' {% if is_incremental() %} -WHERE - modified_timestamp > ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) {% endif %} ), results AS ( From 1d3bd8fc76cd343bdc32a8e45a606a28f24f7aad Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 9 Dec 2025 15:13:16 -0700 Subject: [PATCH 083/164] concat --- .../records/stablecoins/silver_reads__stablecoins_reads.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql index dcd76f20..41b8531a 100644 --- a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql @@ -52,7 +52,7 @@ SELECT metadata, 'stablecoins' AS protocol, 'v1' AS version, - CONCAT('stablecoins','-','v1') AS platform, + CONCAT(protocol,'-',version) AS platform, {{ dbt_utils.generate_surrogate_key( ['contract_address','input','platform'] ) }} AS stablecoins_reads_id, From 37c5ffe41e1b6b63de08220f2d90870c5bcf6763 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 10 Dec 2025 10:34:05 -0700 Subject: [PATCH 084/164] tests --- .../silver/silver__contract_reads.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/models/curated_package/contract_reads/silver/silver__contract_reads.yml b/models/curated_package/contract_reads/silver/silver__contract_reads.yml index 1446c63d..eb2b826b 100644 --- a/models/curated_package/contract_reads/silver/silver__contract_reads.yml +++ b/models/curated_package/contract_reads/silver/silver__contract_reads.yml @@ -4,4 +4,17 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - contract_reads_id \ No newline at end of file + - contract_reads_id + columns: + - name: BLOCK_DATE + tests: + - not_null + - name: CONTRACT_ADDRESS + tests: + - not_null + - name: PLATFORM + tests: + - not_null + - name: RESULT_HEX + tests: + - not_null \ No newline at end of file From 8f0abc44773f165a6573e156ad7f0930a1492085 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 10 Dec 2025 10:52:57 -0700 Subject: [PATCH 085/164] trillions filter --- .../defi/tvl/gold/defi__ez_protocol_tvl.sql | 7 +- .../defi/tvl/gold/defi__ez_protocol_tvl.yml | 22 +++++ .../tvl/silver/silver_tvl__complete_tvl.sql | 1 + .../doc_descriptions/defi/ez_protocol_tvl.md | 96 +++++++++++++++++++ 4 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.yml create mode 100644 models/doc_descriptions/defi/ez_protocol_tvl.md diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql index 5ab5230a..24a089a6 100644 --- a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql @@ -24,12 +24,15 @@ WITH complete_tvl AS ( amount_raw, amount_precise, amount, - amount_usd, + CASE + WHEN t.amount_usd < 1e12 THEN t.amount_usd + ELSE NULL + END AS amount_usd, protocol, version, platform FROM - {{ ref('silver_tvl__complete_tvl') }} + {{ ref('silver_tvl__complete_tvl') }} t {% if is_incremental() %} WHERE diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.yml b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.yml new file mode 100644 index 00000000..931c25ff --- /dev/null +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.yml @@ -0,0 +1,22 @@ +version: 2 +models: + - name: defi__ez_protocol_tvl + description: '{{ doc("ez_protocol_tvl_table_doc") }}' + + columns: + - name: BLOCK_DATE + description: '{{ doc("ez_protocol_tvl_block_date") }}' + - name: TVL_USD + description: '{{ doc("ez_protocol_tvl_tvl_usd") }}' + - name: PROTOCOL + description: '{{ doc("ez_protocol_tvl_protocol") }}' + - name: VERSION + description: '{{ doc("ez_protocol_tvl_version") }}' + - name: PLATFORM + description: '{{ doc("ez_protocol_tvl_platform") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("general_inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("general_modified_timestamp") }}' + - name: EZ_PROTOCOL_TVL_ID + description: '{{ doc("general_pk") }}' \ No newline at end of file diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index 488b2992..c4b19641 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -159,6 +159,7 @@ heal_model AS ( 23, A.block_date ) = p1.hour + WHERE t.decimals IS NULL ), {% endif %} diff --git a/models/doc_descriptions/defi/ez_protocol_tvl.md b/models/doc_descriptions/defi/ez_protocol_tvl.md new file mode 100644 index 00000000..8a8a6e88 --- /dev/null +++ b/models/doc_descriptions/defi/ez_protocol_tvl.md @@ -0,0 +1,96 @@ +{% docs ez_protocol_tvl_table_doc %} + +## What + +This table provides daily Total Value Locked (TVL) metrics for DeFi protocols across EVM-compatible blockchains. It aggregates values by platform to track liquidity and value deposited in various DeFi applications. Methods may vary by protocol and blockchain. + +## Key Use Cases + +- Tracking protocol TVL growth and trends over time +- Comparing TVL across different protocols and platforms +- Analyzing protocol adoption and liquidity depth +- Building TVL-based dashboards and metrics + +## Commonly-used Fields + +- `block_date`: Date of the TVL snapshot +- `tvl_usd`: Total Value Locked in USD +- `protocol`: Name of the DeFi protocol (e.g., Uniswap, Aave) +- `version`: Protocol version (e.g., v2, v3) +- `platform`: Specific deployment or platform identifier + +## Sample queries + +```sql +-- Latest TVL by protocol +SELECT + protocol, + version, + tvl_usd +FROM .defi.ez_protocol_tvl +WHERE block_date = CURRENT_DATE - 1 +ORDER BY tvl_usd DESC; + +-- Daily TVL trend for a specific protocol in aggregate +SELECT + block_date, + SUM(tvl_usd) AS tvl_usd_total +FROM .defi.ez_protocol_tvl +WHERE protocol = 'uniswap' + AND block_date >= CURRENT_DATE - 30 +GROUP BY block_date, protocol +ORDER BY block_date DESC; + +-- TVL comparison across platforms +SELECT + block_date, + platform, + SUM(tvl_usd) AS total_tvl +FROM .defi.ez_protocol_tvl +WHERE block_date >= CURRENT_DATE - 7 +GROUP BY block_date, platform +ORDER BY block_date DESC, total_tvl DESC; +``` + +{% enddocs %} + +{% docs ez_protocol_tvl_block_date %} + +The date of the daily TVL snapshot. + +Example: '2025-06-10' + +{% enddocs %} + +{% docs ez_protocol_tvl_tvl_usd %} + +The total value locked in USD for the protocol on the given date. Values exceeding $1 trillion (1e12) are excluded to filter out erroneous pricing data. + +Example: 1500000000 + +{% enddocs %} + +{% docs ez_protocol_tvl_protocol %} + +The name of the DeFi protocol. + +Example: 'Uniswap' + +{% enddocs %} + +{% docs ez_protocol_tvl_version %} + +The version of the protocol deployment. + +Example: 'v3' + +{% enddocs %} + +{% docs ez_protocol_tvl_platform %} + +The specific platform or deployment identifier for the protocol. + +Example: 'uniswap-v3-ethereum' + +{% enddocs %} + From cba36ff03bfec8275cc631ff92d9d04ed05c4503 Mon Sep 17 00:00:00 2001 From: Austin Date: Wed, 10 Dec 2025 15:08:48 -0500 Subject: [PATCH 086/164] packages --- packages.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages.yml b/packages.yml index f3c70125..28ef3f2b 100644 --- a/packages.yml +++ b/packages.yml @@ -1,11 +1,7 @@ packages: - - package: calogica/dbt_expectations - version: 0.8.2 - package: dbt-labs/dbt_external_tables version: 0.8.2 - - package: dbt-labs/dbt_utils - version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.39.1 + revision: v1.41.1 - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"] From e8d3f87579b64b651ca9cdb5d1e5921b5a39f1e2 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 10 Dec 2025 13:19:45 -0700 Subject: [PATCH 087/164] aave token addresses --- .../records/aave/silver_reads__aave_v2_reads.sql | 7 ++++++- .../records/aave/silver_reads__aave_v3_reads.sql | 6 +++++- .../defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql | 7 ++++--- .../defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql | 7 ++++--- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql index a94795f0..4206fb9c 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql @@ -15,6 +15,7 @@ WITH all_tokens AS ( SELECT atoken_address AS contract_address, + underlying_address, protocol, version, CONCAT( @@ -40,6 +41,7 @@ AND modified_timestamp > ( UNION SELECT atoken_address AS contract_address, + underlying_address, protocol, version, CONCAT( @@ -73,7 +75,10 @@ SELECT 64, '0' ) AS input, - NULL :: VARIANT AS metadata, + OBJECT_CONSTRUCT( + 'underlying_address', + underlying_address + ) :: variant AS metadata, protocol, version, platform, diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql index 03684d09..79d6cd5f 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql @@ -13,6 +13,7 @@ WITH all_tokens AS ( SELECT atoken_address AS contract_address, + underlying_address, protocol, version, CONCAT( @@ -44,7 +45,10 @@ SELECT 64, '0' ) AS input, - NULL :: VARIANT AS metadata, + OBJECT_CONSTRUCT( + 'underlying_address', + underlying_address + ) :: variant AS metadata, protocol, version, platform, diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql index 395afdd7..39f81716 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql @@ -14,7 +14,7 @@ WITH reads AS ( SELECT block_number, block_date, - contract_address, + contract_address AS atoken_address, address, result_hex AS amount_hex, IFNULL( @@ -25,6 +25,7 @@ WITH reads AS ( WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) END ) AS amount_raw, + metadata :underlying_address :: STRING AS underlying_address, protocol, version, platform, @@ -52,8 +53,8 @@ AND modified_timestamp > ( SELECT block_number, block_date, - contract_address, - address, + underlying_address AS contract_address, + atoken_address AS address, amount_hex, amount_raw, protocol, diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql index f2d266d9..f39e2fd5 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql @@ -14,7 +14,7 @@ WITH reads AS ( SELECT block_number, block_date, - contract_address, + contract_address AS atoken_address, address, result_hex AS amount_hex, IFNULL( @@ -25,6 +25,7 @@ WITH reads AS ( WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) END ) AS amount_raw, + metadata :underlying_address :: STRING AS underlying_address, protocol, version, platform, @@ -52,8 +53,8 @@ AND modified_timestamp >= ( SELECT block_number, block_date, - contract_address, - address, + underlying_address AS contract_address, + atoken_address AS address, amount_hex, amount_raw, protocol, From fe8b5dcd7c7391411d358b96a523286a13798f58 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 10 Dec 2025 15:04:22 -0700 Subject: [PATCH 088/164] removed _daily --- ...ete.sql => streamline__contract_reads_complete.sql} | 0 ...reamline__contract_reads_daily_history_requests.sql | 2 +- ...tory.sql => streamline__contract_reads_history.sql} | 10 +++++----- ...eamline__contract_reads_daily_realtime_requests.sql | 2 +- ...ime.sql => streamline__contract_reads_realtime.sql} | 10 +++++----- ...ords.sql => streamline__contract_reads_records.sql} | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) rename models/curated_package/contract_reads/streamline/complete/{streamline__contract_reads_daily_complete.sql => streamline__contract_reads_complete.sql} (100%) rename models/curated_package/contract_reads/streamline/history/{streamline__contract_reads_daily_history.sql => streamline__contract_reads_history.sql} (69%) rename models/curated_package/contract_reads/streamline/realtime/{streamline__contract_reads_daily_realtime.sql => streamline__contract_reads_realtime.sql} (69%) rename models/curated_package/contract_reads/streamline/records/{streamline__contract_reads_daily_records.sql => streamline__contract_reads_records.sql} (94%) diff --git a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql similarity index 100% rename from models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_daily_complete.sql rename to models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql diff --git a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql index 710c498e..4df7f989 100644 --- a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql +++ b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql @@ -50,7 +50,7 @@ to_do AS ( version, platform FROM - {{ ref("streamline__contract_reads_daily_complete") }} + {{ ref("streamline__contract_reads_complete") }} WHERE block_date IS NOT NULL ) diff --git a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history.sql b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_history.sql similarity index 69% rename from models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history.sql rename to models/curated_package/contract_reads/streamline/history/streamline__contract_reads_history.sql index ef708adc..ea9f92f4 100644 --- a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history.sql +++ b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_history.sql @@ -19,11 +19,11 @@ FROM {% if execute %} {% set params = { "external_table": 'contract_reads', - "sql_limit": vars.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_SQL_LIMIT, - "producer_batch_size": vars.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_PRODUCER_BATCH_SIZE, - "worker_batch_size": vars.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_WORKER_BATCH_SIZE, - "async_concurrent_requests": vars.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_ASYNC_CONCURRENT_REQUESTS, - "sql_source": 'contract_reads_daily_history' + "sql_limit": vars.CURATED_SL_CONTRACT_READS_HISTORY_SQL_LIMIT, + "producer_batch_size": vars.CURATED_SL_CONTRACT_READS_HISTORY_PRODUCER_BATCH_SIZE, + "worker_batch_size": vars.CURATED_SL_CONTRACT_READS_HISTORY_WORKER_BATCH_SIZE, + "async_concurrent_requests": vars.CURATED_SL_CONTRACT_READS_HISTORY_ASYNC_CONCURRENT_REQUESTS, + "sql_source": 'contract_reads_history' } %} {% set function_call_sql %} diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index 57f1ec73..1dc0e73f 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -50,7 +50,7 @@ to_do AS ( version, platform FROM - {{ ref("streamline__contract_reads_daily_complete") }} + {{ ref("streamline__contract_reads_complete") }} WHERE block_date >= ( SELECT MIN(block_date) diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_realtime.sql similarity index 69% rename from models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql rename to models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_realtime.sql index cc0e289a..61de6321 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_realtime.sql @@ -19,11 +19,11 @@ FROM {% if execute %} {% set params = { "external_table": 'contract_reads', - "sql_limit": vars.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_SQL_LIMIT, - "producer_batch_size": vars.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_PRODUCER_BATCH_SIZE, - "worker_batch_size": vars.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_WORKER_BATCH_SIZE, - "async_concurrent_requests": vars.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_ASYNC_CONCURRENT_REQUESTS, - "sql_source": 'contract_reads_daily_realtime' + "sql_limit": vars.CURATED_SL_CONTRACT_READS_REALTIME_SQL_LIMIT, + "producer_batch_size": vars.CURATED_SL_CONTRACT_READS_REALTIME_PRODUCER_BATCH_SIZE, + "worker_batch_size": vars.CURATED_SL_CONTRACT_READS_REALTIME_WORKER_BATCH_SIZE, + "async_concurrent_requests": vars.CURATED_SL_CONTRACT_READS_REALTIME_ASYNC_CONCURRENT_REQUESTS, + "sql_source": 'contract_reads_realtime' } %} {% set function_call_sql %} diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql similarity index 94% rename from models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql rename to models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index 90274c06..2f4f6c30 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_daily_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -7,7 +7,7 @@ -- depends_on: {{ ref('silver_reads__lido_reads') }} {{ config ( materialized = "incremental", - unique_key = "contract_reads_daily_records_id", + unique_key = "contract_reads_records_id", full_refresh = vars.GLOBAL_STREAMLINE_FR_ENABLED, tags = ['streamline','contract_reads','records','phase_4'] ) }} @@ -63,11 +63,11 @@ SELECT platform, {{ dbt_utils.generate_surrogate_key( ['contract_address','address','input','platform'] - ) }} AS contract_reads_daily_records_id, + ) }} AS contract_reads_records_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - all_records qualify (ROW_NUMBER() over (PARTITION BY contract_reads_daily_records_id + all_records qualify (ROW_NUMBER() over (PARTITION BY contract_reads_records_id ORDER BY modified_timestamp DESC)) = 1 From 4f8e40db8c4542559dfebb1c3876d8359f01a695 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 10 Dec 2025 15:05:08 -0700 Subject: [PATCH 089/164] update vars --- macros/global/variables/return_vars.sql | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/macros/global/variables/return_vars.sql b/macros/global/variables/return_vars.sql index f7c30a19..ceb9184f 100644 --- a/macros/global/variables/return_vars.sql +++ b/macros/global/variables/return_vars.sql @@ -284,15 +284,15 @@ {# SL Contract Reads Variables #} {% set ns.CURATED_SL_CONTRACT_READS_START_DATE = get_var('CURATED_SL_CONTRACT_READS_START_DATE', '2025-06-10') %} - {% set ns.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_SQL_LIMIT = get_var('CURATED_SL_CONTRACT_READS_DAILY_REALTIME_SQL_LIMIT', 12000000) %} - {% set ns.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_PRODUCER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_DAILY_REALTIME_PRODUCER_BATCH_SIZE', 240000) %} - {% set ns.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_WORKER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_DAILY_REALTIME_WORKER_BATCH_SIZE', 60000) %} - {% set ns.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_ASYNC_CONCURRENT_REQUESTS = get_var('CURATED_SL_CONTRACT_READS_DAILY_REALTIME_ASYNC_CONCURRENT_REQUESTS', 10) %} + {% set ns.CURATED_SL_CONTRACT_READS_REALTIME_SQL_LIMIT = get_var('CURATED_SL_CONTRACT_READS_REALTIME_SQL_LIMIT', 12000000) %} + {% set ns.CURATED_SL_CONTRACT_READS_REALTIME_PRODUCER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_REALTIME_PRODUCER_BATCH_SIZE', 240000) %} + {% set ns.CURATED_SL_CONTRACT_READS_REALTIME_WORKER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_REALTIME_WORKER_BATCH_SIZE', 60000) %} + {% set ns.CURATED_SL_CONTRACT_READS_REALTIME_ASYNC_CONCURRENT_REQUESTS = get_var('CURATED_SL_CONTRACT_READS_REALTIME_ASYNC_CONCURRENT_REQUESTS', 10) %} - {% set ns.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_SQL_LIMIT = get_var('CURATED_SL_CONTRACT_READS_DAILY_HISTORY_SQL_LIMIT', 12000000) %} - {% set ns.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_PRODUCER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_DAILY_HISTORY_PRODUCER_BATCH_SIZE', 600000) %} - {% set ns.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_WORKER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_DAILY_HISTORY_WORKER_BATCH_SIZE', 60000) %} - {% set ns.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_ASYNC_CONCURRENT_REQUESTS = get_var('CURATED_SL_CONTRACT_READS_DAILY_HISTORY_ASYNC_CONCURRENT_REQUESTS', 10) %} + {% set ns.CURATED_SL_CONTRACT_READS_HISTORY_SQL_LIMIT = get_var('CURATED_SL_CONTRACT_READS_HISTORY_SQL_LIMIT', 12000000) %} + {% set ns.CURATED_SL_CONTRACT_READS_HISTORY_PRODUCER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_HISTORY_PRODUCER_BATCH_SIZE', 600000) %} + {% set ns.CURATED_SL_CONTRACT_READS_HISTORY_WORKER_BATCH_SIZE = get_var('CURATED_SL_CONTRACT_READS_HISTORY_WORKER_BATCH_SIZE', 60000) %} + {% set ns.CURATED_SL_CONTRACT_READS_HISTORY_ASYNC_CONCURRENT_REQUESTS = get_var('CURATED_SL_CONTRACT_READS_HISTORY_ASYNC_CONCURRENT_REQUESTS', 10) %} {# Curated Bridge Variables #} {% set ns.CURATED_DEFI_BRIDGE_CONTRACT_MAPPING = get_var('CURATED_DEFI_BRIDGE_CONTRACT_MAPPING', {}) %} From 3331a067e1c05b8397d3f8e7224da9f942ebcf03 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 10 Dec 2025 15:55:43 -0700 Subject: [PATCH 090/164] ref --- .../streamline__contract_reads_daily_history_requests.sql | 2 +- .../streamline__contract_reads_daily_realtime_requests.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql index 4df7f989..b5b3477c 100644 --- a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql +++ b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql @@ -34,7 +34,7 @@ to_do AS ( t.version, t.platform FROM - {{ ref("streamline__contract_reads_daily_records") }} t + {{ ref("streamline__contract_reads_records") }} t CROSS JOIN last_x_days d EXCEPT SELECT diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index 1dc0e73f..f072f7bc 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -34,7 +34,7 @@ to_do AS ( t.version, t.platform FROM - {{ ref("streamline__contract_reads_daily_records") }} t + {{ ref("streamline__contract_reads_records") }} t CROSS JOIN last_x_days d EXCEPT SELECT From b34ab4f76332c8513f72abeb383bdf39ccaeb6ba Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 10 Dec 2025 16:06:48 -0700 Subject: [PATCH 091/164] vars --- .../streamline__contract_reads_daily_history_requests.sql | 2 +- .../streamline__contract_reads_daily_realtime_requests.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql index b5b3477c..b61005d3 100644 --- a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql +++ b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql @@ -103,4 +103,4 @@ FROM to_do ORDER BY partition_key DESC, block_number DESC -LIMIT {{ vars.CURATED_SL_CONTRACT_READS_DAILY_HISTORY_SQL_LIMIT }} \ No newline at end of file +LIMIT {{ vars.CURATED_SL_CONTRACT_READS_HISTORY_SQL_LIMIT }} \ No newline at end of file diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index f072f7bc..1d75023c 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -112,4 +112,4 @@ FROM to_do ORDER BY partition_key DESC, block_number DESC -LIMIT {{ vars.CURATED_SL_CONTRACT_READS_DAILY_REALTIME_SQL_LIMIT }} \ No newline at end of file +LIMIT {{ vars.CURATED_SL_CONTRACT_READS_REALTIME_SQL_LIMIT }} \ No newline at end of file From 47aa5ad3879989b45d514e994a22a8a9eb8c08ab Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 11 Dec 2025 09:40:23 -0700 Subject: [PATCH 092/164] aave v2 reads --- .../aave/silver_reads__aave_v2_reads.sql | 26 ++++++++----------- .../silver/aave/silver_tvl__aave_v2_tvl.sql | 7 +++-- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql index 4206fb9c..63c1966a 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql @@ -14,8 +14,8 @@ WITH all_tokens AS ( SELECT - atoken_address AS contract_address, - underlying_address, + underlying_address AS contract_address, + atoken_address AS address, protocol, version, CONCAT( @@ -40,8 +40,8 @@ AND modified_timestamp > ( {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} UNION SELECT - atoken_address AS contract_address, - underlying_address, + underlying_address AS contract_address, + atoken_address AS address, protocol, version, CONCAT( @@ -67,18 +67,14 @@ AND modified_timestamp > ( ) SELECT contract_address, - NULL AS address, - 'totalSupply' AS function_name, - '0x18160ddd' AS function_sig, - RPAD( - function_sig, - 64, - '0' + address, + 'balanceOf' AS function_name, + '0x70a08231' AS function_sig, + CONCAT( + '0x70a08231', + LPAD(SUBSTR(address, 3), 64, '0') ) AS input, - OBJECT_CONSTRUCT( - 'underlying_address', - underlying_address - ) :: variant AS metadata, + NULL :: variant AS metadata, protocol, version, platform, diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql index 39f81716..395afdd7 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql @@ -14,7 +14,7 @@ WITH reads AS ( SELECT block_number, block_date, - contract_address AS atoken_address, + contract_address, address, result_hex AS amount_hex, IFNULL( @@ -25,7 +25,6 @@ WITH reads AS ( WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) END ) AS amount_raw, - metadata :underlying_address :: STRING AS underlying_address, protocol, version, platform, @@ -53,8 +52,8 @@ AND modified_timestamp > ( SELECT block_number, block_date, - underlying_address AS contract_address, - atoken_address AS address, + contract_address, + address, amount_hex, amount_raw, protocol, From 951e43b75749e58ee246ae3d468979ff447f8856 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 08:31:32 -0700 Subject: [PATCH 093/164] v2 and v3 balance based --- .../aave/silver_reads__aave_v2_reads.sql | 13 +++++++++- .../aave/silver_reads__aave_v3_reads.sql | 24 ++++++++----------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql index 63c1966a..61e6ec9e 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql @@ -63,6 +63,17 @@ AND modified_timestamp > ( {{ this }} ) {% endif %} +UNION +SELECT + '0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9' AS contract_address, --AAVE + '0x4da27a545c0c5b758a6ba100e3a049001de870f5' AS address, --stkAAVE (Staking) + 'aave' AS protocol, + 'v2' AS version, + CONCAT( + protocol, + '-', + version + ) AS platform {% endif %} ) SELECT @@ -79,7 +90,7 @@ SELECT version, platform, {{ dbt_utils.generate_surrogate_key( - ['contract_address','input','platform'] + ['contract_address','address','input','platform'] ) }} AS aave_v2_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql index 79d6cd5f..e4a62ccd 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql @@ -12,8 +12,8 @@ WITH all_tokens AS ( SELECT - atoken_address AS contract_address, - underlying_address, + underlying_address AS contract_address, + atoken_address AS address, protocol, version, CONCAT( @@ -37,23 +37,19 @@ AND modified_timestamp > ( ) SELECT contract_address, - NULL AS address, - 'totalSupply' AS function_name, - '0x18160ddd' AS function_sig, - RPAD( - function_sig, - 64, - '0' + address, + 'balanceOf' AS function_name, + '0x70a08231' AS function_sig, + CONCAT( + '0x70a08231', + LPAD(SUBSTR(address, 3), 64, '0') ) AS input, - OBJECT_CONSTRUCT( - 'underlying_address', - underlying_address - ) :: variant AS metadata, + NULL :: variant AS metadata, protocol, version, platform, {{ dbt_utils.generate_surrogate_key( - ['contract_address','input','platform'] + ['contract_address','address','input','platform'] ) }} AS aave_v3_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, From 4ef1efa60b44664075968735ca32c534be5f4d68 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 08:47:53 -0700 Subject: [PATCH 094/164] aave v1 balance calls --- .../silver_reads__tornado_cash_reads.sql | 17 +++---- .../silver/aave/silver_tvl__aave_v1_tvl.sql | 44 ++++++++++++++++++- .../silver_tvl__tornado_cash_v1_tvl.sql | 20 +-------- 3 files changed, 52 insertions(+), 29 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql b/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql index 46499b22..4a9c003b 100644 --- a/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql @@ -15,13 +15,7 @@ WITH mixers AS ( SELECT token_address AS contract_address, - mixer_address AS address, - 'balanceOf' AS function_name, - '0x70a08231' AS function_sig, - CONCAT( - '0x70a08231', - LPAD(SUBSTR(address, 3), 64, '0') - ) AS input, + mixer_address AS address FROM {{ ref('silver_reads__tornado_cash_mixer_seed') }} WHERE @@ -40,9 +34,12 @@ AND CONCAT(COALESCE(contract_address, 'null'), '-', address) NOT IN ( SELECT contract_address, address, - function_name, - function_sig, - input, + 'balanceOf' AS function_name, + '0x70a08231' AS function_sig, + CONCAT( + '0x70a08231', + LPAD(SUBSTR(address, 3), 64, '0') + ) AS input, NULL :: variant AS metadata, 'tornado_cash' AS protocol, 'v1' AS version, diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql index ed0eaff7..41ad7bcf 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql @@ -48,6 +48,48 @@ AND modified_timestamp > ( {{ this }} ) {% endif %} +), +balances AS ( + SELECT + block_number, + block_date, + '0x0000000000000000000000000000000000000000' AS contract_address, -- Represents native asset, for pricing purposes + address, + balance_hex AS amount_hex, + balance_raw AS amount_raw, + 'aave' AS protocol, + 'v1' AS version, + CONCAT( + protocol, + '-', + version + ) AS platform, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('balances__ez_balances_native_daily') }} + WHERE + address = '0x3dfd23a6c5e8bbcfc9581d2e864a68feb6a076d3' + AND balance_raw IS NOT NULL + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +FINAL AS ( + SELECT + * + FROM + reads + UNION ALL + SELECT + * + FROM + balances ) SELECT block_number, @@ -66,4 +108,4 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads + FINAL diff --git a/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql b/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql index de0f309f..65e2c264 100644 --- a/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql @@ -85,28 +85,12 @@ AND b.modified_timestamp > ( ), FINAL AS ( SELECT - block_number, - block_date, - contract_address, - address, - amount_hex, - amount_raw, - protocol, - version, - platform + * FROM reads UNION ALL SELECT - block_number, - block_date, - contract_address, - address, - amount_hex, - amount_raw, - protocol, - version, - platform + * FROM balances ) From 98e2e194c9f511c1ae95b306de7d131f769ee2f4 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 09:18:55 -0700 Subject: [PATCH 095/164] verified filter --- models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql | 2 +- .../defi/tvl/silver/silver_tvl__complete_tvl.sql | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql index 24a089a6..4262ca89 100644 --- a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql @@ -25,7 +25,7 @@ WITH complete_tvl AS ( amount_precise, amount, CASE - WHEN t.amount_usd < 1e12 THEN t.amount_usd + WHEN t.amount_usd < 1e9 OR t.is_verified THEN t.amount_usd ELSE NULL END AS amount_usd, protocol, diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index c4b19641..b75a8d82 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -200,6 +200,7 @@ SELECT address, decimals, symbol, + is_verified, amount_hex, amount_raw, amount_precise, From e106f3bc2858d3807bd16dee0c0d2496ca28a109 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 09:28:54 -0700 Subject: [PATCH 096/164] filter --- models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql index 4262ca89..c78c0560 100644 --- a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql @@ -25,7 +25,7 @@ WITH complete_tvl AS ( amount_precise, amount, CASE - WHEN t.amount_usd < 1e9 OR t.is_verified THEN t.amount_usd + WHEN t.amount_usd < 1e9 THEN t.amount_usd ELSE NULL END AS amount_usd, protocol, From 8a10ca97030c0935607f5c2f1df3303ef9d2b94c Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 09:52:01 -0700 Subject: [PATCH 097/164] usd exponent --- ...x__complete_dex_liquidity_pool_actions.sql | 42 +++++++++---------- .../defi/tvl/gold/defi__ez_protocol_tvl.sql | 2 +- .../tvl/silver/silver_tvl__complete_tvl.sql | 40 ++++++++++-------- 3 files changed, 45 insertions(+), 39 deletions(-) diff --git a/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pool_actions.sql b/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pool_actions.sql index 820e0bbc..fa7ae8ad 100644 --- a/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pool_actions.sql +++ b/models/curated_package/defi/dex/silver/silver_dex__complete_dex_liquidity_pool_actions.sql @@ -90,7 +90,7 @@ poolcreated_evt_v3 AS ( FROM {{ ref('silver_dex__poolcreated_evt_v3_pool_actions') }} -{% if is_incremental() and 'poolcreated_evt_v3' not in vars.curated_fr_models %} +{% if is_incremental() and 'poolcreated_evt_v3' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -136,7 +136,7 @@ paircreated_evt_v2 AS ( FROM {{ ref('silver_dex__paircreated_evt_v2_pool_actions') }} -{% if is_incremental() and 'paircreated_evt_v2' not in vars.curated_fr_models %} +{% if is_incremental() and 'paircreated_evt_v2' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -182,7 +182,7 @@ aerodrome AS ( FROM {{ ref('silver_dex__aerodrome_pool_actions') }} -{% if is_incremental() and 'aerodrome' not in vars.curated_fr_models %} +{% if is_incremental() and 'aerodrome' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -228,7 +228,7 @@ pharaoh_v1 AS ( FROM {{ ref('silver_dex__pharaoh_v1_pool_actions') }} -{% if is_incremental() and 'pharaoh_v1' not in vars.curated_fr_models %} +{% if is_incremental() and 'pharaoh_v1' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -274,7 +274,7 @@ kyberswap_v1_dynamic AS ( FROM {{ ref('silver_dex__kyberswap_v1_dynamic_pool_actions') }} -{% if is_incremental() and 'kyberswap_v1_dynamic' not in vars.curated_fr_models %} +{% if is_incremental() and 'kyberswap_v1_dynamic' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -320,7 +320,7 @@ kyberswap_v2_elastic AS ( FROM {{ ref('silver_dex__kyberswap_v2_elastic_pool_actions') }} -{% if is_incremental() and 'kyberswap_v2_elastic' not in vars.curated_fr_models %} +{% if is_incremental() and 'kyberswap_v2_elastic' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -366,7 +366,7 @@ dodo_v1 AS ( FROM {{ ref('silver_dex__dodo_v1_pool_actions') }} -{% if is_incremental() and 'dodo_v1' not in vars.curated_fr_models %} +{% if is_incremental() and 'dodo_v1' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -412,7 +412,7 @@ pancakeswap_v3 AS ( FROM {{ ref('silver_dex__pancakeswap_v3_pool_actions') }} -{% if is_incremental() and 'pancakeswap_v3' not in vars.curated_fr_models %} +{% if is_incremental() and 'pancakeswap_v3' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -458,7 +458,7 @@ camelot_v2 AS ( FROM {{ ref('silver_dex__camelot_v2_pool_actions') }} -{% if is_incremental() and 'camelot_v2' not in vars.curated_fr_models %} +{% if is_incremental() and 'camelot_v2' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -504,7 +504,7 @@ quickswap_v2 AS ( FROM {{ ref('silver_dex__quickswap_v2_pool_actions') }} -{% if is_incremental() and 'quickswap_v2' not in vars.curated_fr_models %} +{% if is_incremental() and 'quickswap_v2' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -551,7 +551,7 @@ quickswap_v4 AS ( FROM {{ ref('silver_dex__quickswap_v4_pool_actions') }} -{% if is_incremental() and 'quickswap_v4' not in vars.curated_fr_models %} +{% if is_incremental() and 'quickswap_v4' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -598,7 +598,7 @@ dackie AS ( FROM {{ ref('silver_dex__dackie_pool_actions') }} -{% if is_incremental() and 'dackie' not in vars.curated_fr_models %} +{% if is_incremental() and 'dackie' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -644,7 +644,7 @@ superchain_slipstream AS ( FROM {{ ref('silver_dex__superchain_slipstream_pool_actions') }} -{% if is_incremental() and 'superchain_slipstream' not in vars.curated_fr_models %} +{% if is_incremental() and 'superchain_slipstream' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -690,7 +690,7 @@ sushiswap AS ( FROM {{ ref('silver_dex__sushiswap_pool_actions') }} -{% if is_incremental() and 'sushiswap' not in vars.curated_fr_models %} +{% if is_incremental() and 'sushiswap' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -736,7 +736,7 @@ trader_joe_v2 AS ( FROM {{ ref('silver_dex__trader_joe_v2_pool_actions') }} -{% if is_incremental() and 'trader_joe_v2' not in vars.curated_fr_models %} +{% if is_incremental() and 'trader_joe_v2' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -782,7 +782,7 @@ zyberswap_v2 AS ( FROM {{ ref('silver_dex__zyberswap_v2_pool_actions') }} -{% if is_incremental() and 'zyberswap_v2' not in vars.curated_fr_models %} +{% if is_incremental() and 'zyberswap_v2' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -828,7 +828,7 @@ uniswap_v1 AS ( FROM {{ ref('silver_dex__uniswap_v1_pool_actions') }} -{% if is_incremental() and 'uniswap_v1' not in vars.curated_fr_models %} +{% if is_incremental() and 'uniswap_v1' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -874,7 +874,7 @@ velodrome_v1 AS ( FROM {{ ref('silver_dex__velodrome_v1_pool_actions') }} -{% if is_incremental() and 'velodrome_v1' not in vars.curated_fr_models %} +{% if is_incremental() and 'velodrome_v1' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -920,7 +920,7 @@ velodrome_v2 AS ( FROM {{ ref('silver_dex__velodrome_v2_pool_actions') }} -{% if is_incremental() and 'velodrome_v2' not in vars.curated_fr_models %} +{% if is_incremental() and 'velodrome_v2' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -966,7 +966,7 @@ balancer AS ( FROM {{ ref('silver_dex__balancer_pool_actions') }} -{% if is_incremental() and 'balancer' not in vars.curated_fr_models %} +{% if is_incremental() and 'balancer' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT @@ -1012,7 +1012,7 @@ curve AS ( FROM {{ ref('silver_dex__curve_pool_actions') }} -{% if is_incremental() and 'curve' not in vars.curated_fr_models %} +{% if is_incremental() and 'curve' not in vars.CURATED_FR_MODELS %} WHERE _inserted_timestamp >= ( SELECT diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql index c78c0560..59fb4b62 100644 --- a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql @@ -25,7 +25,7 @@ WITH complete_tvl AS ( amount_precise, amount, CASE - WHEN t.amount_usd < 1e9 THEN t.amount_usd + WHEN t.amount_usd < POWER(10, t.usd_threshold) THEN t.amount_usd ELSE NULL END AS amount_usd, protocol, diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index b75a8d82..68a4c48d 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -9,20 +9,21 @@ tags = ['silver','defi','tvl','complete','heal','curated_daily'] ) }} -{% set models = [] %} -{% set _ = models.append(ref('silver_tvl__aave_v1_tvl')) %} -{% set _ = models.append(ref('silver_tvl__aave_v2_tvl')) %} -{% set _ = models.append(ref('silver_tvl__aave_v3_tvl')) %} -{% set _ = models.append(ref('silver_tvl__curve_v1_tvl')) %} -{% set _ = models.append(ref('silver_tvl__lido_v1_tvl')) %} -{% set _ = models.append(ref('silver_tvl__tornado_cash_v1_tvl')) %} -{% set _ = models.append(ref('silver_tvl__uniswap_v1_tvl')) %} -{% set _ = models.append(ref('silver_tvl__uniswap_v2_tvl')) %} -{% set _ = models.append(ref('silver_tvl__uniswap_v3_tvl')) %} -{% set _ = models.append(ref('silver_tvl__uniswap_v4_tvl')) %} +{% set models = [ + (ref('silver_tvl__aave_v1_tvl'), 12, 'aave-v1'), + (ref('silver_tvl__aave_v2_tvl'), 12, 'aave-v2'), + (ref('silver_tvl__aave_v3_tvl'), 12, 'aave-v3'), + (ref('silver_tvl__curve_v1_tvl'), 10, 'curve-v1'), + (ref('silver_tvl__lido_v1_tvl'), 12, 'lido-v1'), + (ref('silver_tvl__tornado_cash_v1_tvl'), 9, 'tornado_cash-v1'), + (ref('silver_tvl__uniswap_v1_tvl'), 9, 'uniswap-v1'), + (ref('silver_tvl__uniswap_v2_tvl'), 9, 'uniswap-v2'), + (ref('silver_tvl__uniswap_v3_tvl'), 9, 'uniswap-v3'), + (ref('silver_tvl__uniswap_v4_tvl'), 9, 'uniswap-v4') +] %} WITH all_tvl AS ( - {% for model in models %} + {% for model, max_usd_exponent, platform in models %} SELECT block_number, block_date, @@ -32,10 +33,11 @@ WITH all_tvl AS ( amount_raw, protocol, version, - platform + platform, + {{ max_usd_exponent }} AS max_usd_exponent FROM {{ model }} {% if not loop.last %} - {% if is_incremental() %} + {% if is_incremental() and platform not in vars.CURATED_FR_MODELS %} WHERE modified_timestamp > ( SELECT MAX(modified_timestamp) FROM {{ this }} @@ -107,7 +109,8 @@ complete_tvl AS ( ) AS amount_usd, A.protocol, A.version, - A.platform + A.platform, + A.max_usd_exponent FROM all_tvl A LEFT JOIN contracts c1 @@ -146,7 +149,8 @@ heal_model AS ( ) AS amount_usd_heal, t.protocol, t.version, - t.platform + t.platform, + t.max_usd_exponent FROM {{ this }} t @@ -188,7 +192,8 @@ SELECT amount_usd_heal AS amount_usd, protocol, version, - platform + platform, + max_usd_exponent FROM heal_model {% endif %} @@ -209,6 +214,7 @@ SELECT protocol, version, platform, + max_usd_exponent, {{ dbt_utils.generate_surrogate_key( ['block_date','contract_address','address','platform'] ) }} AS complete_tvl_id, From 9c314ddd113a561870d084b62a9af7e5136a8a08 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 10:00:33 -0700 Subject: [PATCH 098/164] column name --- .../curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql index 59fb4b62..ccf4d182 100644 --- a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql @@ -20,12 +20,14 @@ WITH complete_tvl AS ( address, decimals, symbol, + is_verified, + max_usd_exponent, amount_hex, amount_raw, amount_precise, amount, CASE - WHEN t.amount_usd < POWER(10, t.usd_threshold) THEN t.amount_usd + WHEN t.amount_usd < POWER(10, t.max_usd_exponent) THEN t.amount_usd ELSE NULL END AS amount_usd, protocol, From f0f0958c0f1e1eb19f6b768d961b187903ed2de5 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 10:54:58 -0700 Subject: [PATCH 099/164] polymarket tvl --- .../silver_reads__polymarket_v1_reads.sql | 42 +++++++++++++++++++ .../streamline__contract_reads_records.sql | 3 ++ 2 files changed, 45 insertions(+) create mode 100644 models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql diff --git a/models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql new file mode 100644 index 00000000..edffe599 --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql @@ -0,0 +1,42 @@ +{# Get variables #} +{% set vars = return_vars() %} +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'polymarket_v1_reads_id', + tags = ['silver','contract_reads'] +) }} + +WITH contracts AS ( + SELECT + '0x2791bca1f2de4661ed88a30c99a7a9449aa84174' AS contract_address, --USDC + '0x4d97dcd97ec945f40cf65f87097ace5ea0476045' AS address --Conditional Tokens + UNION ALL + SELECT + '0x2791bca1f2de4661ed88a30c99a7a9449aa84174' AS contract_address, --USDC + '0x3a3bd7bb9528e159577f7c2e685cc81a765002e2' AS address --Collateral Tokens +) +SELECT + contract_address, + address, + 'balanceOf' AS function_name, + '0x70a08231' AS function_sig, + CONCAT( + '0x70a08231', + LPAD(SUBSTR(address, 3), 64, '0') + ) AS input, + NULL :: VARIANT AS metadata, + 'polymarket' AS protocol, + 'v1' AS version, + CONCAT(protocol, '-', version) AS platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','address','input','platform'] + ) }} AS polymarket_v1_reads_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + contracts diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index 2f4f6c30..93f05caa 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -16,6 +16,9 @@ {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} {% set _ = models.append(ref('silver_reads__lido_reads')) %} {% endif %} +{% if vars.GLOBAL_PROJECT_NAME == 'polygon' %} + {% set _ = models.append(ref('silver_reads__polymarket_v1_reads')) %} +{% endif %} {% set _ = models.append(ref('silver_reads__stablecoins_reads')) %} {% set _ = models.append(ref('silver_reads__uniswap_v2_reads')) %} {% set _ = models.append(ref('silver_reads__uniswap_v3_reads')) %} From eaedade66ba298cb6a21a5d7f7f75211559df7f4 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 11:21:56 -0700 Subject: [PATCH 100/164] descriptions --- .../silver/aave/silver_tvl__aave_v1_tvl.yml | 4 + .../silver/aave/silver_tvl__aave_v2_tvl.yml | 4 + .../silver/aave/silver_tvl__aave_v3_tvl.yml | 4 + .../silver/curve/silver_tvl__curve_v1_tvl.yml | 4 + .../silver/lido/silver_tvl__lido_v1_tvl.yml | 4 + .../silver_tvl__polymarket_v1_tvl.sql | 66 ++++++++++++++++ .../silver_tvl__polymarket_v1_tvl.yml | 4 + .../silver_tvl__tornado_cash_v1_tvl.yml | 4 + .../uniswap/silver_tvl__uniswap_v1_tvl.sql | 2 +- .../uniswap/silver_tvl__uniswap_v1_tvl.yml | 4 + .../uniswap/silver_tvl__uniswap_v2_tvl.yml | 4 + .../uniswap/silver_tvl__uniswap_v3_tvl.yml | 4 + .../uniswap/silver_tvl__uniswap_v4_tvl.yml | 4 + models/doc_descriptions/defi/silver_tvl.md | 76 +++++++++++++++++++ 14 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql create mode 100644 models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.yml create mode 100644 models/doc_descriptions/defi/silver_tvl.md diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.yml b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.yml new file mode 100644 index 00000000..d1014967 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__aave_v1_tvl + description: '{{ doc("aave_v1_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.yml b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.yml new file mode 100644 index 00000000..ee7fb9a6 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__aave_v2_tvl + description: '{{ doc("aave_v2_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.yml b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.yml new file mode 100644 index 00000000..09b22521 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__aave_v3_tvl + description: '{{ doc("aave_v3_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.yml b/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.yml new file mode 100644 index 00000000..97c47dfc --- /dev/null +++ b/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__curve_v1_tvl + description: '{{ doc("curve_v1_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.yml b/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.yml new file mode 100644 index 00000000..52a3362d --- /dev/null +++ b/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__lido_v1_tvl + description: '{{ doc("lido_v1_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql b/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql new file mode 100644 index 00000000..cdc85053 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql @@ -0,0 +1,66 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'polymarket_v1_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH reads AS ( + + SELECT + block_number, + block_date, + contract_address, + address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + protocol, + version, + platform, + _inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} + WHERE + platform = 'polymarket-v1' + AND amount_raw IS NOT NULL + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS polymarket_v1_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads diff --git a/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.yml b/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.yml new file mode 100644 index 00000000..71d53ed3 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__polymarket_v1_tvl + description: '{{ doc("polymarket_v1_tvl_table_doc") }}' \ No newline at end of file diff --git a/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.yml b/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.yml new file mode 100644 index 00000000..7d7adf14 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__tornado_cash_v1_tvl + description: '{{ doc("tornado_cash_v1_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql index 70d85854..524e9029 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql @@ -49,7 +49,7 @@ SELECT contract_address, address, balance_hex AS amount_hex, - balance_raw AS amount_raw, + balance_raw * 2 AS amount_raw, -- 2x ETH value since all pools are 50/50 ETH-token pairs protocol, version, platform, diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.yml b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.yml new file mode 100644 index 00000000..71d29bc7 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__uniswap_v1_tvl + description: '{{ doc("uniswap_v1_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.yml b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.yml new file mode 100644 index 00000000..b4808914 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__uniswap_v2_tvl + description: '{{ doc("uniswap_v2_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.yml b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.yml new file mode 100644 index 00000000..11847a2b --- /dev/null +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__uniswap_v3_tvl + description: '{{ doc("uniswap_v3_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.yml b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.yml new file mode 100644 index 00000000..bb496dbe --- /dev/null +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__uniswap_v4_tvl + description: '{{ doc("uniswap_v4_tvl_table_doc") }}' diff --git a/models/doc_descriptions/defi/silver_tvl.md b/models/doc_descriptions/defi/silver_tvl.md new file mode 100644 index 00000000..095db801 --- /dev/null +++ b/models/doc_descriptions/defi/silver_tvl.md @@ -0,0 +1,76 @@ +{% docs aave_v1_tvl_table_doc %} + +Methodology: +Sums all underlying token balances held by the LendingPoolCore contract + native ETH in the protocol + +{% enddocs %} + +{% docs aave_v2_tvl_table_doc %} + +Methodology: +Sums all underlying token balances held by each aToken contract (+ stkAAVE staking on Ethereum) + +{% enddocs %} + +{% docs aave_v3_tvl_table_doc %} + +Methodology: +Sums all underlying token balances held by each aToken contract + +{% enddocs %} + +{% docs curve_v1_tvl_table_doc %} + +Methodology: +Sums all token balances held across pool contracts + +{% enddocs %} + +{% docs lido_v1_tvl_table_doc %} + +Methodology: +Total pooled assets reported by the Lido staking contracts + +{% enddocs %} + +{% docs tornado_cash_v1_tvl_table_doc %} + +Methodology: +Sums ERC20 token balances + native asset balances across all mixer contracts + +{% enddocs %} + +{% docs uniswap_v1_tvl_table_doc %} + +Methodology: +Sums native ETH balance across all pool contracts, doubled to represent total pool value (all pools are 50/50 ETH-token pairs) + +{% enddocs %} + +{% docs uniswap_v2_tvl_table_doc %} + +Methodology: +Sums getReserves() values for token0 and token1 across all pools + +{% enddocs %} + +{% docs uniswap_v3_tvl_table_doc %} + +Methodology: +Sums token0 and token1 balances held by each pool contract + +{% enddocs %} + +{% docs uniswap_v4_tvl_table_doc %} + +Methodology: +Sums token0 and token1 balances held by the PoolManager singleton + any associated hook contracts + +{% enddocs %} + +{% docs polymarket_v1_tvl_table_doc %} + +Methodology: +Sums USDC balances held by the Conditional Tokens and Collateral Tokens contracts + +{% enddocs %} From 23df3b5d18e14064243220062ed52bfde3600000 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 11:25:15 -0700 Subject: [PATCH 101/164] complete tests --- .../tvl/silver/silver_tvl__complete_tvl.yml | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.yml diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.yml b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.yml new file mode 100644 index 00000000..0c8fb5ea --- /dev/null +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.yml @@ -0,0 +1,43 @@ +version: 2 +models: + - name: silver_tvl__complete_tvl + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - COMPLETE_TVL_ID + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_DATE + tests: + - not_null + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: AMOUNT_RAW + tests: + - not_null + - name: PLATFORM + tests: + - not_null + - name: PROTOCOL + tests: + - not_null + - name: VERSION + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 From 4281b75ab13867d86df23eb009ed7a30e776d271 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 11:42:24 -0700 Subject: [PATCH 102/164] aave v3 --- .../defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql index f39e2fd5..056fb0b2 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql @@ -14,7 +14,7 @@ WITH reads AS ( SELECT block_number, block_date, - contract_address AS atoken_address, + contract_address, address, result_hex AS amount_hex, IFNULL( @@ -25,7 +25,6 @@ WITH reads AS ( WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) END ) AS amount_raw, - metadata :underlying_address :: STRING AS underlying_address, protocol, version, platform, @@ -53,15 +52,15 @@ AND modified_timestamp >= ( SELECT block_number, block_date, - underlying_address AS contract_address, - atoken_address AS address, + contract_address, + address, amount_hex, amount_raw, protocol, version, platform, {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address','platform'] + ['block_date','contract_address','address','platform'] ) }} AS aave_v3_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, From 77994d9df9ab8448369ac9f3363e24d904bbb74a Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 11:42:37 -0700 Subject: [PATCH 103/164] remove test --- .../defi/tvl/silver/silver_tvl__complete_tvl.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.yml b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.yml index 0c8fb5ea..f780333a 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.yml +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.yml @@ -18,11 +18,6 @@ models: - not_null - dbt_expectations.expect_column_values_to_match_regex: regex: 0[xX][0-9a-fA-F]+ - - name: ADDRESS - tests: - - not_null - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - name: AMOUNT_RAW tests: - not_null From f31632986281ed123696ff7a4a3dd4ba1d7dce4a Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 11:44:29 -0700 Subject: [PATCH 104/164] id --- .../defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql index 395afdd7..d10bfd1a 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql @@ -60,7 +60,7 @@ SELECT version, platform, {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address','platform'] + ['block_date','contract_address','address','platform'] ) }} AS aave_v2_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, From 1e5fec1cf709f85c22000178899845dede08336e Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 11:59:24 -0700 Subject: [PATCH 105/164] desc --- .../records/aave/silver_reads__aave_v3_reads.sql | 11 ++++++----- models/doc_descriptions/defi/silver_tvl.md | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql index e4a62ccd..72ed1602 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql @@ -38,11 +38,12 @@ AND modified_timestamp > ( SELECT contract_address, address, - 'balanceOf' AS function_name, - '0x70a08231' AS function_sig, - CONCAT( - '0x70a08231', - LPAD(SUBSTR(address, 3), 64, '0') + 'totalSupply' AS function_name, + '0x18160ddd' AS function_sig, + RPAD( + function_sig, + 64, + '0' ) AS input, NULL :: variant AS metadata, protocol, diff --git a/models/doc_descriptions/defi/silver_tvl.md b/models/doc_descriptions/defi/silver_tvl.md index 095db801..9b1b1a28 100644 --- a/models/doc_descriptions/defi/silver_tvl.md +++ b/models/doc_descriptions/defi/silver_tvl.md @@ -15,7 +15,7 @@ Sums all underlying token balances held by each aToken contract (+ stkAAVE staki {% docs aave_v3_tvl_table_doc %} Methodology: -Sums all underlying token balances held by each aToken contract +Sums totalSupply of each aToken contract {% enddocs %} From 48375c3db24e9654844350d58e579e0267ad97c9 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 12:16:06 -0700 Subject: [PATCH 106/164] revert aave v3 --- .../records/aave/silver_reads__aave_v3_reads.sql | 13 ++++++++----- .../tvl/silver/aave/silver_tvl__aave_v3_tvl.sql | 10 ++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql index 72ed1602..79d6cd5f 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql @@ -12,8 +12,8 @@ WITH all_tokens AS ( SELECT - underlying_address AS contract_address, - atoken_address AS address, + atoken_address AS contract_address, + underlying_address, protocol, version, CONCAT( @@ -37,7 +37,7 @@ AND modified_timestamp > ( ) SELECT contract_address, - address, + NULL AS address, 'totalSupply' AS function_name, '0x18160ddd' AS function_sig, RPAD( @@ -45,12 +45,15 @@ SELECT 64, '0' ) AS input, - NULL :: variant AS metadata, + OBJECT_CONSTRUCT( + 'underlying_address', + underlying_address + ) :: variant AS metadata, protocol, version, platform, {{ dbt_utils.generate_surrogate_key( - ['contract_address','address','input','platform'] + ['contract_address','input','platform'] ) }} AS aave_v3_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql index 056fb0b2..2978e88c 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql @@ -14,7 +14,7 @@ WITH reads AS ( SELECT block_number, block_date, - contract_address, + contract_address AS atoken_address, address, result_hex AS amount_hex, IFNULL( @@ -25,6 +25,7 @@ WITH reads AS ( WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) END ) AS amount_raw, + metadata :underlying_address :: STRING AS underlying_address, protocol, version, platform, @@ -52,15 +53,16 @@ AND modified_timestamp >= ( SELECT block_number, block_date, - contract_address, - address, + underlying_address AS contract_address, + atoken_address AS address, amount_hex, amount_raw, protocol, version, platform, + 12 AS usd_threshold, {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address','address','platform'] + ['block_date','contract_address','platform'] ) }} AS aave_v3_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, From eed8ffe5a7c95905060f342d54c5c8dd00cc3f92 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 12:55:38 -0700 Subject: [PATCH 107/164] revert aave v3 --- .../defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql | 2 +- .../tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql index 2978e88c..86e8819a 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql @@ -42,7 +42,7 @@ WITH reads AS ( ) {% if is_incremental() %} -AND modified_timestamp >= ( +AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM diff --git a/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql b/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql index 65e2c264..bd6d0e4b 100644 --- a/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql @@ -38,7 +38,7 @@ WITH reads AS ( AND amount_raw IS NOT NULL {% if is_incremental() %} -AND modified_timestamp >= ( +AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM From 01b564aeea6eae4bf6a491497833600a845cabd0 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 12 Dec 2025 13:16:35 -0700 Subject: [PATCH 108/164] qualify --- .../defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql | 4 +++- .../defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql | 4 +++- .../defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql | 6 ++++-- .../defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.sql | 4 +++- .../defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.sql | 4 +++- .../silver/polymarket/silver_tvl__polymarket_v1_tvl.sql | 4 +++- .../tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql | 8 +++++--- .../tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql | 4 +++- .../tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql | 4 +++- .../tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql | 4 +++- .../tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql | 4 +++- 11 files changed, 36 insertions(+), 14 deletions(-) diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql index 41ad7bcf..0bad5902 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql @@ -108,4 +108,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - FINAL + FINAL qualify(ROW_NUMBER() over(PARTITION BY aave_v1_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql index d10bfd1a..db8482e4 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql @@ -66,4 +66,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads + reads qualify(ROW_NUMBER() over(PARTITION BY aave_v2_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql index 86e8819a..30c48d9b 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql @@ -62,10 +62,12 @@ SELECT platform, 12 AS usd_threshold, {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address','platform'] + ['block_date','contract_address','address','platform'] ) }} AS aave_v3_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads + reads qualify(ROW_NUMBER() over(PARTITION BY aave_v3_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.sql b/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.sql index 08177289..90ed20e3 100644 --- a/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.sql @@ -63,4 +63,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads + reads qualify(ROW_NUMBER() over(PARTITION BY curve_v1_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.sql b/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.sql index 5d4bd748..18941913 100644 --- a/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.sql @@ -63,4 +63,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads + reads qualify(ROW_NUMBER() over(PARTITION BY lido_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql b/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql index cdc85053..8ee38ebd 100644 --- a/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql @@ -63,4 +63,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads + reads qualify(ROW_NUMBER() over(PARTITION BY polymarket_v1_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql b/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql index bd6d0e4b..7f68fb60 100644 --- a/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql @@ -7,7 +7,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - unique_key = 'tornado_cash_tvl_id', + unique_key = 'tornado_cash_v1_tvl_id', tags = ['silver','defi','tvl','curated_daily'] ) }} @@ -106,9 +106,11 @@ SELECT platform, {{ dbt_utils.generate_surrogate_key( ['block_date','contract_address','address','platform'] - ) }} AS tornado_cash_tvl_id, + ) }} AS tornado_cash_v1_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - FINAL + FINAL qualify(ROW_NUMBER() over(PARTITION BY tornado_cash_v1_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql index 524e9029..d33f61c4 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql @@ -60,4 +60,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - balances + balances qualify(ROW_NUMBER() over(PARTITION BY uniswap_v1_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql index 4e2aa2c0..b039bce3 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql @@ -116,4 +116,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - all_reads + all_reads qualify(ROW_NUMBER() over(PARTITION BY uniswap_v2_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql index 333f609f..fbd2a3d4 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql @@ -70,4 +70,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads + reads qualify(ROW_NUMBER() over(PARTITION BY uniswap_v3_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql index 33f88450..d8ba4fbb 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql @@ -71,4 +71,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads + reads qualify(ROW_NUMBER() over(PARTITION BY uniswap_v4_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 From fd302112046dd79b04e96e07bc9f46d367e852f9 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 15 Dec 2025 11:03:05 -0700 Subject: [PATCH 109/164] granularity column and filter --- ..._contract_reads_daily_history_requests.sql | 1 + ...contract_reads_daily_realtime_requests.sql | 1 + .../streamline__contract_reads_records.sql | 28 ++++++++++--------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql index b61005d3..b0c13a76 100644 --- a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql +++ b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql @@ -36,6 +36,7 @@ to_do AS ( FROM {{ ref("streamline__contract_reads_records") }} t CROSS JOIN last_x_days d + WHERE granularity = 'daily' EXCEPT SELECT block_number, diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index 1d75023c..6407d3b4 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -36,6 +36,7 @@ to_do AS ( FROM {{ ref("streamline__contract_reads_records") }} t CROSS JOIN last_x_days d + WHERE granularity = 'daily' EXCEPT SELECT block_number, diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index 93f05caa..69480fea 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -14,23 +14,23 @@ {% set models = [] %} {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} - {% set _ = models.append(ref('silver_reads__lido_reads')) %} + {% set _ = models.append((ref('silver_reads__lido_reads'), 'daily')) %} {% endif %} {% if vars.GLOBAL_PROJECT_NAME == 'polygon' %} - {% set _ = models.append(ref('silver_reads__polymarket_v1_reads')) %} + {% set _ = models.append((ref('silver_reads__polymarket_v1_reads'), 'daily')) %} {% endif %} -{% set _ = models.append(ref('silver_reads__stablecoins_reads')) %} -{% set _ = models.append(ref('silver_reads__uniswap_v2_reads')) %} -{% set _ = models.append(ref('silver_reads__uniswap_v3_reads')) %} -{% set _ = models.append(ref('silver_reads__uniswap_v4_reads')) %} -{% set _ = models.append(ref('silver_reads__aave_v1_reads')) %} -{% set _ = models.append(ref('silver_reads__aave_v2_reads')) %} -{% set _ = models.append(ref('silver_reads__aave_v3_reads')) %} -{% set _ = models.append(ref('silver_reads__curve_reads')) %} -{% set _ = models.append(ref('silver_reads__tornado_cash_reads')) %} +{% set _ = models.append((ref('silver_reads__stablecoins_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__uniswap_v2_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__uniswap_v3_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__uniswap_v4_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__aave_v1_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__aave_v2_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__aave_v3_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__curve_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__tornado_cash_reads'), 'daily')) %} WITH all_records AS ( - {% for model in models %} + {% for model, granularity in models %} SELECT contract_address, address, @@ -40,7 +40,8 @@ WITH all_records AS ( metadata, protocol, version, - platform + platform, + '{{ granularity }}' AS granularity FROM {{ model }} {% if not loop.last %} {% if is_incremental() %} @@ -64,6 +65,7 @@ SELECT protocol, version, platform, + granularity, {{ dbt_utils.generate_surrogate_key( ['contract_address','address','input','platform'] ) }} AS contract_reads_records_id, From 36cb5b8aadb4c202f5ebaa6aa26360e14a25bfb9 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 15 Dec 2025 11:28:58 -0700 Subject: [PATCH 110/164] type --- .../contract_reads/silver/silver__contract_reads.sql | 3 ++- .../complete/streamline__contract_reads_complete.sql | 1 + ...streamline__contract_reads_daily_history_requests.sql | 9 ++++++--- ...treamline__contract_reads_daily_realtime_requests.sql | 9 ++++++--- .../records/streamline__contract_reads_records.sql | 6 +++--- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/models/curated_package/contract_reads/silver/silver__contract_reads.sql b/models/curated_package/contract_reads/silver/silver__contract_reads.sql index 1c9d3a45..162615ff 100644 --- a/models/curated_package/contract_reads/silver/silver__contract_reads.sql +++ b/models/curated_package/contract_reads/silver/silver__contract_reads.sql @@ -23,6 +23,7 @@ SELECT VALUE :"PROTOCOL" :: STRING AS protocol, VALUE :"VERSION" :: STRING AS version, VALUE :"PLATFORM" :: STRING AS platform, + VALUE :"TYPE" :: STRING AS type, PARSE_JSON( VALUE :"METADATA_STR" :: STRING ) :: variant AS metadata, @@ -32,7 +33,7 @@ SELECT DATA :result :: STRING AS result_hex, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( - ['block_number','contract_address', 'address', 'input', 'platform'] + ['block_number','contract_address', 'address', 'input', 'platform', 'type'] ) }} AS contract_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql index bb4bd469..452c7fa3 100644 --- a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql +++ b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql @@ -32,6 +32,7 @@ SELECT VALUE :"PROTOCOL" :: STRING AS protocol, VALUE :"VERSION" :: STRING AS version, VALUE :"PLATFORM" :: STRING AS platform, + VALUE :"TYPE" :: STRING AS type, file_name, {{ dbt_utils.generate_surrogate_key( ['contract_address', 'address', 'block_number', 'function_name', 'function_sig', 'input', 'platform'] diff --git a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql index b0c13a76..96f43139 100644 --- a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql +++ b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql @@ -32,11 +32,12 @@ to_do AS ( t.metadata, t.protocol, t.version, - t.platform + t.platform, + t.type FROM {{ ref("streamline__contract_reads_records") }} t CROSS JOIN last_x_days d - WHERE granularity = 'daily' + WHERE type = 'daily' EXCEPT SELECT block_number, @@ -49,7 +50,8 @@ to_do AS ( metadata, protocol, version, - platform + platform, + type FROM {{ ref("streamline__contract_reads_complete") }} WHERE @@ -67,6 +69,7 @@ SELECT protocol, version, platform, + type, ROUND( block_number, -3 diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index 6407d3b4..b25f5d86 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -32,11 +32,12 @@ to_do AS ( t.metadata, t.protocol, t.version, - t.platform + t.platform, + t.type FROM {{ ref("streamline__contract_reads_records") }} t CROSS JOIN last_x_days d - WHERE granularity = 'daily' + WHERE type = 'daily' EXCEPT SELECT block_number, @@ -49,7 +50,8 @@ to_do AS ( metadata, protocol, version, - platform + platform, + type FROM {{ ref("streamline__contract_reads_complete") }} WHERE @@ -76,6 +78,7 @@ SELECT protocol, version, platform, + type, ROUND( block_number, -3 diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index 69480fea..ad991e31 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -30,7 +30,7 @@ {% set _ = models.append((ref('silver_reads__tornado_cash_reads'), 'daily')) %} WITH all_records AS ( - {% for model, granularity in models %} + {% for model, type in models %} SELECT contract_address, address, @@ -41,7 +41,7 @@ WITH all_records AS ( protocol, version, platform, - '{{ granularity }}' AS granularity + '{{ type }}' AS type FROM {{ model }} {% if not loop.last %} {% if is_incremental() %} @@ -65,7 +65,7 @@ SELECT protocol, version, platform, - granularity, + type, {{ dbt_utils.generate_surrogate_key( ['contract_address','address','input','platform'] ) }} AS contract_reads_records_id, From 93155f0f80a493d63d243460cd001b6fb6d055d7 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 15 Dec 2025 16:02:47 -0700 Subject: [PATCH 111/164] names and binance staked eth --- .../silver_reads__binance_v1_reads.sql | 38 +++++++++++++++++++ ...s.sql => silver_reads__curve_v1_reads.sql} | 0 ...ds.sql => silver_reads__lido_v1_reads.sql} | 0 .../streamline__contract_reads_records.sql | 7 ++-- ...> silver_reads__tornado_cash_v1_reads.sql} | 0 5 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql rename models/curated_package/contract_reads/streamline/records/curve/{silver_reads__curve_reads.sql => silver_reads__curve_v1_reads.sql} (100%) rename models/curated_package/contract_reads/streamline/records/lido/{silver_reads__lido_reads.sql => silver_reads__lido_v1_reads.sql} (100%) rename models/curated_package/contract_reads/streamline/records/tornado_cash/{silver_reads__tornado_cash_reads.sql => silver_reads__tornado_cash_v1_reads.sql} (100%) diff --git a/models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql new file mode 100644 index 00000000..ce002d70 --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql @@ -0,0 +1,38 @@ +{# Get variables #} +{% set vars = return_vars() %} +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'binance_v1_reads_id', + tags = ['silver','contract_reads'] +) }} + +WITH contracts AS ( + SELECT + '0xa2e3356610840701bdf5611a53974510ae27e2e1' AS contract_address --Wrapped Binance Beacon ETH (wBETH) +) +SELECT + contract_address, + NULL AS address, + 'totalSupply' AS function_name, + '0x18160ddd' AS function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input, + NULL :: VARIANT AS metadata, + 'binance' AS protocol, + 'v1' AS version, + CONCAT(protocol, '-', version) AS platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','input','platform'] + ) }} AS binance_v1_reads_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + contracts diff --git a/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql b/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_v1_reads.sql similarity index 100% rename from models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_reads.sql rename to models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_v1_reads.sql diff --git a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_v1_reads.sql similarity index 100% rename from models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_reads.sql rename to models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_v1_reads.sql diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index ad991e31..0d287d92 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -14,7 +14,8 @@ {% set models = [] %} {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} - {% set _ = models.append((ref('silver_reads__lido_reads'), 'daily')) %} + {% set _ = models.append((ref('silver_reads__lido_v1_reads'), 'daily')) %} + {% set _ = models.append((ref('silver_reads__binance_v1_reads'), 'daily')) %} {% endif %} {% if vars.GLOBAL_PROJECT_NAME == 'polygon' %} {% set _ = models.append((ref('silver_reads__polymarket_v1_reads'), 'daily')) %} @@ -26,8 +27,8 @@ {% set _ = models.append((ref('silver_reads__aave_v1_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__aave_v2_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__aave_v3_reads'), 'daily')) %} -{% set _ = models.append((ref('silver_reads__curve_reads'), 'daily')) %} -{% set _ = models.append((ref('silver_reads__tornado_cash_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__curve_v1_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__tornado_cash_v1_reads'), 'daily')) %} WITH all_records AS ( {% for model, type in models %} diff --git a/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql b/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_v1_reads.sql similarity index 100% rename from models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_reads.sql rename to models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_v1_reads.sql From f735952a0ab091741f06dcdb6797538a04626650 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 15 Dec 2025 17:01:17 -0700 Subject: [PATCH 112/164] etherfi --- .../defi/reads/silver_reads__etherfi_seed.csv | 19 ++ .../silver_reads__binance_v1_reads.sql | 8 + .../silver_reads__etherfi_v1_reads.sql | 71 ++++++++ .../lido/silver_reads__lido_v1_reads.sql | 10 +- .../silver_reads__polymarket_v1_reads.sql | 8 + .../binance/silver_tvl__binance_v1_tvl.sql | 68 +++++++ .../binance/silver_tvl__binance_v1_tvl.yml | 4 + .../etherfi/silver_tvl__etherfi_v1_tvl.sql | 77 ++++++++ .../etherfi/silver_tvl__etherfi_v1_tvl.yml | 4 + .../silver_tvl__etherfi_v1_tvl_agg.sql | 168 ++++++++++++++++++ .../silver_tvl__etherfi_v1_tvl_agg.yml | 4 + .../tvl/silver/silver_tvl__complete_tvl.sql | 2 + models/sources.yml | 16 +- 13 files changed, 457 insertions(+), 2 deletions(-) create mode 100644 data/curated/defi/reads/silver_reads__etherfi_seed.csv create mode 100644 models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql create mode 100644 models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.sql create mode 100644 models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql create mode 100644 models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.yml create mode 100644 models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql create mode 100644 models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.yml diff --git a/data/curated/defi/reads/silver_reads__etherfi_seed.csv b/data/curated/defi/reads/silver_reads__etherfi_seed.csv new file mode 100644 index 00000000..1414d76b --- /dev/null +++ b/data/curated/defi/reads/silver_reads__etherfi_seed.csv @@ -0,0 +1,19 @@ +contract_address,address,function_name,function_sig,chain,attribution +0xab7590cee3ef1a863e9a5877fbb82d9be11504da,null,categoryTVL,0x820a1175,optimism,ethereum +0x86b5780b606940eb59a062aa85a07959518c0161,null,totalSupply,0x18160ddd,arbitrum,arbitrum +0x86b5780b606940eb59a062aa85a07959518c0161,null,totalSupply,0x18160ddd,base,base +0x86b5780b606940eb59a062aa85a07959518c0161,null,totalSupply,0x18160ddd,ethereum,ethereum +0x939778d83b46b456224a33fb59630b11dec56663,null,totalSupply,0x18160ddd,ethereum,ethereum +0x6329004e903b7f420245e7af3f355186f2432466,null,getTvl,0xd075dd42,optimism,ethereum +0x8236a87084f8b84306f72007f36f2618a5634494,0xd4e20eca1f996dab35883dc0ad5e3428af888d45,balanceOf,0x70a08231,ethereum,ethereum +0x2f2a2543b76a4166549f7aab2e75bef0aefc5b0f,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,arbitrum,ethereum +0x8236a87084f8b84306f72007f36f2618a5634494,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum +0xecac9c5f704e954931349da37f60e39f515c11c1,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,base,ethereum +0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum +0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,base,ethereum +0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,arbitrum,ethereum +0x468c34703f6c648ccf39dbab11305d17c70ba011,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum +0x9c0823d3a1172f9ddf672d438dec79c39a64f448,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum +0x126d4dbf752aaf61f3eaada24ab0db84fecf6891,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum +0x971e5b5d4baa5607863f3748febf287c7bf82618,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum +0x2260fac5e5542a773aa44fbcfedf7c193bc2c599,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum \ No newline at end of file diff --git a/models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql index ce002d70..7e48c75e 100644 --- a/models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql @@ -36,3 +36,11 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM contracts +{% if is_incremental() %} +WHERE contract_address NOT IN ( + SELECT + contract_address + FROM + {{ this }} +) +{% endif %} \ No newline at end of file diff --git a/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql new file mode 100644 index 00000000..7bda20ff --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql @@ -0,0 +1,71 @@ +{# Get variables #} +{% set vars = return_vars() %} +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'etherfi_v1_reads_id', + tags = ['silver','contract_reads'] +) }} + +WITH contracts AS ( + + SELECT + contract_address, + address, + function_name, + function_sig, + chain, + attribution + FROM + {{ ref('silver_reads__etherfi_seed') }} + WHERE + chain = '{{ vars.GLOBAL_PROJECT_NAME }}' + +{% if is_incremental() %} +WHERE + CONCAT(contract_address, '-', COALESCE(address, 'null')) NOT IN ( + SELECT + CONCAT(contract_address, '-', COALESCE(address, 'null')) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + contract_address, + address, + function_name, + function_sig, + CASE + WHEN function_name = 'balanceOf' THEN CONCAT( + '0x70a08231', + LPAD(SUBSTR(address, 3), 64, '0') + ) + ELSE RPAD( + function_sig, + 64, + '0' + ) + END AS input, + OBJECT_CONSTRUCT( + 'attribution', attribution, + 'chain', chain + ) :: variant AS metadata, + 'etherfi' AS protocol, + 'v1' AS version, + CONCAT( + protocol, + '-', + version + ) AS platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','address','input','platform'] + ) }} AS etherfi_v1_reads_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + contracts diff --git a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_v1_reads.sql index 95504dca..b6a40d31 100644 --- a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_v1_reads.sql @@ -49,4 +49,12 @@ SELECT SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id -FROM pooled_assets \ No newline at end of file +FROM pooled_assets +{% if is_incremental() %} +WHERE contract_address NOT IN ( + SELECT + contract_address + FROM + {{ this }} +) +{% endif %} \ No newline at end of file diff --git a/models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql index edffe599..86f367ae 100644 --- a/models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql @@ -40,3 +40,11 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM contracts +{% if is_incremental() %} +WHERE CONCAT(contract_address, '-', address) NOT IN ( + SELECT + CONCAT(contract_address, '-', address) + FROM + {{ this }} +) +{% endif %} diff --git a/models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.sql b/models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.sql new file mode 100644 index 00000000..c98827b4 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.sql @@ -0,0 +1,68 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'binance_v1_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH reads AS ( + + SELECT + block_number, + block_date, + contract_address, + address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + protocol, + version, + platform, + _inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} + WHERE + platform = 'binance-v1' + AND amount_raw IS NOT NULL + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','platform'] + ) }} AS binance_v1_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads qualify(ROW_NUMBER() over(PARTITION BY binance_v1_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.yml b/models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.yml new file mode 100644 index 00000000..62d79f6e --- /dev/null +++ b/models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__binance_v1_tvl + description: '{{ doc("binance_v1_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql new file mode 100644 index 00000000..c5ea4823 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql @@ -0,0 +1,77 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'etherfi_v1_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH reads AS ( + + SELECT + block_number, + block_date, + contract_address, + address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw_unadj, + metadata :attribution :: STRING AS attribution, + metadata :chain :: STRING AS chain, + protocol, + version, + platform, + _inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} + WHERE + platform = 'etherfi-v1' + AND chain = '{{ vars.GLOBAL_PROJECT_NAME }}' + AND amount_raw IS NOT NULL + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + CASE + WHEN contract_address = '0xab7590cee3ef1a863e9a5877fbb82d9be11504da' THEN -1 * amount_raw_unadj + --categoryTVL() must be subtracted from getTVL(), negative value enables SUM + ELSE amount_raw_unadj + END AS amount_raw, + protocol, + version, + platform, + attribution, + chain, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS etherfi_v1_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads qualify(ROW_NUMBER() over(PARTITION BY etherfi_v1_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.yml b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.yml new file mode 100644 index 00000000..1bd3b73e --- /dev/null +++ b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__etherfi_v1_tvl + description: '{{ doc("etherfi_v1_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql new file mode 100644 index 00000000..07d925f7 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql @@ -0,0 +1,168 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'etherfi_v1_tvl_agg_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH chain_tvl AS ( + + SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + attribution, + chain + FROM + {{ ref('silver_tvl__etherfi_v1_tvl') }} + WHERE + chain = '{{ vars.GLOBAL_PROJECT_NAME }}' + AND attribution = '{{ vars.GLOBAL_PROJECT_NAME }}' + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %}, + eth_attributed_tvl AS ( + SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + attribution, + chain + FROM + {{ source( + 'silver_tvl_optimism', + 'silver_tvl__etherfi_v1_tvl' + ) }} + WHERE + attribution = 'ethereum' + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +UNION ALL +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + attribution, + chain +FROM + {{ source( + 'silver_tvl_arbitrum', + 'silver_tvl__etherfi_v1_tvl' + ) }} +WHERE + attribution = 'ethereum' + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +UNION ALL +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + attribution, + chain +FROM + {{ source( + 'silver_tvl_base', + 'silver_tvl__etherfi_v1_tvl' + ) }} +WHERE + attribution = 'ethereum' + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +{% endif %} + +FINAL AS ( + SELECT + * + FROM + chain_tvl + + {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} + UNION ALL + SELECT + * + FROM + eth_attributed_tvl + {% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + amount_hex, + amount_raw, + protocol, + version, + platform, + attribution, + chain, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform','attribution','chain'] + ) }} AS etherfi_v1_tvl_agg_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY etherfi_v1_tvl_agg_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.yml b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.yml new file mode 100644 index 00000000..62e46dc2 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__etherfi_v1_tvl_agg + description: '{{ doc("etherfi_v1_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index 68a4c48d..689972c0 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -13,7 +13,9 @@ (ref('silver_tvl__aave_v1_tvl'), 12, 'aave-v1'), (ref('silver_tvl__aave_v2_tvl'), 12, 'aave-v2'), (ref('silver_tvl__aave_v3_tvl'), 12, 'aave-v3'), + (ref('silver_tvl__binance_v1_tvl'), 12, 'binance-v1'), (ref('silver_tvl__curve_v1_tvl'), 10, 'curve-v1'), + (ref('silver_tvl__etherfi_v1_tvl_agg'), 12, 'etherfi-v1'), (ref('silver_tvl__lido_v1_tvl'), 12, 'lido-v1'), (ref('silver_tvl__tornado_cash_v1_tvl'), 9, 'tornado_cash-v1'), (ref('silver_tvl__uniswap_v1_tvl'), 9, 'uniswap-v1'), diff --git a/models/sources.yml b/models/sources.yml index bb085b89..de93b100 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -140,4 +140,18 @@ sources: tables: - name: _master_keys - name: rpc_node_logs - \ No newline at end of file + - name: silver_tvl_optimism + database: silver_tvl_optimism + schema: silver + tables: + - name: silver_tvl__etherfi_v1_tvl + - name: silver_tvl_arbitrum + database: silver_tvl_arbitrum + schema: silver + tables: + - name: silver_tvl__etherfi_v1_tvl + - name: silver_tvl_base + database: silver_tvl_base + schema: silver + tables: + - name: silver_tvl__etherfi_v1_tvl \ No newline at end of file From 8edcca3a84af0dd74be6be32a58f314506428e12 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 15 Dec 2025 17:03:11 -0700 Subject: [PATCH 113/164] sources --- .../etherfi/silver_tvl__etherfi_v1_tvl_agg.sql | 6 +++--- models/sources.yml | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql index 07d925f7..536a269e 100644 --- a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql +++ b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql @@ -56,7 +56,7 @@ AND modified_timestamp > ( FROM {{ source( 'silver_tvl_optimism', - 'silver_tvl__etherfi_v1_tvl' + 'etherfi_v1_tvl' ) }} WHERE attribution = 'ethereum' @@ -85,7 +85,7 @@ SELECT FROM {{ source( 'silver_tvl_arbitrum', - 'silver_tvl__etherfi_v1_tvl' + 'etherfi_v1_tvl' ) }} WHERE attribution = 'ethereum' @@ -114,7 +114,7 @@ SELECT FROM {{ source( 'silver_tvl_base', - 'silver_tvl__etherfi_v1_tvl' + 'etherfi_v1_tvl' ) }} WHERE attribution = 'ethereum' diff --git a/models/sources.yml b/models/sources.yml index de93b100..e24acbff 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -141,17 +141,17 @@ sources: - name: _master_keys - name: rpc_node_logs - name: silver_tvl_optimism - database: silver_tvl_optimism - schema: silver + database: optimism + schema: silver_tvl tables: - - name: silver_tvl__etherfi_v1_tvl + - name: etherfi_v1_tvl - name: silver_tvl_arbitrum - database: silver_tvl_arbitrum - schema: silver + database: arbitrum + schema: silver_tvl tables: - - name: silver_tvl__etherfi_v1_tvl + - name: etherfi_v1_tvl - name: silver_tvl_base - database: silver_tvl_base - schema: silver + database: base + schema: silver_tvl tables: - - name: silver_tvl__etherfi_v1_tvl \ No newline at end of file + - name: etherfi_v1_tvl \ No newline at end of file From 8c950158fba52efb21440bd35807f99e3d8de242 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 15 Dec 2025 17:07:55 -0700 Subject: [PATCH 114/164] binance and etherfi descriptions --- models/doc_descriptions/defi/silver_tvl.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/models/doc_descriptions/defi/silver_tvl.md b/models/doc_descriptions/defi/silver_tvl.md index 9b1b1a28..18c43b6d 100644 --- a/models/doc_descriptions/defi/silver_tvl.md +++ b/models/doc_descriptions/defi/silver_tvl.md @@ -19,6 +19,13 @@ Sums totalSupply of each aToken contract {% enddocs %} +{% docs binance_staked_eth_v1_tvl_table_doc %} + +Methodology: +Total supply of wBETH + +{% enddocs %} + {% docs curve_v1_tvl_table_doc %} Methodology: @@ -26,6 +33,13 @@ Sums all token balances held across pool contracts {% enddocs %} +{% docs etherfi_v1_tvl_table_doc %} + +Methodology: +Sums values from protocol oracle (minus looped positions), eUSD supply, BTC product balances (WBTC, LBTC, cbBTC) and ETHFI staking where applicable. + +{% enddocs %} + {% docs lido_v1_tvl_table_doc %} Methodology: @@ -73,4 +87,4 @@ Sums token0 and token1 balances held by the PoolManager singleton + any associat Methodology: Sums USDC balances held by the Conditional Tokens and Collateral Tokens contracts -{% enddocs %} +{% enddocs %} \ No newline at end of file From f35a7a1cff06bc9f71e3110355a60916d51ff606 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 16 Dec 2025 12:50:23 -0700 Subject: [PATCH 115/164] depends --- .../streamline/records/streamline__contract_reads_records.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index 0d287d92..1176f5f5 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -4,7 +4,8 @@ {# Log configuration details #} {{ log_model_details() }} --- depends_on: {{ ref('silver_reads__lido_reads') }} +-- depends_on: {{ ref('silver_reads__lido_v1_reads') }} +-- depends_on: {{ ref('silver_reads__binance_v1_reads') }} {{ config ( materialized = "incremental", unique_key = "contract_reads_records_id", From 83a8c73df0b8d1d5e4bf6cef03de57b4acc6b5c1 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 16 Dec 2025 12:54:16 -0700 Subject: [PATCH 116/164] doc name --- models/doc_descriptions/defi/silver_tvl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/doc_descriptions/defi/silver_tvl.md b/models/doc_descriptions/defi/silver_tvl.md index 18c43b6d..ed9c889d 100644 --- a/models/doc_descriptions/defi/silver_tvl.md +++ b/models/doc_descriptions/defi/silver_tvl.md @@ -19,7 +19,7 @@ Sums totalSupply of each aToken contract {% enddocs %} -{% docs binance_staked_eth_v1_tvl_table_doc %} +{% docs binance_v1_tvl_table_doc %} Methodology: Total supply of wBETH From 9f7b9aa81c5d5a5f0dcf55e18e9380d189658849 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 16 Dec 2025 13:43:08 -0700 Subject: [PATCH 117/164] filter --- .../defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql index c5ea4823..01ae9bd9 100644 --- a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql @@ -38,7 +38,7 @@ WITH reads AS ( WHERE platform = 'etherfi-v1' AND chain = '{{ vars.GLOBAL_PROJECT_NAME }}' - AND amount_raw IS NOT NULL + AND amount_raw_unadj IS NOT NULL {% if is_incremental() %} AND modified_timestamp > ( From 84147e7ec31996151617a145d871146cfc92f9eb Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 16 Dec 2025 13:55:06 -0700 Subject: [PATCH 118/164] commas --- .../tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql index 536a269e..1d18c308 100644 --- a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql +++ b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql @@ -39,7 +39,7 @@ AND modified_timestamp > ( {{ this }} ) {% endif %} -) {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %}, +), {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} eth_attributed_tvl AS ( SELECT block_number, @@ -127,7 +127,7 @@ AND modified_timestamp > ( {{ this }} ) {% endif %} -) +), {% endif %} FINAL AS ( From 6092562349d1256aa6a0c9769bafc187e96c340e Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 16 Dec 2025 16:02:19 -0700 Subject: [PATCH 119/164] etherfi reads --- .../streamline/records/streamline__contract_reads_records.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index 1176f5f5..e3530be5 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -30,6 +30,7 @@ {% set _ = models.append((ref('silver_reads__aave_v3_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__curve_v1_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__tornado_cash_v1_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__etherfi_v1_reads'), 'daily')) %} WITH all_records AS ( {% for model, type in models %} From b71494dab906387ac6dd6e4bfa972a64bbf5a6bc Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 17 Dec 2025 09:18:41 -0700 Subject: [PATCH 120/164] token address for eth price --- .../defi/reads/silver_reads__etherfi_seed.csv | 38 +++++++++---------- .../silver_reads__etherfi_v1_reads.sql | 2 + .../etherfi/silver_tvl__etherfi_v1_tvl.sql | 8 +++- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/data/curated/defi/reads/silver_reads__etherfi_seed.csv b/data/curated/defi/reads/silver_reads__etherfi_seed.csv index 1414d76b..9d47fb58 100644 --- a/data/curated/defi/reads/silver_reads__etherfi_seed.csv +++ b/data/curated/defi/reads/silver_reads__etherfi_seed.csv @@ -1,19 +1,19 @@ -contract_address,address,function_name,function_sig,chain,attribution -0xab7590cee3ef1a863e9a5877fbb82d9be11504da,null,categoryTVL,0x820a1175,optimism,ethereum -0x86b5780b606940eb59a062aa85a07959518c0161,null,totalSupply,0x18160ddd,arbitrum,arbitrum -0x86b5780b606940eb59a062aa85a07959518c0161,null,totalSupply,0x18160ddd,base,base -0x86b5780b606940eb59a062aa85a07959518c0161,null,totalSupply,0x18160ddd,ethereum,ethereum -0x939778d83b46b456224a33fb59630b11dec56663,null,totalSupply,0x18160ddd,ethereum,ethereum -0x6329004e903b7f420245e7af3f355186f2432466,null,getTvl,0xd075dd42,optimism,ethereum -0x8236a87084f8b84306f72007f36f2618a5634494,0xd4e20eca1f996dab35883dc0ad5e3428af888d45,balanceOf,0x70a08231,ethereum,ethereum -0x2f2a2543b76a4166549f7aab2e75bef0aefc5b0f,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,arbitrum,ethereum -0x8236a87084f8b84306f72007f36f2618a5634494,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum -0xecac9c5f704e954931349da37f60e39f515c11c1,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,base,ethereum -0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum -0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,base,ethereum -0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,arbitrum,ethereum -0x468c34703f6c648ccf39dbab11305d17c70ba011,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum -0x9c0823d3a1172f9ddf672d438dec79c39a64f448,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum -0x126d4dbf752aaf61f3eaada24ab0db84fecf6891,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum -0x971e5b5d4baa5607863f3748febf287c7bf82618,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum -0x2260fac5e5542a773aa44fbcfedf7c193bc2c599,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,balanceOf,0x70a08231,ethereum,ethereum \ No newline at end of file +contract_address,address,token_address,function_name,function_sig,chain,attribution +0xab7590cee3ef1a863e9a5877fbb82d9be11504da,null,0x0000000000000000000000000000000000000000,categoryTVL,0x820a1175,optimism,ethereum +0x86b5780b606940eb59a062aa85a07959518c0161,null,null,totalSupply,0x18160ddd,arbitrum,arbitrum +0x86b5780b606940eb59a062aa85a07959518c0161,null,null,totalSupply,0x18160ddd,base,base +0x86b5780b606940eb59a062aa85a07959518c0161,null,null,totalSupply,0x18160ddd,ethereum,ethereum +0x939778d83b46b456224a33fb59630b11dec56663,null,null,totalSupply,0x18160ddd,ethereum,ethereum +0x6329004e903b7f420245e7af3f355186f2432466,null,0x0000000000000000000000000000000000000000,getTvl,0xd075dd42,optimism,ethereum +0x8236a87084f8b84306f72007f36f2618a5634494,0xd4e20eca1f996dab35883dc0ad5e3428af888d45,null,balanceOf,0x70a08231,ethereum,ethereum +0x2f2a2543b76a4166549f7aab2e75bef0aefc5b0f,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,arbitrum,ethereum +0x8236a87084f8b84306f72007f36f2618a5634494,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,ethereum,ethereum +0xecac9c5f704e954931349da37f60e39f515c11c1,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,base,ethereum +0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,ethereum,ethereum +0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,base,ethereum +0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,arbitrum,ethereum +0x468c34703f6c648ccf39dbab11305d17c70ba011,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,ethereum,ethereum +0x9c0823d3a1172f9ddf672d438dec79c39a64f448,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,ethereum,ethereum +0x126d4dbf752aaf61f3eaada24ab0db84fecf6891,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,ethereum,ethereum +0x971e5b5d4baa5607863f3748febf287c7bf82618,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,ethereum,ethereum +0x2260fac5e5542a773aa44fbcfedf7c193bc2c599,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,ethereum,ethereum \ No newline at end of file diff --git a/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql index 7bda20ff..997f6547 100644 --- a/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql @@ -15,6 +15,7 @@ WITH contracts AS ( SELECT contract_address, address, + token_address, function_name, function_sig, chain, @@ -51,6 +52,7 @@ SELECT ) END AS input, OBJECT_CONSTRUCT( + 'token_address', token_address, 'attribution', attribution, 'chain', chain ) :: variant AS metadata, diff --git a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql index 01ae9bd9..c5729573 100644 --- a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql @@ -27,6 +27,7 @@ WITH reads AS ( WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) END ) AS amount_raw_unadj, + metadata :token_address :: STRING AS token_address, metadata :attribution :: STRING AS attribution, metadata :chain :: STRING AS chain, protocol, @@ -52,7 +53,10 @@ AND modified_timestamp > ( SELECT block_number, block_date, - contract_address, + CASE + WHEN r.token_address IS NOT NULL + THEN r.token_address ELSE r.contract_address + END AS contract_address, address, amount_hex, CASE @@ -72,6 +76,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads qualify(ROW_NUMBER() over(PARTITION BY etherfi_v1_tvl_id + reads r qualify(ROW_NUMBER() over(PARTITION BY etherfi_v1_tvl_id ORDER BY modified_timestamp DESC)) = 1 From e76e62ffa7aba51cf48636a97f0ef500af6cb611 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 17 Dec 2025 09:20:25 -0700 Subject: [PATCH 121/164] lbtc and wbtc prices --- data/curated/defi/reads/silver_reads__etherfi_seed.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/curated/defi/reads/silver_reads__etherfi_seed.csv b/data/curated/defi/reads/silver_reads__etherfi_seed.csv index 9d47fb58..b304d0fc 100644 --- a/data/curated/defi/reads/silver_reads__etherfi_seed.csv +++ b/data/curated/defi/reads/silver_reads__etherfi_seed.csv @@ -6,9 +6,9 @@ contract_address,address,token_address,function_name,function_sig,chain,attribut 0x939778d83b46b456224a33fb59630b11dec56663,null,null,totalSupply,0x18160ddd,ethereum,ethereum 0x6329004e903b7f420245e7af3f355186f2432466,null,0x0000000000000000000000000000000000000000,getTvl,0xd075dd42,optimism,ethereum 0x8236a87084f8b84306f72007f36f2618a5634494,0xd4e20eca1f996dab35883dc0ad5e3428af888d45,null,balanceOf,0x70a08231,ethereum,ethereum -0x2f2a2543b76a4166549f7aab2e75bef0aefc5b0f,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,arbitrum,ethereum +0x2f2a2543b76a4166549f7aab2e75bef0aefc5b0f,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,0x2260fac5e5542a773aa44fbcfedf7c193bc2c599,balanceOf,0x70a08231,arbitrum,ethereum 0x8236a87084f8b84306f72007f36f2618a5634494,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,ethereum,ethereum -0xecac9c5f704e954931349da37f60e39f515c11c1,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,base,ethereum +0xecac9c5f704e954931349da37f60e39f515c11c1,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,0x8236a87084f8b84306f72007f36f2618a5634494,balanceOf,0x70a08231,base,ethereum 0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,ethereum,ethereum 0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,base,ethereum 0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf,0x657e8c867d8b37dcc18fa4caead9c45eb088c642,null,balanceOf,0x70a08231,arbitrum,ethereum From f114581afbfaf240ba0e814d779713394378143e Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 17 Dec 2025 10:41:39 -0700 Subject: [PATCH 122/164] incremental logic --- .../records/etherfi/silver_reads__etherfi_v1_reads.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql index 997f6547..85a63a5d 100644 --- a/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql @@ -26,7 +26,7 @@ WITH contracts AS ( chain = '{{ vars.GLOBAL_PROJECT_NAME }}' {% if is_incremental() %} -WHERE +AND CONCAT(contract_address, '-', COALESCE(address, 'null')) NOT IN ( SELECT CONCAT(contract_address, '-', COALESCE(address, 'null')) From eb0fece724391d8118d5913d60244e937910f6ab Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 17 Dec 2025 11:34:09 -0700 Subject: [PATCH 123/164] etherfi pricing --- .../silver_reads__etherfi_seed.csv | 6 +++--- .../records/etherfi/silver_reads__etherfi_v1_reads.sql | 2 +- .../defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql | 7 +++++-- 3 files changed, 9 insertions(+), 6 deletions(-) rename data/curated/{defi/reads => contract_reads}/silver_reads__etherfi_seed.csv (84%) diff --git a/data/curated/defi/reads/silver_reads__etherfi_seed.csv b/data/curated/contract_reads/silver_reads__etherfi_seed.csv similarity index 84% rename from data/curated/defi/reads/silver_reads__etherfi_seed.csv rename to data/curated/contract_reads/silver_reads__etherfi_seed.csv index b304d0fc..4421be69 100644 --- a/data/curated/defi/reads/silver_reads__etherfi_seed.csv +++ b/data/curated/contract_reads/silver_reads__etherfi_seed.csv @@ -1,8 +1,8 @@ contract_address,address,token_address,function_name,function_sig,chain,attribution 0xab7590cee3ef1a863e9a5877fbb82d9be11504da,null,0x0000000000000000000000000000000000000000,categoryTVL,0x820a1175,optimism,ethereum -0x86b5780b606940eb59a062aa85a07959518c0161,null,null,totalSupply,0x18160ddd,arbitrum,arbitrum -0x86b5780b606940eb59a062aa85a07959518c0161,null,null,totalSupply,0x18160ddd,base,base -0x86b5780b606940eb59a062aa85a07959518c0161,null,null,totalSupply,0x18160ddd,ethereum,ethereum +0x86b5780b606940eb59a062aa85a07959518c0161,null,0x6c240dda6b5c336df09a4d011139beaaa1ea2aa2,totalSupply,0x18160ddd,arbitrum,arbitrum +0x86b5780b606940eb59a062aa85a07959518c0161,null,0x6c240dda6b5c336df09a4d011139beaaa1ea2aa2,totalSupply,0x18160ddd,base,base +0x86b5780b606940eb59a062aa85a07959518c0161,null,0x6c240dda6b5c336df09a4d011139beaaa1ea2aa2,totalSupply,0x18160ddd,ethereum,ethereum 0x939778d83b46b456224a33fb59630b11dec56663,null,null,totalSupply,0x18160ddd,ethereum,ethereum 0x6329004e903b7f420245e7af3f355186f2432466,null,0x0000000000000000000000000000000000000000,getTvl,0xd075dd42,optimism,ethereum 0x8236a87084f8b84306f72007f36f2618a5634494,0xd4e20eca1f996dab35883dc0ad5e3428af888d45,null,balanceOf,0x70a08231,ethereum,ethereum diff --git a/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql index 85a63a5d..9e51616f 100644 --- a/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql @@ -42,7 +42,7 @@ SELECT function_sig, CASE WHEN function_name = 'balanceOf' THEN CONCAT( - '0x70a08231', + function_sig, LPAD(SUBSTR(address, 3), 64, '0') ) ELSE RPAD( diff --git a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql index c5729573..d9805688 100644 --- a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql @@ -57,10 +57,13 @@ SELECT WHEN r.token_address IS NOT NULL THEN r.token_address ELSE r.contract_address END AS contract_address, - address, + CASE + WHEN r.token_address IS NOT NULL AND r.address IS NULL + THEN r.contract_address ELSE r.address + END AS address, amount_hex, CASE - WHEN contract_address = '0xab7590cee3ef1a863e9a5877fbb82d9be11504da' THEN -1 * amount_raw_unadj + WHEN r.contract_address = '0xab7590cee3ef1a863e9a5877fbb82d9be11504da' THEN -1 * amount_raw_unadj --categoryTVL() must be subtracted from getTVL(), negative value enables SUM ELSE amount_raw_unadj END AS amount_raw, From 1149e189344ca672f1ebc67436fff95d06bf15bb Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 17 Dec 2025 11:35:20 -0700 Subject: [PATCH 124/164] tokens --- data/curated/contract_reads/silver_reads__etherfi_seed.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/curated/contract_reads/silver_reads__etherfi_seed.csv b/data/curated/contract_reads/silver_reads__etherfi_seed.csv index 4421be69..c084e905 100644 --- a/data/curated/contract_reads/silver_reads__etherfi_seed.csv +++ b/data/curated/contract_reads/silver_reads__etherfi_seed.csv @@ -1,8 +1,8 @@ contract_address,address,token_address,function_name,function_sig,chain,attribution 0xab7590cee3ef1a863e9a5877fbb82d9be11504da,null,0x0000000000000000000000000000000000000000,categoryTVL,0x820a1175,optimism,ethereum -0x86b5780b606940eb59a062aa85a07959518c0161,null,0x6c240dda6b5c336df09a4d011139beaaa1ea2aa2,totalSupply,0x18160ddd,arbitrum,arbitrum +0x86b5780b606940eb59a062aa85a07959518c0161,null,0x7189fb5b6504bbff6a852b13b7b82a3c118fdc27,totalSupply,0x18160ddd,arbitrum,arbitrum 0x86b5780b606940eb59a062aa85a07959518c0161,null,0x6c240dda6b5c336df09a4d011139beaaa1ea2aa2,totalSupply,0x18160ddd,base,base -0x86b5780b606940eb59a062aa85a07959518c0161,null,0x6c240dda6b5c336df09a4d011139beaaa1ea2aa2,totalSupply,0x18160ddd,ethereum,ethereum +0x86b5780b606940eb59a062aa85a07959518c0161,null,0xfe0c30065b384f05761f15d0cc899d4f9f9cc0eb,totalSupply,0x18160ddd,ethereum,ethereum 0x939778d83b46b456224a33fb59630b11dec56663,null,null,totalSupply,0x18160ddd,ethereum,ethereum 0x6329004e903b7f420245e7af3f355186f2432466,null,0x0000000000000000000000000000000000000000,getTvl,0xd075dd42,optimism,ethereum 0x8236a87084f8b84306f72007f36f2618a5634494,0xd4e20eca1f996dab35883dc0ad5e3428af888d45,null,balanceOf,0x70a08231,ethereum,ethereum From 20e6ad760d48a5e837e872b075b21c0306a6b899 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 17 Dec 2025 13:47:25 -0700 Subject: [PATCH 125/164] token address column --- .../defi/tvl/gold/defi__ez_protocol_tvl.sql | 1 + .../silver/aave/silver_tvl__aave_v1_tvl.sql | 1 + .../silver/aave/silver_tvl__aave_v2_tvl.sql | 1 + .../silver/aave/silver_tvl__aave_v3_tvl.sql | 7 +-- .../binance/silver_tvl__binance_v1_tvl.sql | 1 + .../silver/curve/silver_tvl__curve_v1_tvl.sql | 1 + .../etherfi/silver_tvl__etherfi_v1_tvl.sql | 12 ++--- .../silver_tvl__etherfi_v1_tvl_agg.sql | 5 ++ .../silver/lido/silver_tvl__lido_v1_tvl.sql | 1 + .../silver_tvl__polymarket_v1_tvl.sql | 1 + .../tvl/silver/silver_tvl__complete_tvl.sql | 15 ++++-- .../silver_tvl__tornado_cash_v1_tvl.sql | 1 + .../uniswap/silver_tvl__uniswap_v1_tvl.sql | 48 ++++++++++++++++--- .../uniswap/silver_tvl__uniswap_v2_tvl.sql | 1 + .../uniswap/silver_tvl__uniswap_v3_tvl.sql | 1 + .../uniswap/silver_tvl__uniswap_v4_tvl.sql | 1 + 16 files changed, 77 insertions(+), 21 deletions(-) diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql index ccf4d182..656f7431 100644 --- a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql @@ -18,6 +18,7 @@ WITH complete_tvl AS ( block_date, contract_address, address, + token_address, decimals, symbol, is_verified, diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql index 0bad5902..4d3e5da4 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v1_tvl.sql @@ -96,6 +96,7 @@ SELECT block_date, contract_address, address, + contract_address AS token_address, amount_hex, amount_raw, protocol, diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql index db8482e4..0c51d542 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v2_tvl.sql @@ -54,6 +54,7 @@ SELECT block_date, contract_address, address, + contract_address AS token_address, amount_hex, amount_raw, protocol, diff --git a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql index 30c48d9b..2d581acf 100644 --- a/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql +++ b/models/curated_package/defi/tvl/silver/aave/silver_tvl__aave_v3_tvl.sql @@ -14,7 +14,7 @@ WITH reads AS ( SELECT block_number, block_date, - contract_address AS atoken_address, + contract_address, address, result_hex AS amount_hex, IFNULL( @@ -53,8 +53,9 @@ AND modified_timestamp > ( SELECT block_number, block_date, - underlying_address AS contract_address, - atoken_address AS address, + contract_address, + address, + underlying_address AS token_address, amount_hex, amount_raw, protocol, diff --git a/models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.sql b/models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.sql index c98827b4..370ae715 100644 --- a/models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/binance/silver_tvl__binance_v1_tvl.sql @@ -51,6 +51,7 @@ SELECT block_date, contract_address, address, + contract_address AS token_address, amount_hex, amount_raw, protocol, diff --git a/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.sql b/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.sql index 90ed20e3..3b3379d2 100644 --- a/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/curve/silver_tvl__curve_v1_tvl.sql @@ -51,6 +51,7 @@ SELECT block_date, contract_address, address, + contract_address AS token_address, amount_hex, amount_raw, protocol, diff --git a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql index d9805688..0e94b650 100644 --- a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl.sql @@ -53,14 +53,12 @@ AND modified_timestamp > ( SELECT block_number, block_date, + contract_address, + address, CASE - WHEN r.token_address IS NOT NULL - THEN r.token_address ELSE r.contract_address - END AS contract_address, - CASE - WHEN r.token_address IS NOT NULL AND r.address IS NULL - THEN r.contract_address ELSE r.address - END AS address, + WHEN r.token_address IS NOT NULL THEN r.token_address + ELSE contract_address + END AS token_address, amount_hex, CASE WHEN r.contract_address = '0xab7590cee3ef1a863e9a5877fbb82d9be11504da' THEN -1 * amount_raw_unadj diff --git a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql index 1d18c308..d4a184a8 100644 --- a/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql +++ b/models/curated_package/defi/tvl/silver/etherfi/silver_tvl__etherfi_v1_tvl_agg.sql @@ -18,6 +18,7 @@ WITH chain_tvl AS ( block_date, contract_address, address, + token_address, amount_hex, amount_raw, protocol, @@ -46,6 +47,7 @@ AND modified_timestamp > ( block_date, contract_address, address, + token_address, amount_hex, amount_raw, protocol, @@ -75,6 +77,7 @@ SELECT block_date, contract_address, address, + token_address, amount_hex, amount_raw, protocol, @@ -104,6 +107,7 @@ SELECT block_date, contract_address, address, + token_address, amount_hex, amount_raw, protocol, @@ -149,6 +153,7 @@ SELECT block_date, contract_address, address, + token_address, amount_hex, amount_raw, protocol, diff --git a/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.sql b/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.sql index 18941913..e84abde2 100644 --- a/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/lido/silver_tvl__lido_v1_tvl.sql @@ -51,6 +51,7 @@ SELECT block_date, contract_address, address, + contract_address AS token_address, amount_hex, amount_raw, protocol, diff --git a/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql b/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql index 8ee38ebd..42ffea84 100644 --- a/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/polymarket/silver_tvl__polymarket_v1_tvl.sql @@ -51,6 +51,7 @@ SELECT block_date, contract_address, address, + contract_address AS token_address, amount_hex, amount_raw, protocol, diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index 689972c0..445fef15 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -31,6 +31,7 @@ WITH all_tvl AS ( block_date, contract_address, address, + token_address, amount_hex, amount_raw, protocol, @@ -95,6 +96,7 @@ complete_tvl AS ( A.block_date, A.contract_address, A.address, + A.token_address, c1.token_decimals AS decimals, c1.token_symbol AS symbol, p1.is_verified, @@ -116,9 +118,9 @@ complete_tvl AS ( FROM all_tvl A LEFT JOIN contracts c1 - ON A.contract_address = c1.contract_address + ON A.token_address = c1.contract_address LEFT JOIN prices p1 - ON A.contract_address = p1.token_address + ON A.token_address = p1.token_address AND DATEADD( 'hour', 23, @@ -135,6 +137,7 @@ heal_model AS ( t.block_date, t.contract_address, t.address, + t.token_address, c1.token_decimals AS decimals_heal, c1.token_symbol AS symbol_heal, p1.is_verified AS is_verified_heal, @@ -157,13 +160,13 @@ heal_model AS ( {{ this }} t LEFT JOIN contracts c1 - ON A.contract_address = c1.contract_address + ON t.token_address = c1.contract_address LEFT JOIN prices p1 - ON A.contract_address = p1.token_address + ON t.token_address = p1.token_address AND DATEADD( 'hour', 23, - A.block_date + t.block_date ) = p1.hour WHERE t.decimals IS NULL ), @@ -184,6 +187,7 @@ SELECT block_date, contract_address, address, + token_address, decimals_heal AS decimals, symbol_heal AS symbol, is_verified_heal AS is_verified, @@ -205,6 +209,7 @@ SELECT block_date, contract_address, address, + token_address, decimals, symbol, is_verified, diff --git a/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql b/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql index 7f68fb60..16645633 100644 --- a/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/tornado_cash/silver_tvl__tornado_cash_v1_tvl.sql @@ -99,6 +99,7 @@ SELECT block_date, contract_address, address, + contract_address AS token_address, amount_hex, amount_raw, protocol, diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql index d33f61c4..ac47dea9 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v1_tvl.sql @@ -16,8 +16,8 @@ WITH balances AS ( SELECT b.block_number, b.block_date, - token1 AS contract_address, - --every pool is token-ETH only, no token-token pairs so TVL = 2x ETH value (pools are always 50/50 and 0x000 e.g. token1, represents native asset) + p.token0, + p.token1, b.address, balance_hex, balance_raw, @@ -42,24 +42,60 @@ AND b.modified_timestamp > ( {{ this }} ) {% endif %} +), + +expanded AS ( + -- Row for token0 as contract_address + SELECT + block_number, + block_date, + token0 AS contract_address, + token1 AS token_address, + address, + balance_hex, + balance_raw, + protocol, + version, + platform + FROM + balances + + UNION ALL + + -- Row for token1 as contract_address + SELECT + block_number, + block_date, + token1 AS contract_address, + token1 AS token_address, + address, + balance_hex, + balance_raw, + protocol, + version, + platform + FROM + balances ) + SELECT block_number, block_date, contract_address, address, + token_address, balance_hex AS amount_hex, - balance_raw * 2 AS amount_raw, -- 2x ETH value since all pools are 50/50 ETH-token pairs + balance_raw AS amount_raw, protocol, version, platform, {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address','platform'] + ['block_date','contract_address','token_address','platform'] ) }} AS uniswap_v1_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - balances qualify(ROW_NUMBER() over(PARTITION BY uniswap_v1_tvl_id + expanded qualify(ROW_NUMBER() over(PARTITION BY uniswap_v1_tvl_id ORDER BY - modified_timestamp DESC)) = 1 + block_number DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql index b039bce3..62af3a93 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql @@ -104,6 +104,7 @@ SELECT block_date, contract_address, address, + contract_address AS token_address, amount_hex, amount_raw, protocol, diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql index fbd2a3d4..641bef33 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql @@ -56,6 +56,7 @@ SELECT block_date, contract_address, address, + contract_address AS token_address, amount_hex, amount_raw, token_0_address, diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql index d8ba4fbb..a0320b29 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql @@ -55,6 +55,7 @@ SELECT block_date, contract_address, address, + contract_address AS token_address, amount_hex, amount_raw, address_type, From d2c8365744eab69180ad83f345fffa88a4ffa10f Mon Sep 17 00:00:00 2001 From: Austin Date: Fri, 19 Dec 2025 10:16:23 -0500 Subject: [PATCH 126/164] coalesce --- .../prices/silver/silver__complete_native_asset_metadata.sql | 4 ++-- .../prices/silver/silver__complete_native_prices.sql | 4 ++-- .../silver/silver__complete_provider_asset_metadata.sql | 4 ++-- .../prices/silver/silver__complete_provider_prices.sql | 4 ++-- .../prices/silver/silver__complete_token_asset_metadata.sql | 4 ++-- .../prices/silver/silver__complete_token_prices.sql | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/models/main_package/prices/silver/silver__complete_native_asset_metadata.sql b/models/main_package/prices/silver/silver__complete_native_asset_metadata.sql index c24119b5..713490c0 100644 --- a/models/main_package/prices/silver/silver__complete_native_asset_metadata.sql +++ b/models/main_package/prices/silver/silver__complete_native_asset_metadata.sql @@ -33,9 +33,9 @@ FROM WHERE modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) diff --git a/models/main_package/prices/silver/silver__complete_native_prices.sql b/models/main_package/prices/silver/silver__complete_native_prices.sql index c128229f..f7a0be57 100644 --- a/models/main_package/prices/silver/silver__complete_native_prices.sql +++ b/models/main_package/prices/silver/silver__complete_native_prices.sql @@ -37,9 +37,9 @@ FROM WHERE modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) diff --git a/models/main_package/prices/silver/silver__complete_provider_asset_metadata.sql b/models/main_package/prices/silver/silver__complete_provider_asset_metadata.sql index dccedac2..5289f3d7 100644 --- a/models/main_package/prices/silver/silver__complete_provider_asset_metadata.sql +++ b/models/main_package/prices/silver/silver__complete_provider_asset_metadata.sql @@ -33,9 +33,9 @@ FROM WHERE modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) diff --git a/models/main_package/prices/silver/silver__complete_provider_prices.sql b/models/main_package/prices/silver/silver__complete_provider_prices.sql index 7de9677c..cb20efa6 100644 --- a/models/main_package/prices/silver/silver__complete_provider_prices.sql +++ b/models/main_package/prices/silver/silver__complete_provider_prices.sql @@ -38,9 +38,9 @@ FROM WHERE p.modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) diff --git a/models/main_package/prices/silver/silver__complete_token_asset_metadata.sql b/models/main_package/prices/silver/silver__complete_token_asset_metadata.sql index 3571415e..05e47be6 100644 --- a/models/main_package/prices/silver/silver__complete_token_asset_metadata.sql +++ b/models/main_package/prices/silver/silver__complete_token_asset_metadata.sql @@ -40,9 +40,9 @@ FROM WHERE modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) diff --git a/models/main_package/prices/silver/silver__complete_token_prices.sql b/models/main_package/prices/silver/silver__complete_token_prices.sql index 4dead8dc..8ddc83fc 100644 --- a/models/main_package/prices/silver/silver__complete_token_prices.sql +++ b/models/main_package/prices/silver/silver__complete_token_prices.sql @@ -44,9 +44,9 @@ FROM WHERE modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) From afebfed2c2e416ad0b6f3271bbeba230dd51f412 Mon Sep 17 00:00:00 2001 From: Austin Date: Fri, 19 Dec 2025 10:56:27 -0500 Subject: [PATCH 127/164] workflow update --- .github/workflows/dbt_run_template_docs_update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_template_docs_update.yml b/.github/workflows/dbt_run_template_docs_update.yml index 27833777..cdaf728a 100644 --- a/.github/workflows/dbt_run_template_docs_update.yml +++ b/.github/workflows/dbt_run_template_docs_update.yml @@ -58,7 +58,7 @@ jobs: git checkout -B docs origin/main - name: generate dbt docs - run: dbt docs generate -t prod --exclude "fsc_evm,tag:streamline" "fsc_evm,tag:scores" + run: dbt docs generate -t prod --select "$PROJECT_NAME.*" "fsc_evm,tag:core" "fsc_evm,tag:prices" "fsc_evm,tag:labels" "fsc_evm,tag:balances" "fsc_evm,tag:decoded_logs" "fsc_evm,tag:abis" "fsc_evm,tag:defi" "fsc_evm,tag:stats" - name: move files to docs directory run: | From 8c91dbe4ded9620817c93f55d6c5afb86de6bde5 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 22 Dec 2025 11:47:06 -0700 Subject: [PATCH 128/164] bigint --- .../tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql index 62af3a93..94dd7576 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql @@ -25,17 +25,17 @@ WITH reads AS ( IFNULL( CASE WHEN LENGTH(reserve_0_hex) <= 4300 - AND reserve_0_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(reserve_0_hex) AS FLOAT)END, + AND reserve_0_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(reserve_0_hex) AS bigint)END, CASE - WHEN reserve_0_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(reserve_0_hex, '0')) AS FLOAT) + WHEN reserve_0_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(reserve_0_hex, '0')) AS bigint) END ) AS reserve_0_raw, IFNULL( CASE WHEN LENGTH(reserve_1_hex) <= 4300 - AND reserve_1_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(reserve_1_hex) AS FLOAT)END, + AND reserve_1_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(reserve_1_hex) AS bigint)END, CASE - WHEN reserve_1_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(reserve_1_hex, '0')) AS FLOAT) + WHEN reserve_1_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(reserve_1_hex, '0')) AS bigint) END ) AS reserve_1_raw, IFNULL( From 43dba945e854bffeb388ca7cef195a8ee8f00b0e Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 23 Dec 2025 09:53:37 -0700 Subject: [PATCH 129/164] alt node vars --- macros/global/variables/project_vars/somnia_vars.sql | 2 ++ macros/global/variables/return_vars.sql | 2 ++ .../streamline__balances_native_daily_history_requests.sql | 7 ++++++- ...streamline__balances_native_daily_realtime_requests.sql | 7 ++++++- .../history/streamline__contract_reads_daily_history.sql | 4 ++++ .../realtime/streamline__contract_reads_daily_realtime.sql | 4 ++++ 6 files changed, 24 insertions(+), 2 deletions(-) diff --git a/macros/global/variables/project_vars/somnia_vars.sql b/macros/global/variables/project_vars/somnia_vars.sql index 0a2e142e..89056418 100644 --- a/macros/global/variables/project_vars/somnia_vars.sql +++ b/macros/global/variables/project_vars/somnia_vars.sql @@ -4,6 +4,8 @@ 'GLOBAL_NODE_PROVIDER': 'flipside', 'GLOBAL_NODE_URL': "{URL}", 'GLOBAL_NODE_VAULT_PATH': 'vault/prod/evm/flipside/somnia/mainnet', + 'GLOBAL_ALT_NODE_ENABLED': true, + 'GLOBAL_ALT_NODE_VAULT_PATH': 'vault/prod/evm/ankr/somnia/mainnet', 'GLOBAL_WRAPPED_NATIVE_ASSET_ADDRESS': '0x046ede9564a72571df6f5e44d0405360c0f4dcab', 'GLOBAL_WRAPPED_NATIVE_ASSET_SYMBOL': 'WSOMI', 'GLOBAL_NATIVE_ASSET_SYMBOL': 'SOMI', diff --git a/macros/global/variables/return_vars.sql b/macros/global/variables/return_vars.sql index 61677e29..9492c2d4 100644 --- a/macros/global/variables/return_vars.sql +++ b/macros/global/variables/return_vars.sql @@ -19,6 +19,8 @@ {% set ns.GLOBAL_NODE_VAULT_PATH = get_var('GLOBAL_NODE_VAULT_PATH', '') %} {% set ns.GLOBAL_NETWORK = get_var('GLOBAL_NETWORK', 'mainnet') %} {% set ns.GLOBAL_START_BLOCK = get_var('GLOBAL_START_BLOCK', 0) %} + {% set ns.GLOBAL_ALT_NODE_ENABLED = get_var('GLOBAL_ALT_NODE_ENABLED', false) %} + {% set ns.GLOBAL_ALT_NODE_VAULT_PATH = get_var('GLOBAL_ALT_NODE_VAULT_PATH', '') %} {% set ns.GLOBAL_BRONZE_FR_ENABLED = none if get_var('GLOBAL_BRONZE_FR_ENABLED', false) else false %} {# Sets to none if true, still requires --full-refresh, otherwise will use incremental #} {% set ns.GLOBAL_SILVER_FR_ENABLED = none if get_var('GLOBAL_SILVER_FR_ENABLED', false) else false %} diff --git a/models/balances_package/native/streamline/history/streamline__balances_native_daily_history_requests.sql b/models/balances_package/native/streamline/history/streamline__balances_native_daily_history_requests.sql index aa31527b..e3c41dbe 100644 --- a/models/balances_package/native/streamline/history/streamline__balances_native_daily_history_requests.sql +++ b/models/balances_package/native/streamline/history/streamline__balances_native_daily_history_requests.sql @@ -77,7 +77,12 @@ SELECT 'x-fsc-livequery', 'true' ), 'method', 'POST', - 'secret_name', '{{ vars.GLOBAL_NODE_VAULT_PATH }}', + 'secret_name', + {% if vars.GLOBAL_ALT_NODE_ENABLED %} + '{{ vars.GLOBAL_ALT_NODE_VAULT_PATH }}' + {% else %} + '{{ vars.GLOBAL_NODE_VAULT_PATH }}' + {% endif %}, 'url', '{{ vars.GLOBAL_NODE_URL }}' ) AS request FROM diff --git a/models/balances_package/native/streamline/realtime/streamline__balances_native_daily_realtime_requests.sql b/models/balances_package/native/streamline/realtime/streamline__balances_native_daily_realtime_requests.sql index cc01a992..98ff2b45 100644 --- a/models/balances_package/native/streamline/realtime/streamline__balances_native_daily_realtime_requests.sql +++ b/models/balances_package/native/streamline/realtime/streamline__balances_native_daily_realtime_requests.sql @@ -79,7 +79,12 @@ SELECT 'x-fsc-livequery', 'true' ), 'method', 'POST', - 'secret_name', '{{ vars.GLOBAL_NODE_VAULT_PATH }}', + 'secret_name', + {% if vars.GLOBAL_ALT_NODE_ENABLED %} + '{{ vars.GLOBAL_ALT_NODE_VAULT_PATH }}' + {% else %} + '{{ vars.GLOBAL_NODE_VAULT_PATH }}' + {% endif %}, 'url', '{{ vars.GLOBAL_NODE_URL }}' ) AS request FROM diff --git a/models/curated_package/defi/stablecoins/reads/streamline/history/streamline__contract_reads_daily_history.sql b/models/curated_package/defi/stablecoins/reads/streamline/history/streamline__contract_reads_daily_history.sql index b17e0f11..eb58e87e 100644 --- a/models/curated_package/defi/stablecoins/reads/streamline/history/streamline__contract_reads_daily_history.sql +++ b/models/curated_package/defi/stablecoins/reads/streamline/history/streamline__contract_reads_daily_history.sql @@ -102,7 +102,11 @@ SELECT block_number ) ), + {% if vars.GLOBAL_ALT_NODE_ENABLED %} + '{{ vars.GLOBAL_ALT_NODE_VAULT_PATH }}' + {% else %} '{{ vars.GLOBAL_NODE_VAULT_PATH }}' + {% endif %} ) AS request FROM ready_reads diff --git a/models/curated_package/defi/stablecoins/reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql b/models/curated_package/defi/stablecoins/reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql index 0e8fd500..e1edd614 100644 --- a/models/curated_package/defi/stablecoins/reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql +++ b/models/curated_package/defi/stablecoins/reads/streamline/realtime/streamline__contract_reads_daily_realtime.sql @@ -98,7 +98,11 @@ SELECT block_number ) ), + {% if vars.GLOBAL_ALT_NODE_ENABLED %} + '{{ vars.GLOBAL_ALT_NODE_VAULT_PATH }}' + {% else %} '{{ vars.GLOBAL_NODE_VAULT_PATH }}' + {% endif %} ) AS request FROM ready_reads From 4e944209a7951f217eb9b6dceeea7c00a8eb7304 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 23 Dec 2025 12:24:17 -0700 Subject: [PATCH 130/164] verified token test --- .../uniswap/silver_reads__uniswap_v2_reads.sql | 15 +++++++++++++-- .../uniswap/silver_reads__uniswap_v3_reads.sql | 16 +++++++++++++--- .../uniswap/silver_reads__uniswap_v4_reads.sql | 17 +++++++++++++---- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql index 3f6983b5..a4bb897c 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql @@ -11,7 +11,16 @@ tags = ['silver','contract_reads'] ) }} -WITH liquidity_pools AS ( +WITH verified_contracts AS ( + SELECT + DISTINCT token_address + FROM + {{ ref('price__ez_asset_metadata') }} + WHERE + is_verified + AND token_address IS NOT NULL +), +liquidity_pools AS ( SELECT DISTINCT pool_address AS contract_address, @@ -21,8 +30,10 @@ WITH liquidity_pools AS ( version, platform FROM {{ ref('silver_dex__paircreated_evt_v2_pools') }} + WHERE token0 IN (SELECT token_address FROM verified_contracts) + AND token1 IN (SELECT token_address FROM verified_contracts) {% if is_incremental() %} - WHERE modified_timestamp > ( + AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM {{ this }} ) diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql index 797801d2..a268133b 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql @@ -11,7 +11,16 @@ tags = ['silver','contract_reads'] ) }} -WITH liquidity_pools AS ( +WITH verified_contracts AS ( + SELECT + DISTINCT token_address + FROM + {{ ref('price__ez_asset_metadata') }} + WHERE + is_verified + AND token_address IS NOT NULL +), +liquidity_pools AS ( SELECT DISTINCT pool_address, @@ -21,14 +30,15 @@ WITH liquidity_pools AS ( version, platform FROM {{ ref('silver_dex__poolcreated_evt_v3_pools') }} + WHERE token0 IN (SELECT token_address FROM verified_contracts) + AND token1 IN (SELECT token_address FROM verified_contracts) {% if is_incremental() %} - WHERE modified_timestamp > ( + AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM {{ this }} ) {% endif %} ), - lp_balances AS ( SELECT token0 AS contract_address, diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql index 5dad6ca1..0a1a128f 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql @@ -11,7 +11,16 @@ tags = ['silver','contract_reads'] ) }} -WITH liquidity_pools AS ( +WITH verified_contracts AS ( + SELECT + DISTINCT token_address + FROM + {{ ref('price__ez_asset_metadata') }} + WHERE + is_verified + AND token_address IS NOT NULL +), +liquidity_pools AS ( SELECT token0, @@ -23,10 +32,10 @@ WITH liquidity_pools AS ( platform FROM {{ ref('silver_dex__uniswap_v4_pools') }} - + WHERE token0 IN (SELECT token_address FROM verified_contracts) + AND token1 IN (SELECT token_address FROM verified_contracts) {% if is_incremental() %} -WHERE - modified_timestamp > ( +AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM From 9524636f5e8265f9e69bd989478e0409153db25f Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 29 Dec 2025 08:11:07 -0700 Subject: [PATCH 131/164] recency vars --- macros/global/variables/project_vars/arbitrum_vars.sql | 2 +- macros/global/variables/project_vars/avalanche_vars.sql | 4 ++-- macros/global/variables/project_vars/ethereum_vars.sql | 2 +- macros/global/variables/project_vars/optimism_vars.sql | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/macros/global/variables/project_vars/arbitrum_vars.sql b/macros/global/variables/project_vars/arbitrum_vars.sql index 8f676508..2006f491 100644 --- a/macros/global/variables/project_vars/arbitrum_vars.sql +++ b/macros/global/variables/project_vars/arbitrum_vars.sql @@ -23,7 +23,7 @@ 'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['radiant-v2','radiant-v1'], 'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['hashflow-v1','gmx-v1','woofi-v3','woofi-v1'], 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['trader_joe-v2','kyberswap-v1','kyberswap-v2','zyberswap-v2','fraxswap-v1'], - 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['hyperliquid-v1','across-v2','multichain-v7','symbiosis-v1','hop-v1'], + 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['hyperliquid-v1','across-v2','multichain-v7','symbiosis-v1','hop-v1','everclear-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'uniswap': { 'v2': { diff --git a/macros/global/variables/project_vars/avalanche_vars.sql b/macros/global/variables/project_vars/avalanche_vars.sql index 280cecbd..fded5e3d 100644 --- a/macros/global/variables/project_vars/avalanche_vars.sql +++ b/macros/global/variables/project_vars/avalanche_vars.sql @@ -22,8 +22,8 @@ 'CUSTOM_GHA_SCHEDULED_DEXALOT_MAIN_CRON': '5 * * * *', 'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['aave-v2'], 'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['gmx-v1','hashflow-v1','woofi-v1'], - 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['pharaoh-v1','kyberswap-v2','curve-v1','uniswap-v2','kyberswap-v1','fraxswap-v1'], - 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['symbiosis-v1','multichain-v7','allbridge-v1'], + 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['pharaoh-v1','kyberswap-v2','curve-v1','uniswap-v2','kyberswap-v1','fraxswap-v1','pharaoh-v2'], + 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['symbiosis-v1','multichain-v7','allbridge-v1','everclear-v1','eywa-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'arena_trade': { 'v1': { diff --git a/macros/global/variables/project_vars/ethereum_vars.sql b/macros/global/variables/project_vars/ethereum_vars.sql index b113e0a4..630d0f0d 100644 --- a/macros/global/variables/project_vars/ethereum_vars.sql +++ b/macros/global/variables/project_vars/ethereum_vars.sql @@ -28,7 +28,7 @@ 'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['silo-v1','aave-v1','flux-v1','aave-v2','aave-v2.1','sturdy-v1','silo-v2'], 'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['hashflow-v1','pancakeswap-v2','synthetix-v1'], 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['kyberswap-v2'], - 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['ronin_axie_bridge-v1','multichain-v7','symbiosis-v1','axie_infinity-v2','across-v2','near_rainbow_bridge-v1','hop-v1'], + 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['ronin_axie_bridge-v1','multichain-v7','symbiosis-v1','axie_infinity-v2','across-v2','near_rainbow_bridge-v1','hop-v1','everclear-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'uniswap': { 'v2': { diff --git a/macros/global/variables/project_vars/optimism_vars.sql b/macros/global/variables/project_vars/optimism_vars.sql index 5a0bf41d..4fca6e5a 100644 --- a/macros/global/variables/project_vars/optimism_vars.sql +++ b/macros/global/variables/project_vars/optimism_vars.sql @@ -22,7 +22,7 @@ 'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['granary-v1','sonne-v1'], 'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['woofi-v1','hashflow-v1'], 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['fraxswap-v1','kyberswap-v2','sushiswap-v1','uniswap-v2'], - 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['eywa-v1','across-v2','meson-v1','symbiosis-v1','multichain-v7','hop-v1'], + 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['eywa-v1','across-v2','meson-v1','symbiosis-v1','multichain-v7','hop-v1','everclear-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'uniswap': { 'v2': { From 141dca64cadd20be7328d52d096db62f4de6c150 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 29 Dec 2025 08:23:58 -0700 Subject: [PATCH 132/164] ink --- macros/global/variables/project_vars/ink_vars.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/macros/global/variables/project_vars/ink_vars.sql b/macros/global/variables/project_vars/ink_vars.sql index 0774c79b..b9fbea85 100644 --- a/macros/global/variables/project_vars/ink_vars.sql +++ b/macros/global/variables/project_vars/ink_vars.sql @@ -18,6 +18,7 @@ 'MAIN_GHA_SCHEDULED_SCORES_CRON': '50 5 * * *', 'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['morpho-v1'], 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['squidswap-v1'], + 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['everclear-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'uniswap': { 'v2': { From ae233e19105d73e26099bb26c86ef35be1fcb6e3 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 29 Dec 2025 14:38:07 -0700 Subject: [PATCH 133/164] miner addresses --- ...eamline__balances_native_daily_records.sql | 16 +++++++++-- ...er__balances_validator_addresses_daily.sql | 28 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) rename models/balances_package/native/streamline/{ => records}/streamline__balances_native_daily_records.sql (88%) create mode 100644 models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql diff --git a/models/balances_package/native/streamline/streamline__balances_native_daily_records.sql b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql similarity index 88% rename from models/balances_package/native/streamline/streamline__balances_native_daily_records.sql rename to models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql index e6f0463a..02fad7f4 100644 --- a/models/balances_package/native/streamline/streamline__balances_native_daily_records.sql +++ b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql @@ -111,10 +111,22 @@ native_transfers_history AS ( WHERE block_date > ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE ), -all_transfers AS ( +validator_addresses AS ( + SELECT + DISTINCT + block_date, + address + FROM + {{ ref('silver__balances_validator_addresses_daily')}} + WHERE + block_date >= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE +) +all_records AS ( SELECT * FROM native_transfers_snapshot UNION SELECT * FROM native_transfers_history + UNION + SELECT * FROM validator_addresses ) SELECT block_date, @@ -124,6 +136,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - all_transfers qualify (ROW_NUMBER() over (PARTITION BY balances_native_daily_records_id + all_records qualify (ROW_NUMBER() over (PARTITION BY balances_native_daily_records_id ORDER BY modified_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql b/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql new file mode 100644 index 00000000..953024bb --- /dev/null +++ b/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql @@ -0,0 +1,28 @@ +{# Get variables #} +{% set vars = return_vars() %} +{# Log configuration details #} +{{ log_model_details() }} +{# Set up dbt configuration #} +{{ config ( + materialized = "incremental", + unique_key = "balances_validator_addresses_daily_id", + cluster_by = "block_date", + tags = ['silver','balances','records','native','phase_4'] +) }} + +WITH miner_addresses AS ( + SELECT + DISTINCT block_timestamp :: DATE AS block_date, + miner AS address + FROM + {{ ref('core__fact_blocks') }} + ) +SELECT + block_date, + address, + {{ dbt_utils.generate_surrogate_key(['block_date', 'address']) }} AS balances_validator_addresses_daily_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + miner_addresses From 49e1293fec0fda7bccda5e1982519d38800860c7 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 29 Dec 2025 14:39:08 -0700 Subject: [PATCH 134/164] incremental --- .../silver__balances_validator_addresses_daily.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql b/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql index 953024bb..1aa9ea76 100644 --- a/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql +++ b/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql @@ -16,6 +16,15 @@ WITH miner_addresses AS ( miner AS address FROM {{ ref('core__fact_blocks') }} + {% if is_incremental() %} + WHERE + modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} ) SELECT block_date, @@ -26,3 +35,6 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM miner_addresses +QUALIFY(ROW_NUMBER() over (PARTITION BY balances_validator_addresses_daily_id +ORDER BY + modified_timestamp DESC)) = 1 \ No newline at end of file From d24a606f012e8fc278b128b37a2fd9feb0f417de Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 29 Dec 2025 14:42:47 -0700 Subject: [PATCH 135/164] 0x00 filter --- .../validators/silver__balances_validator_addresses_daily.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql b/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql index 1aa9ea76..fe923dad 100644 --- a/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql +++ b/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql @@ -16,8 +16,9 @@ WITH miner_addresses AS ( miner AS address FROM {{ ref('core__fact_blocks') }} + WHERE miner <> '0x0000000000000000000000000000000000000000' {% if is_incremental() %} - WHERE + AND modified_timestamp > ( SELECT MAX(modified_timestamp) From 1771a9651354195f24359ba6e9b7df2c2719603d Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 29 Dec 2025 16:02:30 -0700 Subject: [PATCH 136/164] unions --- ...eamline__balances_native_daily_records.sql | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql index 02fad7f4..1f677bc0 100644 --- a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql +++ b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql @@ -51,7 +51,7 @@ tx_fees AS ( ) {% endif %} ), -native_transfers_snapshot AS ( +native_records_snapshot AS ( SELECT DISTINCT ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE AS block_date, @@ -80,8 +80,15 @@ native_transfers_snapshot AS ( tx_fees WHERE block_timestamp :: DATE <= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE + UNION + SELECT + DISTINCT + ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE AS block_date, + address + FROM + {{ ref('silver__balances_validator_addresses_daily')}} ), -native_transfers_history AS ( +native_records_history AS ( SELECT DISTINCT block_timestamp :: DATE AS block_date, @@ -110,8 +117,7 @@ native_transfers_history AS ( tx_fees WHERE block_date > ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE -), -validator_addresses AS ( + UNION SELECT DISTINCT block_date, @@ -119,14 +125,12 @@ validator_addresses AS ( FROM {{ ref('silver__balances_validator_addresses_daily')}} WHERE - block_date >= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE + block_date > ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE ) all_records AS ( - SELECT * FROM native_transfers_snapshot + SELECT * FROM native_records_snapshot UNION - SELECT * FROM native_transfers_history - UNION - SELECT * FROM validator_addresses + SELECT * FROM native_records_history ) SELECT block_date, From c6b2a819ae653afd33087107002b0cac12fbfa43 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 29 Dec 2025 16:10:36 -0700 Subject: [PATCH 137/164] ms incremental --- ...treamline__balances_native_daily_records.sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql index 1f677bc0..ecfabb8d 100644 --- a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql +++ b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql @@ -87,6 +87,15 @@ native_records_snapshot AS ( address FROM {{ ref('silver__balances_validator_addresses_daily')}} + {% if is_incremental() %} + WHERE + modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} ), native_records_history AS ( SELECT @@ -126,6 +135,14 @@ native_records_history AS ( {{ ref('silver__balances_validator_addresses_daily')}} WHERE block_date > ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE + {% if is_incremental() %} + AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} ) all_records AS ( SELECT * FROM native_records_snapshot From 20f84f5b9f2c3925d2b195ace5a5e66200481420 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 30 Dec 2025 09:15:06 -0700 Subject: [PATCH 138/164] incremental filter --- .../streamline__balances_native_daily_records.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql index ecfabb8d..cec90e31 100644 --- a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql +++ b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql @@ -86,12 +86,13 @@ native_records_snapshot AS ( ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE AS block_date, address FROM - {{ ref('silver__balances_validator_addresses_daily')}} + {{ ref('silver__balances_validator_addresses_daily')}} b + WHERE MIN(b.block_date) <= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE {% if is_incremental() %} - WHERE - modified_timestamp > ( + AND + CONCAT((('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE) :: STRING,'-',address) NOT IN ( SELECT - MAX(modified_timestamp) + CONCAT(block_date :: STRING,'-',address) FROM {{ this }} ) From c5bc4ea49260917443822162e99f005b911ef1dc Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 30 Dec 2025 09:46:23 -0700 Subject: [PATCH 139/164] aerodrome v1 --- .../silver_reads__aerodrome_v1_reads.sql | 65 ++++++++++ .../streamline__contract_reads_records.sql | 6 +- .../silver_tvl__aerodrome_v1_tvl.sql | 122 ++++++++++++++++++ .../silver_tvl__aerodrome_v1_tvl.yml | 4 + .../tvl/silver/silver_tvl__complete_tvl.sql | 1 + models/doc_descriptions/defi/silver_tvl.md | 7 + 6 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql create mode 100644 models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v1_tvl.sql create mode 100644 models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v1_tvl.yml diff --git a/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql new file mode 100644 index 00000000..bd766bea --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql @@ -0,0 +1,65 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'aerodrome_v1_reads_id', + tags = ['silver','contract_reads'] +) }} + +WITH verified_contracts AS ( + SELECT + DISTINCT token_address + FROM + {{ ref('price__ez_asset_metadata') }} + WHERE + is_verified + AND token_address IS NOT NULL +), +liquidity_pools AS ( + SELECT + DISTINCT + pool_address AS contract_address, + token0, + token1, + protocol, + version, + platform + FROM {{ ref('silver_dex__aerodrome_pools') }} + WHERE token0 IN (SELECT token_address FROM verified_contracts) + AND token1 IN (SELECT token_address FROM verified_contracts) + {% if is_incremental() %} + AND modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} +) +SELECT + contract_address, + NULL AS address, + 'getReserves' AS function_name, + '0x0902f1ac' AS function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input, + OBJECT_CONSTRUCT( + 'token0', token0, + 'token1', token1 + ) :: VARIANT AS metadata, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','input','platform'] + ) }} AS aerodrome_v1_reads_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM liquidity_pools diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index e3530be5..49c142fe 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -6,6 +6,7 @@ -- depends_on: {{ ref('silver_reads__lido_v1_reads') }} -- depends_on: {{ ref('silver_reads__binance_v1_reads') }} +-- depends_on: {{ ref('silver_reads__polymarket_v1_reads') }} {{ config ( materialized = "incremental", unique_key = "contract_reads_records_id", @@ -13,7 +14,9 @@ tags = ['streamline','contract_reads','records','phase_4'] ) }} -{% set models = [] %} +-- only specify chains for _reads models with hardcoded or seed driven address data +-- for dynamic models, the underlying upstream data will already be filtered or made relevant for that chain +{% set models = [] %} {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} {% set _ = models.append((ref('silver_reads__lido_v1_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__binance_v1_reads'), 'daily')) %} @@ -21,6 +24,7 @@ {% if vars.GLOBAL_PROJECT_NAME == 'polygon' %} {% set _ = models.append((ref('silver_reads__polymarket_v1_reads'), 'daily')) %} {% endif %} +{% set _ = models.append((ref('silver_reads__aerodrome_v1_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__stablecoins_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__uniswap_v2_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__uniswap_v3_reads'), 'daily')) %} diff --git a/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v1_tvl.sql b/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v1_tvl.sql new file mode 100644 index 00000000..f286662d --- /dev/null +++ b/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v1_tvl.sql @@ -0,0 +1,122 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'aerodrome_v1_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH reads AS ( + + SELECT + block_number, + block_date, + contract_address AS pool_address, + address, --NULL + regexp_substr_all(SUBSTR(result_hex, 3, len(result_hex)), '.{64}') AS segmented_data, + segmented_data [0] :: STRING AS reserve_0_hex, + segmented_data [1] :: STRING AS reserve_1_hex, + segmented_data [2] :: STRING AS block_timestamp_last_hex, + IFNULL( + CASE + WHEN LENGTH(reserve_0_hex) <= 4300 + AND reserve_0_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(reserve_0_hex) AS bigint)END, + CASE + WHEN reserve_0_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(reserve_0_hex, '0')) AS bigint) + END + ) AS reserve_0_raw, + IFNULL( + CASE + WHEN LENGTH(reserve_1_hex) <= 4300 + AND reserve_1_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(reserve_1_hex) AS bigint)END, + CASE + WHEN reserve_1_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(reserve_1_hex, '0')) AS bigint) + END + ) AS reserve_1_raw, + IFNULL( + CASE + WHEN LENGTH(block_timestamp_last_hex) <= 4300 + AND block_timestamp_last_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(block_timestamp_last_hex) AS TIMESTAMP)END, + CASE + WHEN block_timestamp_last_hex IS NOT NULL THEN TRY_CAST( + utils.udf_hex_to_int(RTRIM(block_timestamp_last_hex, '0')) AS TIMESTAMP + ) + END + ) AS block_timestamp_last_raw, + metadata :token0 :: STRING AS token_0_address, + metadata :token1 :: STRING AS token_1_address, + protocol, + version, + platform, + _inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} + WHERE + reserve_0_raw IS NOT NULL + AND reserve_1_raw IS NOT NULL + AND block_timestamp_last_raw IS NOT NULL + AND platform IN ( + SELECT DISTINCT platform + FROM {{ ref('silver_reads__aerodrome_v1_reads') }} + ) + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +all_reads AS ( +SELECT + block_number, + block_date, + token_0_address AS contract_address, + pool_address AS address, + reserve_0_hex AS amount_hex, + reserve_0_raw AS amount_raw, + protocol, + version, + platform +FROM reads +UNION ALL +SELECT + block_number, + block_date, + token_1_address AS contract_address, + pool_address AS address, + reserve_1_hex AS amount_hex, + reserve_1_raw AS amount_raw, + protocol, + version, + platform +FROM reads +) +SELECT + block_number, + block_date, + contract_address, + address, + contract_address AS token_address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS aerodrome_v1_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + all_reads qualify(ROW_NUMBER() over(PARTITION BY aerodrome_v1_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v1_tvl.yml b/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v1_tvl.yml new file mode 100644 index 00000000..b8844729 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v1_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__aerodrome_v1_tvl + description: '{{ doc("aerodrome_v1_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index 445fef15..0664f24d 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -13,6 +13,7 @@ (ref('silver_tvl__aave_v1_tvl'), 12, 'aave-v1'), (ref('silver_tvl__aave_v2_tvl'), 12, 'aave-v2'), (ref('silver_tvl__aave_v3_tvl'), 12, 'aave-v3'), + (ref('silver_tvl__aerodrome_v1_tvl'), 9, 'aerodrome-v1'), (ref('silver_tvl__binance_v1_tvl'), 12, 'binance-v1'), (ref('silver_tvl__curve_v1_tvl'), 10, 'curve-v1'), (ref('silver_tvl__etherfi_v1_tvl_agg'), 12, 'etherfi-v1'), diff --git a/models/doc_descriptions/defi/silver_tvl.md b/models/doc_descriptions/defi/silver_tvl.md index ed9c889d..20156a42 100644 --- a/models/doc_descriptions/defi/silver_tvl.md +++ b/models/doc_descriptions/defi/silver_tvl.md @@ -87,4 +87,11 @@ Sums token0 and token1 balances held by the PoolManager singleton + any associat Methodology: Sums USDC balances held by the Conditional Tokens and Collateral Tokens contracts +{% enddocs %} + +{% docs aerodrome_v1_tvl_table_doc %} + +Methodology: +Sums getReserves() values for token0 and token1 across all AMM pools + {% enddocs %} \ No newline at end of file From 2bc34254c89c9186c29f465c0babaf5a1fac6d32 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 30 Dec 2025 11:04:51 -0700 Subject: [PATCH 140/164] FR --- .../streamline__balances_native_daily_records.sql | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql index cec90e31..43d0bb83 100644 --- a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql +++ b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql @@ -80,6 +80,7 @@ native_records_snapshot AS ( tx_fees WHERE block_timestamp :: DATE <= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE + {% if not is_incremental() %} UNION SELECT DISTINCT @@ -88,14 +89,6 @@ native_records_snapshot AS ( FROM {{ ref('silver__balances_validator_addresses_daily')}} b WHERE MIN(b.block_date) <= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE - {% if is_incremental() %} - AND - CONCAT((('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE) :: STRING,'-',address) NOT IN ( - SELECT - CONCAT(block_date :: STRING,'-',address) - FROM - {{ this }} - ) {% endif %} ), native_records_history AS ( From 1b956ebeb87b2b9e9eb5e65c349531f3f6242a1f Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 30 Dec 2025 11:47:42 -0700 Subject: [PATCH 141/164] aerodrome slipstream v2 --- .../silver_reads__aerodrome_v2_reads.sql | 93 +++++++++++++++++++ .../streamline__contract_reads_records.sql | 1 + .../silver_tvl__aerodrome_v2_tvl.sql | 74 +++++++++++++++ .../silver_tvl__aerodrome_v2_tvl.yml | 4 + .../tvl/silver/silver_tvl__complete_tvl.sql | 1 + models/doc_descriptions/defi/silver_tvl.md | 7 ++ 6 files changed, 180 insertions(+) create mode 100644 models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v2_reads.sql create mode 100644 models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.sql create mode 100644 models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.yml diff --git a/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v2_reads.sql new file mode 100644 index 00000000..cefa4982 --- /dev/null +++ b/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v2_reads.sql @@ -0,0 +1,93 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'aerodrome_v2_reads_id', + tags = ['silver','contract_reads'] +) }} + +WITH verified_contracts AS ( + SELECT + DISTINCT token_address + FROM + {{ ref('price__ez_asset_metadata') }} + WHERE + is_verified + AND token_address IS NOT NULL +), +-- Blacklisted tokens from DefiLlama adapter +blacklisted_tokens AS ( + SELECT LOWER('0xdbfefd2e8460a6ee4955a68582f85708baea60a3') AS token_address -- superOETHb: excluded to avoid double-counting with Origin Protocol TVL (their Aerodrome AMO position) +), +liquidity_pools AS ( + SELECT + DISTINCT + pool_address, + token0_address AS token0, + token1_address AS token1, + protocol, + version, + platform + FROM {{ ref('silver_dex__superchain_slipstream_pools') }} + WHERE protocol = 'aerodrome' + AND token0 IN (SELECT token_address FROM verified_contracts) + AND token1 IN (SELECT token_address FROM verified_contracts) + AND token0 NOT IN (SELECT token_address FROM blacklisted_tokens) + AND token1 NOT IN (SELECT token_address FROM blacklisted_tokens) + {% if is_incremental() %} + AND modified_timestamp > ( + SELECT MAX(modified_timestamp) + FROM {{ this }} + ) + {% endif %} +), +lp_balances AS ( + SELECT + token0 AS contract_address, + pool_address AS address, + token0, + token1, + protocol, + version, + platform + FROM liquidity_pools + UNION + SELECT + token1 AS contract_address, + pool_address AS address, + token0, + token1, + protocol, + version, + platform + FROM liquidity_pools +) + +SELECT + contract_address, + address, + 'balanceOf' AS function_name, + '0x70a08231' AS function_sig, + CONCAT( + '0x70a08231', + LPAD(SUBSTR(address, 3), 64, '0') + ) AS input, + OBJECT_CONSTRUCT( + 'token0', token0, + 'token1', token1 + ) :: VARIANT AS metadata, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['contract_address','input','platform'] + ) }} AS aerodrome_v2_reads_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM lp_balances diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index 49c142fe..87cb9895 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -25,6 +25,7 @@ {% set _ = models.append((ref('silver_reads__polymarket_v1_reads'), 'daily')) %} {% endif %} {% set _ = models.append((ref('silver_reads__aerodrome_v1_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__aerodrome_v2_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__stablecoins_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__uniswap_v2_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__uniswap_v3_reads'), 'daily')) %} diff --git a/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.sql b/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.sql new file mode 100644 index 00000000..b1526056 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.sql @@ -0,0 +1,74 @@ +{# Get variables #} +{% set vars = return_vars() %} + +{# Log configuration details #} +{{ log_model_details() }} + +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'aerodrome_v2_tvl_id', + tags = ['silver','defi','tvl','curated_daily'] +) }} + +WITH reads AS ( + + SELECT + block_number, + block_date, + contract_address, + address, + result_hex AS amount_hex, + IFNULL( + CASE + WHEN LENGTH(amount_hex) <= 4300 + AND amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(amount_hex) AS bigint)END, + CASE + WHEN amount_hex IS NOT NULL THEN TRY_CAST(utils.udf_hex_to_int(RTRIM(amount_hex, '0')) AS bigint) + END + ) AS amount_raw, + metadata :token0 :: STRING AS token_0_address, + metadata :token1 :: STRING AS token_1_address, + protocol, + version, + platform, + _inserted_timestamp + FROM + {{ ref('silver__contract_reads') }} + WHERE + amount_raw IS NOT NULL + AND platform IN ( + SELECT DISTINCT platform + FROM {{ ref('silver_reads__aerodrome_v2_reads') }} + ) + +{% if is_incremental() %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_date, + contract_address, + address, + contract_address AS token_address, + amount_hex, + amount_raw, + protocol, + version, + platform, + {{ dbt_utils.generate_surrogate_key( + ['block_date','contract_address','address','platform'] + ) }} AS aerodrome_v2_tvl_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + reads qualify(ROW_NUMBER() over(PARTITION BY aerodrome_v2_tvl_id +ORDER BY + modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.yml b/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.yml new file mode 100644 index 00000000..4cb22766 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__aerodrome_v2_tvl + description: '{{ doc("aerodrome_v2_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index 0664f24d..42dd24a2 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -14,6 +14,7 @@ (ref('silver_tvl__aave_v2_tvl'), 12, 'aave-v2'), (ref('silver_tvl__aave_v3_tvl'), 12, 'aave-v3'), (ref('silver_tvl__aerodrome_v1_tvl'), 9, 'aerodrome-v1'), + (ref('silver_tvl__aerodrome_v2_tvl'), 9, 'aerodrome-v2'), (ref('silver_tvl__binance_v1_tvl'), 12, 'binance-v1'), (ref('silver_tvl__curve_v1_tvl'), 10, 'curve-v1'), (ref('silver_tvl__etherfi_v1_tvl_agg'), 12, 'etherfi-v1'), diff --git a/models/doc_descriptions/defi/silver_tvl.md b/models/doc_descriptions/defi/silver_tvl.md index 20156a42..128068b9 100644 --- a/models/doc_descriptions/defi/silver_tvl.md +++ b/models/doc_descriptions/defi/silver_tvl.md @@ -94,4 +94,11 @@ Sums USDC balances held by the Conditional Tokens and Collateral Tokens contract Methodology: Sums getReserves() values for token0 and token1 across all AMM pools +{% enddocs %} + +{% docs aerodrome_v2_tvl_table_doc %} + +Methodology: +Sums token0 and token1 balances held by each Slipstream (CL) pool contract via balanceOf() + {% enddocs %} \ No newline at end of file From affeeb29714bc2953c8ad487fc6b1501963e10c0 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 30 Dec 2025 15:00:04 -0700 Subject: [PATCH 142/164] comma --- .../records/streamline__balances_native_daily_records.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql index 43d0bb83..eb97e6d1 100644 --- a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql +++ b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql @@ -137,7 +137,7 @@ native_records_history AS ( {{ this }} ) {% endif %} -) +), all_records AS ( SELECT * FROM native_records_snapshot UNION From 87e716b4bcc5a18319f94278c9dc7321595d152e Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 30 Dec 2025 15:03:58 -0700 Subject: [PATCH 143/164] group by --- .../records/streamline__balances_native_daily_records.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql index eb97e6d1..d8fb71c6 100644 --- a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql +++ b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql @@ -88,7 +88,9 @@ native_records_snapshot AS ( address FROM {{ ref('silver__balances_validator_addresses_daily')}} b - WHERE MIN(b.block_date) <= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE + GROUP BY + address + HAVING MIN(b.block_date) <= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE {% endif %} ), native_records_history AS ( From fbe1e45cad2c02b6d54246ea71a2ff5b5e043df6 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 30 Dec 2025 15:50:18 -0700 Subject: [PATCH 144/164] update tag --- .../validators/silver__balances_validator_addresses_daily.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql b/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql index fe923dad..4f504d1a 100644 --- a/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql +++ b/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql @@ -7,7 +7,7 @@ materialized = "incremental", unique_key = "balances_validator_addresses_daily_id", cluster_by = "block_date", - tags = ['silver','balances','records','native','phase_4'] + tags = ['silver','streamline','balances','records','native','phase_4'] ) }} WITH miner_addresses AS ( From c5f6b09a57e07139c08d8f2ee85b759fa01c0f8b Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 31 Dec 2025 10:22:52 -0700 Subject: [PATCH 145/164] remove verified token filters on aerodrome --- .../silver_reads__aerodrome_v1_reads.sql | 15 ++------------- ...reads__superchain_slipstream_v1_reads.sql} | 19 +++---------------- 2 files changed, 5 insertions(+), 29 deletions(-) rename models/curated_package/contract_reads/streamline/records/{aerodrome/silver_reads__aerodrome_v2_reads.sql => superchain/silver_reads__superchain_slipstream_v1_reads.sql} (77%) diff --git a/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql index bd766bea..bed2aade 100644 --- a/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql @@ -11,16 +11,7 @@ tags = ['silver','contract_reads'] ) }} -WITH verified_contracts AS ( - SELECT - DISTINCT token_address - FROM - {{ ref('price__ez_asset_metadata') }} - WHERE - is_verified - AND token_address IS NOT NULL -), -liquidity_pools AS ( +WITH liquidity_pools AS ( SELECT DISTINCT pool_address AS contract_address, @@ -30,10 +21,8 @@ liquidity_pools AS ( version, platform FROM {{ ref('silver_dex__aerodrome_pools') }} - WHERE token0 IN (SELECT token_address FROM verified_contracts) - AND token1 IN (SELECT token_address FROM verified_contracts) {% if is_incremental() %} - AND modified_timestamp > ( + WHERE modified_timestamp > ( SELECT MAX(modified_timestamp) FROM {{ this }} ) diff --git a/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/superchain/silver_reads__superchain_slipstream_v1_reads.sql similarity index 77% rename from models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v2_reads.sql rename to models/curated_package/contract_reads/streamline/records/superchain/silver_reads__superchain_slipstream_v1_reads.sql index cefa4982..860806ba 100644 --- a/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/superchain/silver_reads__superchain_slipstream_v1_reads.sql @@ -11,18 +11,8 @@ tags = ['silver','contract_reads'] ) }} -WITH verified_contracts AS ( - SELECT - DISTINCT token_address - FROM - {{ ref('price__ez_asset_metadata') }} - WHERE - is_verified - AND token_address IS NOT NULL -), --- Blacklisted tokens from DefiLlama adapter -blacklisted_tokens AS ( - SELECT LOWER('0xdbfefd2e8460a6ee4955a68582f85708baea60a3') AS token_address -- superOETHb: excluded to avoid double-counting with Origin Protocol TVL (their Aerodrome AMO position) +WITH blacklisted_tokens AS ( + SELECT LOWER('0xdbfefd2e8460a6ee4955a68582f85708baea60a3') AS token_address -- superOETHb: excluded to avoid double-counting with Origin Protocol TVL (Aerodrome AMO position) ), liquidity_pools AS ( SELECT @@ -34,10 +24,7 @@ liquidity_pools AS ( version, platform FROM {{ ref('silver_dex__superchain_slipstream_pools') }} - WHERE protocol = 'aerodrome' - AND token0 IN (SELECT token_address FROM verified_contracts) - AND token1 IN (SELECT token_address FROM verified_contracts) - AND token0 NOT IN (SELECT token_address FROM blacklisted_tokens) + WHERE token0 NOT IN (SELECT token_address FROM blacklisted_tokens) AND token1 NOT IN (SELECT token_address FROM blacklisted_tokens) {% if is_incremental() %} AND modified_timestamp > ( From ff564200c9414b7e55401238f68c789e168893a0 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 31 Dec 2025 10:25:54 -0700 Subject: [PATCH 146/164] naming --- .../records/streamline__contract_reads_records.sql | 2 +- .../silver_reads__superchain_slipstream_v1_reads.sql | 4 ++-- .../tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.yml | 4 ---- .../defi/tvl/silver/silver_tvl__complete_tvl.sql | 2 +- .../silver_tvl__superchain_slipstream_v1_tvl.sql} | 8 ++++---- .../silver_tvl__superchain_slipstream_v1_tvl.yml | 4 ++++ models/doc_descriptions/defi/silver_tvl.md | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) delete mode 100644 models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.yml rename models/curated_package/defi/tvl/silver/{aerodrome/silver_tvl__aerodrome_v2_tvl.sql => superchain/silver_tvl__superchain_slipstream_v1_tvl.sql} (87%) create mode 100644 models/curated_package/defi/tvl/silver/superchain/silver_tvl__superchain_slipstream_v1_tvl.yml diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index 87cb9895..c838bd72 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -25,7 +25,7 @@ {% set _ = models.append((ref('silver_reads__polymarket_v1_reads'), 'daily')) %} {% endif %} {% set _ = models.append((ref('silver_reads__aerodrome_v1_reads'), 'daily')) %} -{% set _ = models.append((ref('silver_reads__aerodrome_v2_reads'), 'daily')) %} +{% set _ = models.append((ref('silver_reads__superchain_slipstream_v1_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__stablecoins_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__uniswap_v2_reads'), 'daily')) %} {% set _ = models.append((ref('silver_reads__uniswap_v3_reads'), 'daily')) %} diff --git a/models/curated_package/contract_reads/streamline/records/superchain/silver_reads__superchain_slipstream_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/superchain/silver_reads__superchain_slipstream_v1_reads.sql index 860806ba..05466ffa 100644 --- a/models/curated_package/contract_reads/streamline/records/superchain/silver_reads__superchain_slipstream_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/superchain/silver_reads__superchain_slipstream_v1_reads.sql @@ -7,7 +7,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - unique_key = 'aerodrome_v2_reads_id', + unique_key = 'superchain_slipstream_v1_reads_id', tags = ['silver','contract_reads'] ) }} @@ -73,7 +73,7 @@ SELECT platform, {{ dbt_utils.generate_surrogate_key( ['contract_address','input','platform'] - ) }} AS aerodrome_v2_reads_id, + ) }} AS superchain_slipstream_v1_reads_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.yml b/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.yml deleted file mode 100644 index 4cb22766..00000000 --- a/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.yml +++ /dev/null @@ -1,4 +0,0 @@ -version: 2 -models: - - name: silver_tvl__aerodrome_v2_tvl - description: '{{ doc("aerodrome_v2_tvl_table_doc") }}' diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index 42dd24a2..10d9ee0b 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -14,7 +14,7 @@ (ref('silver_tvl__aave_v2_tvl'), 12, 'aave-v2'), (ref('silver_tvl__aave_v3_tvl'), 12, 'aave-v3'), (ref('silver_tvl__aerodrome_v1_tvl'), 9, 'aerodrome-v1'), - (ref('silver_tvl__aerodrome_v2_tvl'), 9, 'aerodrome-v2'), + (ref('silver_tvl__superchain_slipstream_v1_tvl'), 9, 'superchain-slipstream-v1'), (ref('silver_tvl__binance_v1_tvl'), 12, 'binance-v1'), (ref('silver_tvl__curve_v1_tvl'), 10, 'curve-v1'), (ref('silver_tvl__etherfi_v1_tvl_agg'), 12, 'etherfi-v1'), diff --git a/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.sql b/models/curated_package/defi/tvl/silver/superchain/silver_tvl__superchain_slipstream_v1_tvl.sql similarity index 87% rename from models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.sql rename to models/curated_package/defi/tvl/silver/superchain/silver_tvl__superchain_slipstream_v1_tvl.sql index b1526056..ef12607c 100644 --- a/models/curated_package/defi/tvl/silver/aerodrome/silver_tvl__aerodrome_v2_tvl.sql +++ b/models/curated_package/defi/tvl/silver/superchain/silver_tvl__superchain_slipstream_v1_tvl.sql @@ -7,7 +7,7 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - unique_key = 'aerodrome_v2_tvl_id', + unique_key = 'superchain_slipstream_v1_tvl_id', tags = ['silver','defi','tvl','curated_daily'] ) }} @@ -39,7 +39,7 @@ WITH reads AS ( amount_raw IS NOT NULL AND platform IN ( SELECT DISTINCT platform - FROM {{ ref('silver_reads__aerodrome_v2_reads') }} + FROM {{ ref('silver_reads__superchain_slipstream_v1_reads') }} ) {% if is_incremental() %} @@ -64,11 +64,11 @@ SELECT platform, {{ dbt_utils.generate_surrogate_key( ['block_date','contract_address','address','platform'] - ) }} AS aerodrome_v2_tvl_id, + ) }} AS superchain_slipstream_v1_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - reads qualify(ROW_NUMBER() over(PARTITION BY aerodrome_v2_tvl_id + reads qualify(ROW_NUMBER() over(PARTITION BY superchain_slipstream_v1_tvl_id ORDER BY modified_timestamp DESC)) = 1 diff --git a/models/curated_package/defi/tvl/silver/superchain/silver_tvl__superchain_slipstream_v1_tvl.yml b/models/curated_package/defi/tvl/silver/superchain/silver_tvl__superchain_slipstream_v1_tvl.yml new file mode 100644 index 00000000..54159579 --- /dev/null +++ b/models/curated_package/defi/tvl/silver/superchain/silver_tvl__superchain_slipstream_v1_tvl.yml @@ -0,0 +1,4 @@ +version: 2 +models: + - name: silver_tvl__superchain_slipstream_v1_tvl + description: '{{ doc("superchain_slipstream_v1_tvl_table_doc") }}' diff --git a/models/doc_descriptions/defi/silver_tvl.md b/models/doc_descriptions/defi/silver_tvl.md index 128068b9..d8366769 100644 --- a/models/doc_descriptions/defi/silver_tvl.md +++ b/models/doc_descriptions/defi/silver_tvl.md @@ -96,7 +96,7 @@ Sums getReserves() values for token0 and token1 across all AMM pools {% enddocs %} -{% docs aerodrome_v2_tvl_table_doc %} +{% docs superchain_slipstream_v1_tvl_table_doc %} Methodology: Sums token0 and token1 balances held by each Slipstream (CL) pool contract via balanceOf() From d7d888398ebb8d6b5378c607ab6bb0c64c8ed3d4 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 31 Dec 2025 13:08:45 -0700 Subject: [PATCH 147/164] decimals --- .../defi/tvl/silver/silver_tvl__complete_tvl.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index 10d9ee0b..ad857420 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -13,8 +13,8 @@ (ref('silver_tvl__aave_v1_tvl'), 12, 'aave-v1'), (ref('silver_tvl__aave_v2_tvl'), 12, 'aave-v2'), (ref('silver_tvl__aave_v3_tvl'), 12, 'aave-v3'), - (ref('silver_tvl__aerodrome_v1_tvl'), 9, 'aerodrome-v1'), - (ref('silver_tvl__superchain_slipstream_v1_tvl'), 9, 'superchain-slipstream-v1'), + (ref('silver_tvl__aerodrome_v1_tvl'), 12, 'aerodrome-v1'), + (ref('silver_tvl__superchain_slipstream_v1_tvl'), 12, 'superchain-slipstream-v1'), (ref('silver_tvl__binance_v1_tvl'), 12, 'binance-v1'), (ref('silver_tvl__curve_v1_tvl'), 10, 'curve-v1'), (ref('silver_tvl__etherfi_v1_tvl_agg'), 12, 'etherfi-v1'), From 4794a393cc5a861d52634d4292e532143dee8c51 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 5 Jan 2026 09:33:06 -0700 Subject: [PATCH 148/164] contract reads FR filter --- .../streamline/complete/streamline__contract_reads_complete.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql index 452c7fa3..f254fe91 100644 --- a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql +++ b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql @@ -53,6 +53,7 @@ WHERE {{ this }}) {% else %} {{ ref('bronze__contract_reads_fr') }} + WHERE platform IS NOT NULL {% endif %} qualify(ROW_NUMBER() over (PARTITION BY contract_reads_complete_id From a52f95899a75fef2f588f9e79a17351654d8d65e Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 5 Jan 2026 10:08:39 -0700 Subject: [PATCH 149/164] newly verified/unverify uni pools --- .../unverify_contract_reads.sql | 19 +++++++++++++++++++ .../streamline__contract_reads_records.sql | 5 +++-- .../silver_reads__uniswap_v2_reads.sql | 10 ++++++---- .../silver_reads__uniswap_v3_reads.sql | 10 ++++++---- .../silver_reads__uniswap_v4_reads.sql | 16 ++++++++-------- 5 files changed, 42 insertions(+), 18 deletions(-) create mode 100644 macros/curated_package/unverify_contract_reads.sql diff --git a/macros/curated_package/unverify_contract_reads.sql b/macros/curated_package/unverify_contract_reads.sql new file mode 100644 index 00000000..44eab415 --- /dev/null +++ b/macros/curated_package/unverify_contract_reads.sql @@ -0,0 +1,19 @@ +{% macro unverify_contract_reads() %} + {% if var('HEAL_MODEL', false) and is_incremental() %} + DELETE FROM {{ this }} + WHERE protocol = 'uniswap' + AND ( + metadata:token0::STRING NOT IN ( + SELECT token_address + FROM {{ ref('price__ez_asset_metadata') }} + WHERE is_verified AND token_address IS NOT NULL + ) + OR metadata:token1::STRING NOT IN ( + SELECT token_address + FROM {{ ref('price__ez_asset_metadata') }} + WHERE is_verified AND token_address IS NOT NULL + ) + ); + {% endif %} +{% endmacro %} + diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index c838bd72..6673cc9f 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -11,10 +11,11 @@ materialized = "incremental", unique_key = "contract_reads_records_id", full_refresh = vars.GLOBAL_STREAMLINE_FR_ENABLED, - tags = ['streamline','contract_reads','records','phase_4'] + post_hook = '{{ unverify_contract_reads() }}', + tags = ['streamline','contract_reads','records','heal','phase_4'] ) }} --- only specify chains for _reads models with hardcoded or seed driven address data +-- only specify chains/exclusions for _reads models with hardcoded or seed driven address data -- for dynamic models, the underlying upstream data will already be filtered or made relevant for that chain {% set models = [] %} {% if vars.GLOBAL_PROJECT_NAME == 'ethereum' %} diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql index a4bb897c..254c848d 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql @@ -8,7 +8,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'uniswap_v2_reads_id', - tags = ['silver','contract_reads'] + post_hook = '{{ unverify_contract_reads() }}', + tags = ['silver','contract_reads','heal'] ) }} WITH verified_contracts AS ( @@ -33,9 +34,10 @@ liquidity_pools AS ( WHERE token0 IN (SELECT token_address FROM verified_contracts) AND token1 IN (SELECT token_address FROM verified_contracts) {% if is_incremental() %} - AND modified_timestamp > ( - SELECT MAX(modified_timestamp) - FROM {{ this }} + AND ( + modified_timestamp > (SELECT MAX(modified_timestamp) FROM {{ this }}) + OR pool_address NOT IN (SELECT contract_address FROM {{ this }}) + -- pull in pools with newly verified tokens ) {% endif %} ) diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql index a268133b..add13de3 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql @@ -8,7 +8,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'uniswap_v3_reads_id', - tags = ['silver','contract_reads'] + post_hook = '{{ unverify_contract_reads() }}', + tags = ['silver','contract_reads','heal'] ) }} WITH verified_contracts AS ( @@ -33,9 +34,10 @@ liquidity_pools AS ( WHERE token0 IN (SELECT token_address FROM verified_contracts) AND token1 IN (SELECT token_address FROM verified_contracts) {% if is_incremental() %} - AND modified_timestamp > ( - SELECT MAX(modified_timestamp) - FROM {{ this }} + AND ( + modified_timestamp > (SELECT MAX(modified_timestamp) FROM {{ this }}) + OR pool_address NOT IN (SELECT contract_address FROM {{ this }}) + -- pull in pools with newly verified tokens ) {% endif %} ), diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql index 0a1a128f..a7a09dad 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql @@ -8,7 +8,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'uniswap_v4_reads_id', - tags = ['silver','contract_reads'] + post_hook = '{{ unverify_contract_reads() }}', + tags = ['silver','contract_reads','heal'] ) }} WITH verified_contracts AS ( @@ -34,14 +35,13 @@ liquidity_pools AS ( {{ ref('silver_dex__uniswap_v4_pools') }} WHERE token0 IN (SELECT token_address FROM verified_contracts) AND token1 IN (SELECT token_address FROM verified_contracts) -{% if is_incremental() %} -AND modified_timestamp > ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} + {% if is_incremental() %} + AND ( + modified_timestamp > (SELECT MAX(modified_timestamp) FROM {{ this }}) + OR pool_address NOT IN (SELECT contract_address FROM {{ this }}) + -- pull in pools with newly verified tokens ) -{% endif %} + {% endif %} ), all_balances AS ( SELECT From ce675cab4a7fb525ecdd36cc432b108b64d9e314 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 5 Jan 2026 10:15:42 -0700 Subject: [PATCH 150/164] platform filter --- .../contract_reads/silver/silver__contract_reads.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/curated_package/contract_reads/silver/silver__contract_reads.sql b/models/curated_package/contract_reads/silver/silver__contract_reads.sql index 162615ff..462622d1 100644 --- a/models/curated_package/contract_reads/silver/silver__contract_reads.sql +++ b/models/curated_package/contract_reads/silver/silver__contract_reads.sql @@ -53,6 +53,7 @@ WHERE {{ ref('bronze__contract_reads_fr') }} WHERE DATA :result :: STRING <> '0x' + AND platform IS NOT NULL {% endif %} qualify(ROW_NUMBER() over (PARTITION BY contract_reads_id From 19d90c3959a281dc2dbe6c78c1134be87d5efea2 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 5 Jan 2026 11:53:23 -0700 Subject: [PATCH 151/164] unverify tvl --- .../unverify_contract_reads.sql | 3 +- macros/curated_package/unverify_tvl.sql | 28 +++++++++++++++++++ .../silver_reads__uniswap_v2_reads.sql | 3 +- .../silver_reads__uniswap_v3_reads.sql | 3 +- .../silver_reads__uniswap_v4_reads.sql | 8 ++++-- .../tvl/silver/silver_tvl__complete_tvl.sql | 1 + .../uniswap/silver_tvl__uniswap_v2_tvl.sql | 3 +- .../uniswap/silver_tvl__uniswap_v3_tvl.sql | 3 +- .../uniswap/silver_tvl__uniswap_v4_tvl.sql | 3 +- 9 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 macros/curated_package/unverify_tvl.sql diff --git a/macros/curated_package/unverify_contract_reads.sql b/macros/curated_package/unverify_contract_reads.sql index 44eab415..7c718424 100644 --- a/macros/curated_package/unverify_contract_reads.sql +++ b/macros/curated_package/unverify_contract_reads.sql @@ -1,7 +1,7 @@ {% macro unverify_contract_reads() %} {% if var('HEAL_MODEL', false) and is_incremental() %} DELETE FROM {{ this }} - WHERE protocol = 'uniswap' + WHERE metadata:verified_check_enabled::STRING = 'true' AND ( metadata:token0::STRING NOT IN ( SELECT token_address @@ -16,4 +16,3 @@ ); {% endif %} {% endmacro %} - diff --git a/macros/curated_package/unverify_tvl.sql b/macros/curated_package/unverify_tvl.sql new file mode 100644 index 00000000..747b0174 --- /dev/null +++ b/macros/curated_package/unverify_tvl.sql @@ -0,0 +1,28 @@ +{% macro unverify_tvl() %} + {% if var('HEAL_MODEL', false) and is_incremental() %} + -- Only target platforms that use verified_check_enabled + -- Delete rows that are invalid in BOTH v2 and v3/v4 styles + DELETE FROM {{ this }} + WHERE platform IN ( + SELECT DISTINCT platform + FROM {{ ref('streamline__contract_reads_records') }} + WHERE metadata:verified_check_enabled::STRING = 'true' + ) --necessary for the complete_tvl model + AND + -- Not valid in v2 style (address as pool) + address NOT IN ( + SELECT DISTINCT contract_address + FROM {{ ref('streamline__contract_reads_records') }} + WHERE metadata:verified_check_enabled::STRING = 'true' + AND address IS NULL + ) + AND + -- Not valid in v3/v4 style (token-address combo) + CONCAT(contract_address, '-', address) NOT IN ( + SELECT DISTINCT CONCAT(contract_address, '-', address) + FROM {{ ref('streamline__contract_reads_records') }} + WHERE metadata:verified_check_enabled::STRING = 'true' + AND address IS NOT NULL + ); + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql index 254c848d..cc5810ce 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql @@ -53,7 +53,8 @@ SELECT ) AS input, OBJECT_CONSTRUCT( 'token0', token0, - 'token1', token1 + 'token1', token1, + 'verified_check_enabled','true' ) :: VARIANT AS metadata, protocol, version, diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql index add13de3..e3357baa 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql @@ -74,7 +74,8 @@ SELECT ) AS input, OBJECT_CONSTRUCT( 'token0', token0, - 'token1', token1 + 'token1', token1, + 'verified_check_enabled','true' ) :: VARIANT AS metadata, protocol, version, diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql index a7a09dad..c7273fd5 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql @@ -38,7 +38,10 @@ liquidity_pools AS ( {% if is_incremental() %} AND ( modified_timestamp > (SELECT MAX(modified_timestamp) FROM {{ this }}) - OR pool_address NOT IN (SELECT contract_address FROM {{ this }}) + OR CONCAT(token0,'-',token1) NOT IN ( + SELECT CONCAT(metadata:token0::STRING,'-',metadata:token1::STRING) + FROM {{ this }} + ) -- pull in pools with newly verified tokens ) {% endif %} @@ -129,7 +132,8 @@ SELECT 'hook_address', hook_address, 'address_type', - address_type + address_type, + 'verified_check_enabled','true' ) :: variant AS metadata, protocol, version, diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index ad857420..159a6f92 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -6,6 +6,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'complete_tvl_id', + post_hook = '{{ unverify_tvl() }}', tags = ['silver','defi','tvl','complete','heal','curated_daily'] ) }} diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql index 94dd7576..cb1804a9 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql @@ -8,7 +8,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'uniswap_v2_tvl_id', - tags = ['silver','defi','tvl','curated_daily'] + post_hook = '{{ unverify_tvl() }}', + tags = ['silver','defi','tvl','heal','curated_daily'] ) }} WITH reads AS ( diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql index 641bef33..b61b971d 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql @@ -8,7 +8,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'uniswap_v3_tvl_id', - tags = ['silver','defi','tvl','curated_daily'] + post_hook = '{{ unverify_tvl() }}', + tags = ['silver','defi','tvl','heal','curated_daily'] ) }} WITH reads AS ( diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql index a0320b29..988476bf 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql @@ -8,7 +8,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'uniswap_v4_tvl_id', - tags = ['silver','defi','tvl','curated_daily'] + post_hook = '{{ unverify_tvl() }}', + tags = ['silver','defi','tvl','heal','curated_daily'] ) }} WITH reads AS ( From 90efbb7ecd0fed07084393da883bf8974ad5aca1 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 5 Jan 2026 11:58:59 -0700 Subject: [PATCH 152/164] view --- .../defi/tvl/gold/defi__ez_protocol_tvl.sql | 68 +++---------------- 1 file changed, 10 insertions(+), 58 deletions(-) diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql index 656f7431..c61b83c4 100644 --- a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql @@ -3,78 +3,30 @@ {# Log configuration details #} {{ log_model_details() }} {{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'ez_protocol_tvl_id', + materialized = 'view', persist_docs ={ "relation": true, "columns": true }, tags = ['gold','defi','tvl','curated_daily'] ) }} -WITH complete_tvl AS ( - - SELECT - block_number, - block_date, - contract_address, - address, - token_address, - decimals, - symbol, - is_verified, - max_usd_exponent, - amount_hex, - amount_raw, - amount_precise, - amount, - CASE - WHEN t.amount_usd < POWER(10, t.max_usd_exponent) THEN t.amount_usd - ELSE NULL - END AS amount_usd, - protocol, - version, - platform - FROM - {{ ref('silver_tvl__complete_tvl') }} t - -{% if is_incremental() %} -WHERE - CONCAT( - block_date, - '-', - platform - ) IN ( - SELECT - DISTINCT CONCAT( - block_date, - '-', - platform - ) - FROM - {{ ref('silver_tvl__complete_tvl') }} - WHERE - modified_timestamp > ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - ) -{% endif %} -) SELECT block_date, - SUM(COALESCE(amount_usd, 0)) AS tvl_usd, + SUM( + CASE + WHEN COALESCE(amount_usd, 0) < POWER(10, max_usd_exponent) THEN COALESCE(amount_usd, 0) + ELSE 0 + END + ) AS tvl_usd, protocol, version, platform, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, + inserted_timestamp, + modified_timestamp, {{ dbt_utils.generate_surrogate_key( ['block_date','platform'] ) }} AS ez_protocol_tvl_id FROM - complete_tvl + {{ ref('silver_tvl__complete_tvl') }} GROUP BY block_date, protocol, From d5c6f6e8fa8b5aa016b247276b1926e5d6a7692d Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 5 Jan 2026 12:04:57 -0700 Subject: [PATCH 153/164] group by all --- .../curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql index c61b83c4..08f757ac 100644 --- a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql @@ -28,7 +28,4 @@ SELECT FROM {{ ref('silver_tvl__complete_tvl') }} GROUP BY - block_date, - protocol, - version, - platform + ALL From 440b27079cce12af02d24dc05d16d475b9e2870f Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 5 Jan 2026 12:13:50 -0700 Subject: [PATCH 154/164] docs --- models/doc_descriptions/defi/ez_protocol_tvl.md | 4 ++++ models/doc_descriptions/defi/silver_tvl.md | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/models/doc_descriptions/defi/ez_protocol_tvl.md b/models/doc_descriptions/defi/ez_protocol_tvl.md index 8a8a6e88..bdc28496 100644 --- a/models/doc_descriptions/defi/ez_protocol_tvl.md +++ b/models/doc_descriptions/defi/ez_protocol_tvl.md @@ -4,6 +4,10 @@ This table provides daily Total Value Locked (TVL) metrics for DeFi protocols across EVM-compatible blockchains. It aggregates values by platform to track liquidity and value deposited in various DeFi applications. Methods may vary by protocol and blockchain. +## Methodology Note + +For certain protocols (e.g., Uniswap v2/v3/v4 and forks), TVL is calculated only for pools where both tokens are verified. This filtering removes low-quality or spam pools, providing a more accurate representation of "real" TVL rather than values inflated by low-liquidity tokens. + ## Key Use Cases - Tracking protocol TVL growth and trends over time diff --git a/models/doc_descriptions/defi/silver_tvl.md b/models/doc_descriptions/defi/silver_tvl.md index d8366769..77633f3c 100644 --- a/models/doc_descriptions/defi/silver_tvl.md +++ b/models/doc_descriptions/defi/silver_tvl.md @@ -64,21 +64,21 @@ Sums native ETH balance across all pool contracts, doubled to represent total po {% docs uniswap_v2_tvl_table_doc %} Methodology: -Sums getReserves() values for token0 and token1 across all pools +Sums getReserves() values for token0 and token1 across all pools where both tokens are verified. This filtering removes spam/low-liquidity pools to provide a more accurate TVL value. {% enddocs %} {% docs uniswap_v3_tvl_table_doc %} Methodology: -Sums token0 and token1 balances held by each pool contract +Sums token0 and token1 balances held by each pool contract where both tokens are verified. This filtering removes spam/low-liquidity pools to provide a more accurate TVL value. {% enddocs %} {% docs uniswap_v4_tvl_table_doc %} Methodology: -Sums token0 and token1 balances held by the PoolManager singleton + any associated hook contracts +Sums token0 and token1 balances held by the PoolManager singleton + any associated hook contracts where both tokens are verified. This filtering removes spam/low-liquidity pools to provide a more accurate TVL value. {% enddocs %} From 13a658a6720f467a79fad1c518aa544393216714 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 5 Jan 2026 14:59:55 -0700 Subject: [PATCH 155/164] platform filter --- .../contract_reads/silver/silver__contract_reads.sql | 1 + .../streamline/complete/streamline__contract_reads_complete.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/models/curated_package/contract_reads/silver/silver__contract_reads.sql b/models/curated_package/contract_reads/silver/silver__contract_reads.sql index 462622d1..7661e3a2 100644 --- a/models/curated_package/contract_reads/silver/silver__contract_reads.sql +++ b/models/curated_package/contract_reads/silver/silver__contract_reads.sql @@ -49,6 +49,7 @@ WHERE FROM {{ this }}) AND DATA :result :: STRING <> '0x' + AND platform IS NOT NULL {% else %} {{ ref('bronze__contract_reads_fr') }} WHERE diff --git a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql index f254fe91..bb6ef6cc 100644 --- a/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql +++ b/models/curated_package/contract_reads/streamline/complete/streamline__contract_reads_complete.sql @@ -51,6 +51,7 @@ WHERE COALESCE (MAX(_inserted_timestamp), '1970-01-01' :: TIMESTAMP) FROM {{ this }}) + AND platform IS NOT NULL {% else %} {{ ref('bronze__contract_reads_fr') }} WHERE platform IS NOT NULL From ac487d2bac4c401b2ca0162d538edcaf3c3914f0 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 6 Jan 2026 10:34:55 -0700 Subject: [PATCH 156/164] delete macro updates --- .../unverify_contract_reads.sql | 30 ++++++++++++------- .../uniswap/silver_tvl__uniswap_v2_tvl.sql | 1 + .../uniswap/silver_tvl__uniswap_v3_tvl.sql | 1 + .../uniswap/silver_tvl__uniswap_v4_tvl.sql | 1 + 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/macros/curated_package/unverify_contract_reads.sql b/macros/curated_package/unverify_contract_reads.sql index 7c718424..7ad3a4f7 100644 --- a/macros/curated_package/unverify_contract_reads.sql +++ b/macros/curated_package/unverify_contract_reads.sql @@ -1,17 +1,27 @@ {% macro unverify_contract_reads() %} {% if var('HEAL_MODEL', false) and is_incremental() %} - DELETE FROM {{ this }} - WHERE metadata:verified_check_enabled::STRING = 'true' + DELETE FROM {{ this }} t + WHERE t.metadata:verified_check_enabled::STRING = 'true' AND ( - metadata:token0::STRING NOT IN ( - SELECT token_address - FROM {{ ref('price__ez_asset_metadata') }} - WHERE is_verified AND token_address IS NOT NULL + ( + t.metadata:token0::STRING IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM {{ ref('price__ez_asset_metadata') }} v + WHERE v.is_verified + AND v.token_address IS NOT NULL + AND v.token_address = t.metadata:token0::STRING + ) ) - OR metadata:token1::STRING NOT IN ( - SELECT token_address - FROM {{ ref('price__ez_asset_metadata') }} - WHERE is_verified AND token_address IS NOT NULL + OR ( + t.metadata:token1::STRING IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM {{ ref('price__ez_asset_metadata') }} v + WHERE v.is_verified + AND v.token_address IS NOT NULL + AND v.token_address = t.metadata:token1::STRING + ) ) ); {% endif %} diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql index cb1804a9..e19a3605 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v2_tvl.sql @@ -4,6 +4,7 @@ {# Log configuration details #} {{ log_model_details() }} +-- depends_on: {{ ref('streamline__contract_reads_records') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql index b61b971d..677261b7 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v3_tvl.sql @@ -4,6 +4,7 @@ {# Log configuration details #} {{ log_model_details() }} +-- depends_on: {{ ref('streamline__contract_reads_records') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', diff --git a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql index 988476bf..52c2df47 100644 --- a/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql +++ b/models/curated_package/defi/tvl/silver/uniswap/silver_tvl__uniswap_v4_tvl.sql @@ -4,6 +4,7 @@ {# Log configuration details #} {{ log_model_details() }} +-- depends_on: {{ ref('streamline__contract_reads_records') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', From 7fa896dc7f964ec10c58e334d61ba83d64f546be Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 6 Jan 2026 10:39:08 -0700 Subject: [PATCH 157/164] depends --- .../streamline/records/streamline__contract_reads_records.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql index 6673cc9f..1a1bec3a 100644 --- a/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql +++ b/models/curated_package/contract_reads/streamline/records/streamline__contract_reads_records.sql @@ -4,6 +4,7 @@ {# Log configuration details #} {{ log_model_details() }} +-- depends_on: {{ ref('price__ez_asset_metadata') }} -- depends_on: {{ ref('silver_reads__lido_v1_reads') }} -- depends_on: {{ ref('silver_reads__binance_v1_reads') }} -- depends_on: {{ ref('silver_reads__polymarket_v1_reads') }} From 9ed5d11c791fb2a3fdd1053474400594ad9b222d Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 6 Jan 2026 10:42:45 -0700 Subject: [PATCH 158/164] depends --- .../defi/tvl/silver/silver_tvl__complete_tvl.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index 159a6f92..80429f4b 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -1,7 +1,10 @@ {# Get variables #} {% set vars = return_vars() %} + {# Log configuration details #} {{ log_model_details() }} + +-- depends_on: {{ ref('streamline__contract_reads_records') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', From 6937463f77ad3fee3e36a8080a7e80feb14ad575 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 6 Jan 2026 11:47:54 -0700 Subject: [PATCH 159/164] heal filter --- macros/curated_package/unverify_tvl.sql | 30 +++++++++++-------- .../tvl/silver/silver_tvl__complete_tvl.sql | 3 +- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/macros/curated_package/unverify_tvl.sql b/macros/curated_package/unverify_tvl.sql index 747b0174..f5576c53 100644 --- a/macros/curated_package/unverify_tvl.sql +++ b/macros/curated_package/unverify_tvl.sql @@ -2,27 +2,31 @@ {% if var('HEAL_MODEL', false) and is_incremental() %} -- Only target platforms that use verified_check_enabled -- Delete rows that are invalid in BOTH v2 and v3/v4 styles - DELETE FROM {{ this }} - WHERE platform IN ( - SELECT DISTINCT platform + DELETE FROM {{ this }} t + WHERE EXISTS ( + SELECT 1 FROM {{ ref('streamline__contract_reads_records') }} WHERE metadata:verified_check_enabled::STRING = 'true' + AND platform = t.platform ) --necessary for the complete_tvl model AND -- Not valid in v2 style (address as pool) - address NOT IN ( - SELECT DISTINCT contract_address - FROM {{ ref('streamline__contract_reads_records') }} - WHERE metadata:verified_check_enabled::STRING = 'true' - AND address IS NULL + NOT EXISTS ( + SELECT 1 + FROM {{ ref('streamline__contract_reads_records') }} r + WHERE r.metadata:verified_check_enabled::STRING = 'true' + AND r.address IS NULL + AND r.contract_address = t.address ) AND -- Not valid in v3/v4 style (token-address combo) - CONCAT(contract_address, '-', address) NOT IN ( - SELECT DISTINCT CONCAT(contract_address, '-', address) - FROM {{ ref('streamline__contract_reads_records') }} - WHERE metadata:verified_check_enabled::STRING = 'true' - AND address IS NOT NULL + NOT EXISTS ( + SELECT 1 + FROM {{ ref('streamline__contract_reads_records') }} r + WHERE r.metadata:verified_check_enabled::STRING = 'true' + AND r.address IS NOT NULL + AND r.contract_address = t.contract_address + AND r.address = t.address ); {% endif %} {% endmacro %} \ No newline at end of file diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index 80429f4b..a5c56612 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -45,13 +45,13 @@ WITH all_tvl AS ( platform, {{ max_usd_exponent }} AS max_usd_exponent FROM {{ model }} - {% if not loop.last %} {% if is_incremental() and platform not in vars.CURATED_FR_MODELS %} WHERE modified_timestamp > ( SELECT MAX(modified_timestamp) FROM {{ this }} ) {% endif %} + {% if not loop.last %} UNION ALL {% endif %} {% endfor %} @@ -175,6 +175,7 @@ heal_model AS ( t.block_date ) = p1.hour WHERE t.decimals IS NULL + AND c1.token_decimals IS NOT NULL -- only heal if contract data now exists ), {% endif %} From 8fc159906917191957b80055de899e49edb3f027 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 7 Jan 2026 09:13:09 -0700 Subject: [PATCH 160/164] block num --- .../defi/tvl/silver/silver_tvl__complete_tvl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql index a5c56612..9510079c 100644 --- a/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql +++ b/models/curated_package/defi/tvl/silver/silver_tvl__complete_tvl.sql @@ -230,7 +230,7 @@ SELECT platform, max_usd_exponent, {{ dbt_utils.generate_surrogate_key( - ['block_date','contract_address','address','platform'] + ['block_number','block_date','contract_address','address','platform'] ) }} AS complete_tvl_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, From bf5a101de1c5c52e804af010919551b02a4ad294 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 7 Jan 2026 11:48:29 -0700 Subject: [PATCH 161/164] reads model alt node --- .../streamline__contract_reads_daily_history_requests.sql | 7 ++++++- .../streamline__contract_reads_daily_realtime_requests.sql | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql index 96f43139..f65f28c7 100644 --- a/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql +++ b/models/curated_package/contract_reads/streamline/history/streamline__contract_reads_daily_history_requests.sql @@ -100,7 +100,12 @@ SELECT 'x-fsc-livequery', 'true' ), 'method', 'POST', - 'secret_name', '{{ vars.GLOBAL_NODE_VAULT_PATH }}', + 'secret_name', + {% if vars.GLOBAL_ALT_NODE_ENABLED %} + '{{ vars.GLOBAL_ALT_NODE_VAULT_PATH }}' + {% else %} + '{{ vars.GLOBAL_NODE_VAULT_PATH }}' + {% endif %}, 'url', '{{ vars.GLOBAL_NODE_URL }}' ) AS request FROM diff --git a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql index b25f5d86..803797d7 100644 --- a/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql +++ b/models/curated_package/contract_reads/streamline/realtime/streamline__contract_reads_daily_realtime_requests.sql @@ -109,7 +109,12 @@ SELECT 'x-fsc-livequery', 'true' ), 'method', 'POST', - 'secret_name', '{{ vars.GLOBAL_NODE_VAULT_PATH }}', + 'secret_name', + {% if vars.GLOBAL_ALT_NODE_ENABLED %} + '{{ vars.GLOBAL_ALT_NODE_VAULT_PATH }}' + {% else %} + '{{ vars.GLOBAL_NODE_VAULT_PATH }}' + {% endif %}, 'url', '{{ vars.GLOBAL_NODE_URL }}' ) AS request FROM From a372b3c211fa898226bc2fa1b1ad541cfcfabdb0 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 7 Jan 2026 13:16:28 -0700 Subject: [PATCH 162/164] erc20 --- .../streamline__balances_erc20_daily_history_requests.sql | 7 ++++++- .../streamline__balances_erc20_daily_realtime_requests.sql | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/models/balances_package/erc20/streamline/history/streamline__balances_erc20_daily_history_requests.sql b/models/balances_package/erc20/streamline/history/streamline__balances_erc20_daily_history_requests.sql index 6963167c..01f64f2e 100644 --- a/models/balances_package/erc20/streamline/history/streamline__balances_erc20_daily_history_requests.sql +++ b/models/balances_package/erc20/streamline/history/streamline__balances_erc20_daily_history_requests.sql @@ -87,7 +87,12 @@ SELECT 'x-fsc-livequery', 'true' ), 'method', 'POST', - 'secret_name', '{{ vars.GLOBAL_NODE_VAULT_PATH }}', + 'secret_name', + {% if vars.GLOBAL_ALT_NODE_ENABLED %} + '{{ vars.GLOBAL_ALT_NODE_VAULT_PATH }}' + {% else %} + '{{ vars.GLOBAL_NODE_VAULT_PATH }}' + {% endif %}, 'url', '{{ vars.GLOBAL_NODE_URL }}' ) AS request FROM diff --git a/models/balances_package/erc20/streamline/realtime/streamline__balances_erc20_daily_realtime_requests.sql b/models/balances_package/erc20/streamline/realtime/streamline__balances_erc20_daily_realtime_requests.sql index 78acc9f2..2580636c 100644 --- a/models/balances_package/erc20/streamline/realtime/streamline__balances_erc20_daily_realtime_requests.sql +++ b/models/balances_package/erc20/streamline/realtime/streamline__balances_erc20_daily_realtime_requests.sql @@ -89,7 +89,12 @@ SELECT 'x-fsc-livequery', 'true' ), 'method', 'POST', - 'secret_name', '{{ vars.GLOBAL_NODE_VAULT_PATH }}', + 'secret_name', + {% if vars.GLOBAL_ALT_NODE_ENABLED %} + '{{ vars.GLOBAL_ALT_NODE_VAULT_PATH }}' + {% else %} + '{{ vars.GLOBAL_NODE_VAULT_PATH }}' + {% endif %}, 'url', '{{ vars.GLOBAL_NODE_URL }}' ) AS request FROM From 31d18a3f914d0defa98fe68a78b1cd5fef3adbf7 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 8 Jan 2026 15:29:04 -0700 Subject: [PATCH 163/164] group --- .../defi/tvl/gold/defi__ez_protocol_tvl.sql | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql index 08f757ac..7a4dd6c2 100644 --- a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql @@ -9,6 +9,7 @@ tags = ['gold','defi','tvl','curated_daily'] ) }} +WITH tvl_usd AS ( SELECT block_date, SUM( @@ -19,13 +20,25 @@ SELECT ) AS tvl_usd, protocol, version, + platform +FROM + {{ ref('silver_tvl__complete_tvl') }} +GROUP BY + block_date, + protocol, + version, + platform +) +SELECT + block_date, + tvl_usd, + protocol, + version, platform, - inserted_timestamp, - modified_timestamp, + SYSDATE() AS modified_timestamp, + SYSDATE() AS inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['block_date','platform'] ) }} AS ez_protocol_tvl_id FROM - {{ ref('silver_tvl__complete_tvl') }} -GROUP BY - ALL + tvl_usd \ No newline at end of file From e3d9d2379e4f39cd4be9f9c3f3c30b89a33e6305 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 8 Jan 2026 15:36:31 -0700 Subject: [PATCH 164/164] ts --- .../defi/tvl/gold/defi__ez_protocol_tvl.sql | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql index 7a4dd6c2..035d4f1f 100644 --- a/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql +++ b/models/curated_package/defi/tvl/gold/defi__ez_protocol_tvl.sql @@ -20,7 +20,9 @@ SELECT ) AS tvl_usd, protocol, version, - platform + platform, + MAX(modified_timestamp) AS modified_timestamp, + MAX(inserted_timestamp) AS inserted_timestamp FROM {{ ref('silver_tvl__complete_tvl') }} GROUP BY @@ -35,8 +37,8 @@ SELECT protocol, version, platform, - SYSDATE() AS modified_timestamp, - SYSDATE() AS inserted_timestamp, + modified_timestamp, + inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['block_date','platform'] ) }} AS ez_protocol_tvl_id