From 766defdd5c7273a5abcb800904afbb8485715ca6 Mon Sep 17 00:00:00 2001 From: drethereum <71602799+drethereum@users.noreply.github.com> Date: Thu, 30 May 2024 10:28:39 -0600 Subject: [PATCH] AN-4820/heal-logic (#279) * vars * tx success * bridge heal * cte name * lps heal * all pools union * contract alias * var format * heal logic * snx and dex swaps heal * pool name * column names * round in gold --- README.md | 60 +- dbt_project.yml | 2 +- models/gold/defi/defi__ez_bridge_activity.sql | 8 +- models/gold/defi/defi__ez_dex_swaps.sql | 20 +- models/silver/core/silver__transfers.sql | 6 +- .../silver_bridge__across_fundsdeposited.sql | 2 + ...silver_bridge__across_v3fundsdeposited.sql | 1 + ...r_bridge__axelar_contractcallwithtoken.sql | 2 + .../silver_bridge__celer_cbridge_send.sql | 1 + ...lver_bridge__dln_debridge_createdorder.sql | 1 + .../eywa/silver_bridge__eywa_requestsent.sql | 1 + .../hop/silver_bridge__hop_ammwrapper.sql | 1 + .../hop/silver_bridge__hop_transfersent.sql | 1 + .../meson/silver_bridge__meson_transfers.sql | 1 + ...er_bridge__multichain_v7_loganyswapout.sql | 1 + ...ilver_bridge__complete_bridge_activity.sql | 254 ++- .../silver_bridge__stargate_createpool.sql | 3 +- .../stargate/silver_bridge__stargate_swap.sql | 1 + ...er_bridge__symbiosis_synthesizerequest.sql | 1 + .../silver_bridge__synapse_token_bridge.sql | 1 + ...ver_bridge__synapse_tokenbridgeandswap.sql | 1 + .../silver_dex__beethovenx_pools.sql | 2 + .../silver_dex__beethovenx_swaps.sql | 1 + .../dex/curve/silver_dex__curve_pools.sql | 3 +- .../dex/curve/silver_dex__curve_swaps.sql | 2 + .../dex/dodo/silver_dex__dodo_v2_pools.sql | 4 +- .../dex/dodo/silver_dex__dodo_v2_swaps.sql | 1 + .../dex/frax/silver_dex__fraxswap_pools.sql | 1 + .../dex/frax/silver_dex__fraxswap_swaps.sql | 1 + .../hashflow/silver_dex__hashflow_pools.sql | 3 +- .../hashflow/silver_dex__hashflow_swaps.sql | 2 + .../silver_dex__hashflow_v3_pools.sql | 1 + .../silver_dex__hashflow_v3_swaps.sql | 1 + .../silver_dex__kyberswap_v1_static_pools.sql | 1 + .../silver_dex__kyberswap_v1_static_swaps.sql | 1 + ...silver_dex__kyberswap_v2_elastic_pools.sql | 1 + ...silver_dex__kyberswap_v2_elastic_swaps.sql | 1 + ...lver_dex__complete_dex_liquidity_pools.sql | 914 +++++++--- .../dex/silver_dex__complete_dex_swaps.sql | 1531 ++++++----------- .../dex/sushi/silver_dex__sushi_pools.sql | 1 + .../synthetix/silver_dex__synthetix_swaps.sql | 56 +- .../dex/uniswap/silver_dex__univ2_pools.sql | 1 + .../dex/uniswap/silver_dex__univ3_pools.sql | 2 + .../silver_dex__velodrome_v2_pools.sql | 1 + .../dex/woofi/silver_dex__woofi_swaps.sql | 2 + .../silver__complete_lending_borrows.sql | 147 +- .../silver__complete_lending_deposits.sql | 147 +- .../silver__complete_lending_flashloans.sql | 201 ++- .../silver__complete_lending_liquidations.sql | 371 ++-- .../silver__complete_lending_repayments.sql | 326 ++-- .../silver__complete_lending_withdraws.sql | 246 ++- .../exactly/silver__exactly_borrows.sql | 1 + .../exactly/silver__exactly_deposits.sql | 1 + .../exactly/silver__exactly_liquidations.sql | 1 + .../exactly/silver__exactly_repayments.sql | 1 + .../exactly/silver__exactly_withdraws.sql | 1 + .../lending/sonne/silver__sonne_borrows.sql | 1 + .../lending/sonne/silver__sonne_deposits.sql | 1 + .../sonne/silver__sonne_liquidations.sql | 1 + .../sonne/silver__sonne_repayments.sql | 1 + .../lending/sonne/silver__sonne_withdraws.sql | 1 + .../lending/tarot/silver__tarot_borrows.sql | 1 + .../lending/tarot/silver__tarot_deposits.sql | 1 + .../tarot/silver__tarot_liquidations.sql | 1 + .../tarot/silver__tarot_repayments.sql | 1 + .../lending/tarot/silver__tarot_withdraws.sql | 1 + .../nft/sales/silver__complete_nft_sales.sql | 30 +- models/silver/nft/silver__nft_transfers.sql | 2 +- 68 files changed, 2715 insertions(+), 1673 deletions(-) diff --git a/README.md b/README.md index 3b3ace69..5c88664b 100644 --- a/README.md +++ b/README.md @@ -35,36 +35,43 @@ optimism: query_tag: ``` -### Variables +## Variables To control the creation of UDF or SP macros with dbt run: * UPDATE_UDFS_AND_SPS -When True, executes all macros included in the on-run-start hooks within dbt_project.yml on model run as normal -When False, none of the on-run-start macros are executed on model run + * Default values are False + * When True, executes all macros included in the on-run-start hooks within dbt_project.yml on model run as normal + * When False, none of the on-run-start macros are executed on model run -Default values are False + * Usage: `dbt run --vars '{"UPDATE_UDFS_AND_SPS":True}' -m ...` -* Usage: -dbt run --var '{"UPDATE_UDFS_AND_SPS":True}' -m ... +Use a variable to heal a model incrementally: +* HEAL_MODEL + * Default is FALSE (Boolean) + * When FALSE, logic will be negated + * When TRUE, heal logic will apply + * Include `heal` in model tags within the config block for inclusion in the `dbt_run_heal_models` workflow, e.g. `tags = 'heal'` -To reload records in a curated complete table without a full-refresh, such as `silver_bridge.complete_bridge_activity`: -* HEAL_CURATED_MODEL -Default is an empty array [] -When item is included in var array [], incremental logic will be skipped for that CTE / code block -When item is not included in var array [] or does not match specified item in model, incremental logic will apply -Example set up: `{% if is_incremental() and 'axelar' not in var('HEAL_CURATED_MODEL') %}` + * Usage: `dbt run --vars '{"HEAL_MODEL":True}' -m ...` -* Usage: -Single CTE: dbt run --var '{"HEAL_CURATED_MODEL":"axelar"}' -m ... -Multiple CTEs: dbt run --var '{"HEAL_CURATED_MODEL":["axelar","across","celer_cbridge"]}' -m ... +Use a variable to negate incremental logic: +* Example use case: reload records in a curated complete table without a full-refresh, such as `silver_bridge.complete_bridge_activity`: +* HEAL_MODELS + * Default is an empty array [] + * When item is included in var array [], incremental logic will be skipped for that CTE / code block + * When item is not included in var array [] or does not match specified item in model, incremental logic will apply + * Example set up: `{% if is_incremental() and 'axelar' not in var('HEAL_MODELS') %}` + * Usage: + * Single CTE: `dbt run --vars '{"HEAL_MODELS":"axelar"}' -m ...` + * Multiple CTEs: `dbt run --vars '{"HEAL_MODELS":["axelar","across","celer_cbridge"]}' -m ...` -### Resources: -- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) -- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers -- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support -- Find [dbt events](https://events.getdbt.com) near you -- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices +Use a variable to extend the incremental lookback period: +* LOOKBACK + * Default is a string representing the specified time interval e.g. '12 hours', '7 days' etc. + * Example set up: `SELECT MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}'` + + * Usage: `dbt run --vars '{"LOOKBACK":"36 hours"}' -m ...` ## Applying Model Tags @@ -98,7 +105,7 @@ To add/update a model's snowflake tags, add/modify the `meta` model property und By default, model tags are pushed to Snowflake on each load. You can disable this by setting the `UPDATE_SNOWFLAKE_TAGS` project variable to `False` during a run. ``` -dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":False}' -s models/core/core__fact_blocks.sql +dbt run --vars '{"UPDATE_SNOWFLAKE_TAGS":False}' -s models/core/core__fact_blocks.sql ``` ### Querying for existing tags on a model in snowflake @@ -106,4 +113,11 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":False}' -s models/core/core__fact_blocks ``` select * from table(optimism.information_schema.tag_references('optimism.core.fact_blocks', 'table')); -``` \ No newline at end of file +``` + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 2fea9aee..eb155f0e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -66,7 +66,7 @@ vars: OBSERV_FULL_TEST: False OVERFLOWED_TRACES: False HEAL_MODEL: False - HEAL_CURATED_MODEL: [] + HEAL_MODELS: [] START_GHA_TASKS: False API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}' diff --git a/models/gold/defi/defi__ez_bridge_activity.sql b/models/gold/defi/defi__ez_bridge_activity.sql index 65410dfc..b86c26f4 100644 --- a/models/gold/defi/defi__ez_bridge_activity.sql +++ b/models/gold/defi/defi__ez_bridge_activity.sql @@ -33,7 +33,13 @@ SELECT token_symbol, amount_unadj, amount, - amount_usd, + ROUND( + CASE + WHEN amount_usd < 1e+15 THEN amount_usd + ELSE NULL + END, + 2 + ) AS amount_usd, COALESCE ( complete_bridge_activity_id, {{ dbt_utils.generate_surrogate_key( diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql index 8d548422..30e47995 100644 --- a/models/gold/defi/defi__ez_dex_swaps.sql +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -24,10 +24,26 @@ SELECT event_name, amount_in_unadj, amount_in, - amount_in_usd, + ROUND( + CASE + WHEN amount_out_usd IS NULL + OR ABS((amount_in_usd - amount_out_usd) / NULLIF(amount_out_usd, 0)) > 0.75 + OR ABS((amount_in_usd - amount_out_usd) / NULLIF(amount_in_usd, 0)) > 0.75 THEN NULL + ELSE amount_in_usd + END, + 2 + ) AS amount_in_usd, amount_out_unadj, amount_out, - amount_out_usd, + ROUND( + CASE + WHEN amount_in_usd IS NULL + OR ABS((amount_out_usd - amount_in_usd) / NULLIF(amount_in_usd, 0)) > 0.75 + OR ABS((amount_out_usd - amount_in_usd) / NULLIF(amount_out_usd, 0)) > 0.75 THEN NULL + ELSE amount_out_usd + END, + 2 + ) AS amount_out_usd, sender, tx_to, event_index, diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index 7db266f5..0592226a 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -35,7 +35,7 @@ AND _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -169,7 +169,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -195,7 +195,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) diff --git a/models/silver/defi/bridge/across/silver_bridge__across_fundsdeposited.sql b/models/silver/defi/bridge/across/silver_bridge__across_fundsdeposited.sql index fcb841d7..d3a6dc0f 100644 --- a/models/silver/defi/bridge/across/silver_bridge__across_fundsdeposited.sql +++ b/models/silver/defi/bridge/across/silver_bridge__across_fundsdeposited.sql @@ -52,6 +52,7 @@ WITH base_evt AS ( WHERE topics [0] :: STRING = '0xafc4df6845a4ab948b492800d3d8a25d538a102a2bc07cd01f1cfa097fddcff6' AND contract_address = '0x6f26bf09b1c792e3228e5467807a900a503c0281' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -94,6 +95,7 @@ bridge_to AS ( WHERE topics [0] :: STRING = '0x0cf77fd2585a4d672259e86a6adb2f6b05334cbb420727afcfbc689d018bb456' AND contract_address = '0x6bff3581f871e317aa49fc6e6ebef8b46b628781' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/across/silver_bridge__across_v3fundsdeposited.sql b/models/silver/defi/bridge/across/silver_bridge__across_v3fundsdeposited.sql index b6c5091a..a6cdeb0b 100644 --- a/models/silver/defi/bridge/across/silver_bridge__across_v3fundsdeposited.sql +++ b/models/silver/defi/bridge/across/silver_bridge__across_v3fundsdeposited.sql @@ -60,6 +60,7 @@ WITH base_evt AS ( WHERE topics [0] :: STRING = '0xa123dc29aebf7d0c3322c8eeb5b999e859f39937950ed31056532713d0de396f' AND contract_address = '0x6f26bf09b1c792e3228e5467807a900a503c0281' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql b/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql index 38cca4cf..b6335be6 100644 --- a/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql +++ b/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql @@ -45,6 +45,7 @@ WITH base_evt AS ( '0xe19bb3b98f7727c520c757b8a00753eb47358b14', '0xe432150cce91c13a887f7d836923d5597add8e31' ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -92,6 +93,7 @@ native_gas_paid AS ( WHERE topics [0] :: STRING = '0x999d431b58761213cf53af96262b67a069cbd963499fd8effd1e21556217b841' AND contract_address = '0x2d5d7d31f671f86c782533cc367f14109a082712' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql b/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql index afabc764..5e44b961 100644 --- a/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql +++ b/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql @@ -46,6 +46,7 @@ WITH base_evt AS ( WHERE topics [0] :: STRING = '0x89d8051e597ab4178a863a5190407b98abfeff406aa8db90c59af76612e58f01' AND contract_address = '0x9d39fc627a6d9d9f8c831c16995b209548cc3401' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql b/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql index 43b1e36f..c1f6be99 100644 --- a/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql +++ b/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql @@ -70,6 +70,7 @@ WITH base_evt AS ( WHERE topics [0] :: STRING = '0xfc8703fd57380f9dd234a89dce51333782d49c5902f307b02f03e014d18fe471' --CreatedOrder AND contract_address = '0xef4fb24ad0916217251f553c0596f8edc630eb66' --Dln: Source + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql b/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql index 4a466bb3..124dfba8 100644 --- a/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql +++ b/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql @@ -40,6 +40,7 @@ WITH base_evt AS ( --PortalV2 '0xbf0b5d561b986809924f88099c4ff0e6bcce60c9' --PortalV2 ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/hop/silver_bridge__hop_ammwrapper.sql b/models/silver/defi/bridge/hop/silver_bridge__hop_ammwrapper.sql index 5ee9fb39..4b999da8 100644 --- a/models/silver/defi/bridge/hop/silver_bridge__hop_ammwrapper.sql +++ b/models/silver/defi/bridge/hop/silver_bridge__hop_ammwrapper.sql @@ -14,6 +14,7 @@ WITH base_contracts AS ( {{ ref('silver__logs') }} WHERE topics [0] :: STRING = '0xe35dddd4ea75d7e9b3fe93af4f4e40e778c3da4074c9d93e7c6536f1e803c1eb' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/hop/silver_bridge__hop_transfersent.sql b/models/silver/defi/bridge/hop/silver_bridge__hop_transfersent.sql index c782709d..acfb4094 100644 --- a/models/silver/defi/bridge/hop/silver_bridge__hop_transfersent.sql +++ b/models/silver/defi/bridge/hop/silver_bridge__hop_transfersent.sql @@ -51,6 +51,7 @@ WITH base_evt AS ( WHERE topics [0] :: STRING = '0xe35dddd4ea75d7e9b3fe93af4f4e40e778c3da4074c9d93e7c6536f1e803c1eb' AND origin_to_address IS NOT NULL + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql b/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql index 4172c316..a1f185ed 100644 --- a/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql +++ b/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql @@ -126,6 +126,7 @@ dst_info AS ( WHERE contract_address = '0x25ab3efd52e6470681ce037cd546dc60726948d3' AND topics [0] :: STRING = '0x5ce4019f772fda6cb703b26bce3ec3006eb36b73f1d3a0eb441213317d9f5e9d' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/multichain/silver_bridge__multichain_v7_loganyswapout.sql b/models/silver/defi/bridge/multichain/silver_bridge__multichain_v7_loganyswapout.sql index 7e0389f7..eba479ba 100644 --- a/models/silver/defi/bridge/multichain/silver_bridge__multichain_v7_loganyswapout.sql +++ b/models/silver/defi/bridge/multichain/silver_bridge__multichain_v7_loganyswapout.sql @@ -40,6 +40,7 @@ WITH base_evt AS ( WHERE topics [0] :: STRING = '0x0d969ae475ff6fcaf0dcfa760d4d8607244e8d95e9bf426f8d5d69f9a3e525af' AND contract_address = '0x1633d66ca91ce4d81f63ea047b7b19beb92df7f3' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql b/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql index 27f8ee27..484f47f8 100644 --- a/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql +++ b/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql @@ -1,9 +1,10 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform','version'], cluster_by = ['block_timestamp::DATE'], - tags = ['curated','reorg'] + tags = ['curated','reorg','heal'] ) }} WITH across AS ( @@ -33,18 +34,17 @@ WITH across AS ( FROM {{ ref('silver_bridge__across_fundsdeposited') }} -{% if is_incremental() and 'across' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'across' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), across_v3 AS ( - SELECT block_number, block_timestamp, @@ -70,11 +70,11 @@ across_v3 AS ( FROM {{ ref('silver_bridge__across_v3fundsdeposited') }} -{% if is_incremental() and 'across_v3' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'across_v3' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -106,11 +106,11 @@ axelar AS ( FROM {{ ref('silver_bridge__axelar_contractcallwithtoken') }} -{% if is_incremental() and 'axelar' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'axelar' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -142,11 +142,11 @@ celer_cbridge AS ( FROM {{ ref('silver_bridge__celer_cbridge_send') }} -{% if is_incremental() and 'celer_cbridge' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'celer_cbridge' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -178,11 +178,11 @@ dln_debridge AS ( FROM {{ ref('silver_bridge__dln_debridge_createdorder') }} -{% if is_incremental() and 'dln_debridge' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dln_debridge' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -214,11 +214,11 @@ eywa AS ( FROM {{ ref('silver_bridge__eywa_requestsent') }} -{% if is_incremental() and 'eywa' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'eywa' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -250,11 +250,11 @@ hop AS ( FROM {{ ref('silver_bridge__hop_transfersent') }} -{% if is_incremental() and 'hop' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'hop' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -286,11 +286,11 @@ meson AS ( FROM {{ ref('silver_bridge__meson_transfers') }} -{% if is_incremental() and 'meson' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'meson' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -322,11 +322,11 @@ multichain AS ( FROM {{ ref('silver_bridge__multichain_v7_loganyswapout') }} -{% if is_incremental() and 'multichain' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'multichain' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -358,11 +358,11 @@ stargate AS ( FROM {{ ref('silver_bridge__stargate_swap') }} -{% if is_incremental() and 'stargate' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'stargate' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -394,11 +394,11 @@ symbiosis AS ( FROM {{ ref('silver_bridge__symbiosis_synthesizerequest') }} -{% if is_incremental() and 'symbiosis' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'symbiosis' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -430,11 +430,11 @@ synapse_tb AS ( FROM {{ ref('silver_bridge__synapse_token_bridge') }} -{% if is_incremental() and 'synapse_tb' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'synapse_tb' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -466,11 +466,11 @@ synapse_tbs AS ( FROM {{ ref('silver_bridge__synapse_tokenbridgeandswap') }} -{% if is_incremental() and 'synapse_tbs' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'synapse_tbs' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -502,11 +502,11 @@ wormhole AS ( FROM {{ ref('silver_bridge__wormhole_transfers') }} -{% if is_incremental() and 'wormhole' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'wormhole' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -583,7 +583,7 @@ all_protocols AS ( FROM wormhole ), -FINAL AS ( +complete_bridge_activity AS ( SELECT block_number, block_timestamp, @@ -639,7 +639,7 @@ FINAL AS ( 2 ) ELSE NULL - END AS amount_usd_unadj, + END AS amount_usd, _id, b._inserted_timestamp FROM @@ -664,6 +664,183 @@ FINAL AS ( ) = LOWER( b.destination_chain ) +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + block_number, + block_timestamp, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_hash, + event_index, + bridge_address, + event_name, + platform, + version, + sender, + receiver, + destination_chain_receiver, + destination_chain_id, + destination_chain, + t0.token_address, + C.token_symbol AS token_symbol, + C.token_decimals AS token_decimals, + amount_unadj, + CASE + WHEN C.token_decimals IS NOT NULL THEN (amount_unadj / pow(10, C.token_decimals)) + ELSE amount_unadj + END AS amount_heal, + CASE + WHEN C.token_decimals IS NOT NULL THEN amount_heal * p.price + ELSE NULL + END AS amount_usd_heal, + _id, + t0._inserted_timestamp + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('silver__contracts') }} C + ON t0.token_address = C.contract_address + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.token_address = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform, + '-', + t1.version + ) + FROM + {{ this }} + t1 + WHERE + t1.token_decimals IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t1.token_address) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t2.block_number, + '-', + t2.platform, + '-', + t2.version + ) + FROM + {{ this }} + t2 + WHERE + t2.amount_usd IS NULL + AND t2._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t2.token_address + AND p.hour = DATE_TRUNC( + 'hour', + t2.block_timestamp + ) + ) + GROUP BY + 1 + ) + ), + {% endif %} + + FINAL AS ( + SELECT + * + FROM + complete_bridge_activity + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + block_number, + block_timestamp, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_hash, + event_index, + bridge_address, + event_name, + platform, + version, + sender, + receiver, + destination_chain_receiver, + destination_chain_id, + destination_chain, + token_address, + token_symbol, + token_decimals, + amount_unadj, + amount_heal AS amount, + amount_usd_heal AS amount_usd, + _id, + _inserted_timestamp +FROM + heal_model +{% endif %} ) SELECT block_number, @@ -687,21 +864,18 @@ SELECT token_decimals, amount_unadj, amount, - CASE - WHEN amount_usd_unadj < 1e+15 THEN amount_usd_unadj - ELSE NULL - END AS amount_usd, + amount_usd, _id, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( - ['_id'] + ['_id'] ) }} AS complete_bridge_activity_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - FINAL -WHERE destination_chain <> 'optimism' -qualify (ROW_NUMBER() over (PARTITION BY _id + FINAL +WHERE + destination_chain <> 'optimism' qualify (ROW_NUMBER() over (PARTITION BY _id ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql b/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql index ffc7a84a..ae4d4205 100644 --- a/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql +++ b/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql @@ -20,7 +20,8 @@ WITH base_contracts AS ( WHERE from_address = '0xe3b53af74a4bf62ae5511055290838050bf764df' AND TYPE ILIKE 'create%' - AND tx_status ILIKE 'success' + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql b/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql index 42d500fa..2c199dee 100644 --- a/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql +++ b/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql @@ -66,6 +66,7 @@ base_evt AS ( ON d.contract_address = p.pool_address WHERE topics [0] :: STRING = '0x34660fc8af304464529f48a778e03d03e4d34bcd5f9b6f0cfbf3cd238c642f7f' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql b/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql index dafdb4fc..6eb00b4d 100644 --- a/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql +++ b/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql @@ -41,6 +41,7 @@ WITH base_evt AS ( WHERE topics [0] :: STRING = '0x31325fe0a1a2e6a5b1e41572156ba5b4e94f0fae7e7f63ec21e9b5ce1e4b3eab' AND contract_address = '0x292fc50e4eb66c3f6514b9e402dbc25961824d62' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql b/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql index 3eb7a640..a54abee2 100644 --- a/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql +++ b/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql @@ -43,6 +43,7 @@ WITH base_evt AS ( ) AND contract_address = '0xaf41a65f786339e7911f4acdad6bd49426f2dc6b' AND origin_to_address IS NOT NULL + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql b/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql index e668eca8..971d593d 100644 --- a/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql +++ b/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql @@ -50,6 +50,7 @@ WITH base_evt AS ( WHERE topics [0] :: STRING = '0x91f25e9be0134ec851830e0e76dc71e06f9dade75a9b84e9524071dbbc319425' AND contract_address = '0xaf41a65f786339e7911f4acdad6bd49426f2dc6b' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/beethovenx/silver_dex__beethovenx_pools.sql b/models/silver/defi/dex/beethovenx/silver_dex__beethovenx_pools.sql index 15fc76fe..eec0401f 100644 --- a/models/silver/defi/dex/beethovenx/silver_dex__beethovenx_pools.sql +++ b/models/silver/defi/dex/beethovenx/silver_dex__beethovenx_pools.sql @@ -31,6 +31,7 @@ WITH pools_registered AS ( WHERE topics [0] :: STRING = '0x3c13bc30b8e878c53fd2a36b679409c073afd75950be43d8858768e956fbc20e' --PoolRegistered AND contract_address = '0xba12222222228d8ba445958a75a0704d566bf2c8' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -72,6 +73,7 @@ tokens_registered AS ( FROM pools_registered ) + AND tx_status = 'SUCCESS' ), function_sigs AS ( SELECT diff --git a/models/silver/defi/dex/beethovenx/silver_dex__beethovenx_swaps.sql b/models/silver/defi/dex/beethovenx/silver_dex__beethovenx_swaps.sql index 71c41b67..2b89eab9 100644 --- a/models/silver/defi/dex/beethovenx/silver_dex__beethovenx_swaps.sql +++ b/models/silver/defi/dex/beethovenx/silver_dex__beethovenx_swaps.sql @@ -53,6 +53,7 @@ swaps_base AS ( WHERE topics[0]::STRING = '0x2170c741c41531aec20e7c107c24eecfdd15e69c9bb0a8dd37b1840b9e0b207b' AND contract_address = '0xba12222222228d8ba445958a75a0704d566bf2c8' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/curve/silver_dex__curve_pools.sql b/models/silver/defi/dex/curve/silver_dex__curve_pools.sql index a5a2abd6..1b844370 100644 --- a/models/silver/defi/dex/curve/silver_dex__curve_pools.sql +++ b/models/silver/defi/dex/curve/silver_dex__curve_pools.sql @@ -32,7 +32,8 @@ WITH contract_deployments AS ( '0x745748bcfd8f9c2de519a71d789be8a63dd7d66c' ) AND TYPE ILIKE 'create%' - AND tx_status ILIKE 'success' + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/curve/silver_dex__curve_swaps.sql b/models/silver/defi/dex/curve/silver_dex__curve_swaps.sql index f8666b3d..676755a6 100644 --- a/models/silver/defi/dex/curve/silver_dex__curve_swaps.sql +++ b/models/silver/defi/dex/curve/silver_dex__curve_swaps.sql @@ -70,6 +70,7 @@ curve_base AS ( '0xb2e76ae99761dc136e598d4a629bb347eccb9532a5f8bbd72e18467c3c34cc98', '0xd013ca23e77a65003c2c659c5442c00c805371b7fc1ebd4c206c41d1536bd90b' ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -117,6 +118,7 @@ token_transfers AS ( curve_base ) AND CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) <> '0x0000000000000000000000000000000000000000' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql index 0437ad94..2ccfdad0 100644 --- a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql +++ b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql @@ -24,7 +24,8 @@ WITH pool_tr AS ( '0x386a28709a31532d4f68b06fd28a27e4ea378364' ) AND TYPE ILIKE 'create%' - AND tx_status ILIKE 'success' + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -66,6 +67,7 @@ pool_evt AS ( --NewDSP '0xaf5c5f12a80fc937520df6fcaed66262a4cc775e0f3fceaf7a7cfe476d9a751d' --NewDVM ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql index 54c983da..0ea498f3 100644 --- a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql +++ b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql @@ -89,6 +89,7 @@ swaps_base AS ( FROM proxies ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/frax/silver_dex__fraxswap_pools.sql b/models/silver/defi/dex/frax/silver_dex__fraxswap_pools.sql index e8c53a60..73d71f6e 100644 --- a/models/silver/defi/dex/frax/silver_dex__fraxswap_pools.sql +++ b/models/silver/defi/dex/frax/silver_dex__fraxswap_pools.sql @@ -27,6 +27,7 @@ WITH pool_creation AS ( WHERE contract_address = '0x67a1412d2d6cbf211bb71f8e851b4393b491b10f' --v2 factory AND topics [0] :: STRING = '0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9' --pairCreated + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/frax/silver_dex__fraxswap_swaps.sql b/models/silver/defi/dex/frax/silver_dex__fraxswap_swaps.sql index 909b297f..9a35e06c 100644 --- a/models/silver/defi/dex/frax/silver_dex__fraxswap_swaps.sql +++ b/models/silver/defi/dex/frax/silver_dex__fraxswap_swaps.sql @@ -60,6 +60,7 @@ swaps_base AS ( ON l.contract_address = pool_address WHERE l.topics [0] :: STRING = '0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822' --Swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql index 800df2f4..03f4af1d 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql @@ -23,7 +23,8 @@ WITH contract_deployments AS ( '0x63ae536fec0b57bdeb1fd6a893191b4239f61bff' ) AND TYPE ILIKE 'create%' - AND tx_status ILIKE 'success' + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql index a1d9df01..3dc59f62 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql @@ -68,6 +68,7 @@ router_swaps_base AS ( ON l.contract_address = p.pool_address WHERE l.topics [0] :: STRING = '0xb709ddcc6550418e9b89df1f4938071eeaa3f6376309904c77e15d46b16066f5' --swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -132,6 +133,7 @@ swaps_base AS ( ON l.contract_address = p.pool_address WHERE l.topics [0] :: STRING = '0x8cf3dec1929508e5677d7db003124e74802bfba7250a572205a9986d86ca9f1e' --swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql index 762b0a30..75134b08 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql @@ -21,6 +21,7 @@ WITH contract_deployments AS ( WHERE contract_address = '0x6d551f4d999fac0984eb75b2b230ba7e7651bde7' AND topics [0] :: STRING = '0xdbd2a1ea6808362e6adbec4db4969cbc11e3b0b28fb6c74cb342defaaf1daada' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql index a8335617..a4e14f5c 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql @@ -52,6 +52,7 @@ swaps AS ( ON l.contract_address = p.pool_address WHERE l.topics [0] :: STRING = '0x34f57786fb01682fb4eec88d340387ef01a168fe345ea5b76f709d4e560c10eb' --Trade + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql index fccdd17d..55a046d7 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql @@ -39,6 +39,7 @@ WITH pool_creation AS ( WHERE contract_address = LOWER('0x1c758aF0688502e49140230F6b0EBd376d429be5') --static pool factory AND topics [0] :: STRING = '0xb6bce363b712c921bead4bcc977289440eb6172eb89e258e3a25bd49ca806de6' --create pool + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql index c69f4760..dcbf4fd1 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql @@ -65,6 +65,7 @@ swaps_base AS ( ON p.pool_address = l.contract_address WHERE l.topics [0] :: STRING = '0x606ecd02b3e3b4778f8e97b2e03351de14224efaa5fa64e62200afc9395c2499' -- static swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql index f20cb9f6..7e4add34 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql @@ -30,6 +30,7 @@ WITH pool_creation AS ( WHERE contract_address = '0x5f1dddbf348ac2fbe22a163e30f99f9ece3dd50a' --Elastic Pool Deployer AND topics [0] :: STRING = '0x783cca1c0412dd0d695e784568c96da2e9c22ff989357a2e8b1d9b2b4e6b7118' --Create pool + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql index bc7d94ea..a131ee17 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql @@ -78,6 +78,7 @@ swaps_base AS ( ON p.pool_address = l.contract_address WHERE topics [0] :: STRING = '0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67' -- elastic swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql b/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql index 0cc86dba..86393903 100644 --- a/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql +++ b/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql @@ -3,15 +3,15 @@ incremental_strategy = 'delete+insert', unique_key = ['block_number','platform','version'], cluster_by = ['block_timestamp::DATE'], - tags = ['curated','reorg'] + tags = ['curated','reorg','heal'] ) }} WITH contracts AS ( SELECT - contract_address AS address, - token_symbol AS symbol, - token_decimals AS decimals, + contract_address, + token_symbol, + token_decimals, _inserted_timestamp FROM {{ ref('silver__contracts') }} @@ -24,10 +24,8 @@ beethovenx AS ( contract_address, pool_address, pool_name, - 'beethoven-x' AS platform, - 'v1' AS version, - _log_id AS _id, - _inserted_timestamp, + NULL AS fee, + NULL AS tick_spacing, token0, token1, token2, @@ -35,15 +33,19 @@ beethovenx AS ( token4, token5, token6, - token7 + token7, + 'beethoven-x' AS platform, + 'v1' AS version, + _log_id AS _id, + _inserted_timestamp FROM {{ ref('silver_dex__beethovenx_pools') }} -{% if is_incremental() and 'beethovenx' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'beethovenx' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -57,10 +59,8 @@ curve AS ( deployer_address AS contract_address, pool_address, pool_name, - 'curve' AS platform, - 'v1' AS version, - _call_id AS _id, - _inserted_timestamp, + NULL AS fee, + NULL AS tick_spacing, MAX( CASE WHEN token_num = 1 THEN token_address @@ -100,15 +100,19 @@ curve AS ( CASE WHEN token_num = 8 THEN token_address END - ) AS token7 + ) AS token7, + 'curve' AS platform, + 'v1' AS version, + _call_id AS _id, + _inserted_timestamp FROM {{ ref('silver_dex__curve_pools') }} -{% if is_incremental() and 'curve' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'curve' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -124,8 +128,16 @@ dodo_v2 AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, base_token AS token0, quote_token AS token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'dodo-v2' AS platform, 'v2' AS version, _id, @@ -135,10 +147,10 @@ dodo_v2 AS ( WHERE token0 IS NOT NULL -{% if is_incremental() and 'dodo_v2' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dodo_v2' not in var('HEAL_MODELS') %} AND _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -152,8 +164,16 @@ frax AS ( factory_address AS contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'fraxswap' AS platform, 'v1' AS version, _log_id AS _id, @@ -161,11 +181,11 @@ frax AS ( FROM {{ ref('silver_dex__fraxswap_pools') }} -{% if is_incremental() and 'frax' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'frax' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -179,8 +199,16 @@ velodrome_v2 AS ( factory_address AS contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'velodrome-v2' AS platform, 'v2' AS version, _log_id AS _id, @@ -188,11 +216,11 @@ velodrome_v2 AS ( FROM {{ ref('silver_dex__velodrome_v2_pools') }} -{% if is_incremental() and 'velodrome_v2' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'velodrome_v2' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -206,8 +234,16 @@ kyberswap_v1_static AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'kyberswap-v1' AS platform, 'v1-static' AS version, _log_id AS _id, @@ -215,11 +251,11 @@ kyberswap_v1_static AS ( FROM {{ ref('silver_dex__kyberswap_v1_static_pools') }} -{% if is_incremental() and 'kyberswap_v1_static' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kyberswap_v1_static' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -232,10 +268,17 @@ kyberswap_v2_elastic AS ( tx_hash, contract_address, pool_address, + NULL AS pool_name, swap_fee_units AS fee, tick_distance AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'kyberswap-v2' AS platform, 'v2' AS version, _log_id AS _id, @@ -243,11 +286,11 @@ kyberswap_v2_elastic AS ( FROM {{ ref('silver_dex__kyberswap_v2_elastic_pools') }} -{% if is_incremental() and 'kyberswap_v2_elastic' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kyberswap_v2_elastic' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -261,8 +304,16 @@ sushi AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'sushiswap' AS platform, 'v1' AS version, _log_id AS _id, @@ -270,11 +321,11 @@ sushi AS ( FROM {{ ref('silver_dex__sushi_pools') }} -{% if is_incremental() and 'sushi' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'sushi' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -287,10 +338,17 @@ uni_v3 AS ( tx_hash, contract_address, pool_address, + NULL AS pool_name, fee, tick_spacing, token0_address AS token0, token1_address AS token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'uniswap-v3' AS platform, 'v3' AS version, _id, @@ -298,45 +356,51 @@ uni_v3 AS ( FROM {{ ref('silver_dex__univ3_pools') }} -{% if is_incremental() and 'uni_v3' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'uni_v3' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - uni_v2 AS ( - -SELECT + SELECT block_number, block_timestamp, tx_hash, contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'uniswap-v2' AS platform, 'v2' AS version, _log_id AS _id, _inserted_timestamp -FROM + FROM {{ ref('silver_dex__univ2_pools') }} -{% if is_incremental() and 'uni_v2' not in var('HEAL_CURATED_MODEL') %} + +{% if is_incremental() and 'uni_v2' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - velodrome AS ( SELECT created_block AS block_number, @@ -345,6 +409,14 @@ velodrome AS ( deployer_address AS contract_address, pool_address, pool_name, + NULL AS fee, + NULL AS tick_spacing, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, token0_address AS token0, token1_address AS token1, 'velodrome' AS platform, @@ -354,17 +426,17 @@ velodrome AS ( FROM {{ ref('silver__velodrome_pool_details') }} -{% if is_incremental() and 'velodrome' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'velodrome' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -all_pools_standard AS ( +all_pools AS ( SELECT * FROM @@ -399,8 +471,7 @@ all_pools_standard AS ( * FROM velodrome -), -all_pools_v3 AS ( + UNION ALL SELECT * FROM @@ -410,8 +481,7 @@ all_pools_v3 AS ( * FROM kyberswap_v2_elastic -), -all_pools_other AS ( + UNION ALL SELECT * FROM @@ -422,92 +492,27 @@ all_pools_other AS ( FROM curve ), -FINAL AS ( +complete_lps AS ( SELECT block_number, block_timestamp, tx_hash, - contract_address, + p.contract_address, pool_address, CASE - WHEN pool_name IS NULL THEN CONCAT( + WHEN pool_name IS NOT NULL THEN pool_name + WHEN pool_name IS NULL + AND platform IN ( + 'uniswap-v3', + 'kyberswap-v2' + ) THEN CONCAT( COALESCE( - c0.symbol, + c0.token_symbol, CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) ), '-', COALESCE( - c1.symbol, - CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) - ) - ) - ELSE pool_name - END AS pool_name, - OBJECT_CONSTRUCT( - 'token0', - token0, - 'token1', - token1 - ) AS tokens, - OBJECT_CONSTRUCT( - 'token0', - c0.symbol, - 'token1', - c1.symbol - ) AS symbols, - OBJECT_CONSTRUCT( - 'token0', - c0.decimals, - 'token1', - c1.decimals - ) AS decimals, - platform, - version, - _id, - p._inserted_timestamp - FROM - all_pools_standard p - LEFT JOIN contracts c0 - ON c0.address = p.token0 - LEFT JOIN contracts c1 - ON c1.address = p.token1 - UNION ALL - SELECT - block_number, - block_timestamp, - tx_hash, - contract_address, - pool_address, - CASE - WHEN platform = 'kyberswap-v2' THEN CONCAT( - COALESCE( - c0.symbol, - CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) - ), - '-', - COALESCE( - c1.symbol, - CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) - ), - ' ', - COALESCE( - fee, - 0 - ), - ' ', - COALESCE( - tick_spacing, - 0 - ) - ) - WHEN platform = 'uniswap-v3' THEN CONCAT( - COALESCE( - c0.symbol, - CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) - ), - '-', - COALESCE( - c1.symbol, + c1.token_symbol, CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) ), ' ', @@ -520,78 +525,68 @@ FINAL AS ( tick_spacing, 0 ), - ' UNI-V3 LP' + CASE + WHEN platform = 'uniswap-v3' THEN ' UNI-V3 LP' + WHEN platform = 'kyberswap-v2' THEN '' + END ) - END AS pool_name, - OBJECT_CONSTRUCT( - 'token0', - token0, - 'token1', - token1 - ) AS tokens, - OBJECT_CONSTRUCT( - 'token0', - c0.symbol, - 'token1', - c1.symbol - ) AS symbols, - OBJECT_CONSTRUCT( - 'token0', - c0.decimals, - 'token1', - c1.decimals - ) AS decimals, - platform, - version, - _id, - p._inserted_timestamp - FROM - all_pools_v3 p - LEFT JOIN contracts c0 - ON c0.address = p.token0 - LEFT JOIN contracts c1 - ON c1.address = p.token1 - UNION ALL - SELECT - block_number, - block_timestamp, - tx_hash, - contract_address, - pool_address, - CASE - WHEN pool_name IS NULL THEN CONCAT( - COALESCE(c0.symbol, SUBSTRING(token0, 1, 5) || '...' || SUBSTRING(token0, 39, 42)), + WHEN pool_name IS NULL + AND platform IN ( + 'beethoven-x', + 'curve' + ) THEN CONCAT( + COALESCE(c0.token_symbol, SUBSTRING(token0, 1, 5) || '...' || SUBSTRING(token0, 39, 42)), CASE - WHEN token1 IS NOT NULL THEN '-' || COALESCE(c1.symbol, SUBSTRING(token1, 1, 5) || '...' || SUBSTRING(token1, 39, 42)) + WHEN token1 IS NOT NULL THEN '-' || COALESCE(c1.token_symbol, SUBSTRING(token1, 1, 5) || '...' || SUBSTRING(token1, 39, 42)) ELSE '' END, CASE - WHEN token2 IS NOT NULL THEN '-' || COALESCE(c2.symbol, SUBSTRING(token2, 1, 5) || '...' || SUBSTRING(token2, 39, 42)) + WHEN token2 IS NOT NULL THEN '-' || COALESCE(c2.token_symbol, SUBSTRING(token2, 1, 5) || '...' || SUBSTRING(token2, 39, 42)) ELSE '' END, CASE - WHEN token3 IS NOT NULL THEN '-' || COALESCE(c3.symbol, SUBSTRING(token3, 1, 5) || '...' || SUBSTRING(token3, 39, 42)) + WHEN token3 IS NOT NULL THEN '-' || COALESCE(c3.token_symbol, SUBSTRING(token3, 1, 5) || '...' || SUBSTRING(token3, 39, 42)) ELSE '' END, CASE - WHEN token4 IS NOT NULL THEN '-' || COALESCE(c4.symbol, SUBSTRING(token4, 1, 5) || '...' || SUBSTRING(token4, 39, 42)) + WHEN token4 IS NOT NULL THEN '-' || COALESCE(c4.token_symbol, SUBSTRING(token4, 1, 5) || '...' || SUBSTRING(token4, 39, 42)) ELSE '' END, CASE - WHEN token5 IS NOT NULL THEN '-' || COALESCE(c5.symbol, SUBSTRING(token5, 1, 5) || '...' || SUBSTRING(token5, 39, 42)) + WHEN token5 IS NOT NULL THEN '-' || COALESCE(c5.token_symbol, SUBSTRING(token5, 1, 5) || '...' || SUBSTRING(token5, 39, 42)) ELSE '' END, CASE - WHEN token6 IS NOT NULL THEN '-' || COALESCE(c6.symbol, SUBSTRING(token6, 1, 5) || '...' || SUBSTRING(token6, 39, 42)) + WHEN token6 IS NOT NULL THEN '-' || COALESCE(c6.token_symbol, SUBSTRING(token6, 1, 5) || '...' || SUBSTRING(token6, 39, 42)) ELSE '' END, CASE - WHEN token7 IS NOT NULL THEN '-' || COALESCE(c7.symbol, SUBSTRING(token7, 1, 5) || '...' || SUBSTRING(token7, 39, 42)) + WHEN token7 IS NOT NULL THEN '-' || COALESCE(c7.token_symbol, SUBSTRING(token7, 1, 5) || '...' || SUBSTRING(token7, 39, 42)) ELSE '' END ) - ELSE pool_name + ELSE CONCAT( + COALESCE( + c0.token_symbol, + CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) + ), + '-', + COALESCE( + c1.token_symbol, + CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) + ) + ) END AS pool_name, + fee, + tick_spacing, + token0, + token1, + token2, + token3, + token4, + token5, + token6, + token7, OBJECT_CONSTRUCT( 'token0', token0, @@ -612,62 +607,583 @@ FINAL AS ( ) AS tokens, OBJECT_CONSTRUCT( 'token0', - c0.symbol, + c0.token_symbol, 'token1', - c1.symbol, + c1.token_symbol, 'token2', - c2.symbol, + c2.token_symbol, 'token3', - c3.symbol, + c3.token_symbol, 'token4', - c4.symbol, + c4.token_symbol, 'token5', - c5.symbol, + c5.token_symbol, 'token6', - c6.symbol, + c6.token_symbol, 'token7', - c7.symbol + c7.token_symbol ) AS symbols, OBJECT_CONSTRUCT( 'token0', - c0.decimals, + c0.token_decimals, 'token1', - c1.decimals, + c1.token_decimals, 'token2', - c2.decimals, + c2.token_decimals, 'token3', - c3.decimals, + c3.token_decimals, 'token4', - c4.decimals, + c4.token_decimals, 'token5', - c5.decimals, + c5.token_decimals, 'token6', - c6.decimals, + c6.token_decimals, 'token7', - c7.decimals + c7.token_decimals ) AS decimals, platform, version, _id, p._inserted_timestamp FROM - all_pools_other p + all_pools p LEFT JOIN contracts c0 - ON c0.address = p.token0 + ON c0.contract_address = p.token0 LEFT JOIN contracts c1 - ON c1.address = p.token1 + ON c1.contract_address = p.token1 LEFT JOIN contracts c2 - ON c2.address = p.token2 + ON c2.contract_address = p.token2 LEFT JOIN contracts c3 - ON c3.address = p.token3 + ON c3.contract_address = p.token3 LEFT JOIN contracts c4 - ON c4.address = p.token4 + ON c4.contract_address = p.token4 LEFT JOIN contracts c5 - ON c5.address = p.token5 + ON c5.contract_address = p.token5 LEFT JOIN contracts c6 - ON c6.address = p.token6 + ON c6.contract_address = p.token6 LEFT JOIN contracts c7 - ON c7.address = p.token7 + ON c7.contract_address = p.token7 +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + t0.contract_address, + pool_address, + CASE + WHEN pool_name IS NOT NULL THEN pool_name + WHEN pool_name IS NULL + AND platform IN ( + 'uniswap-v3', + 'kyberswap-v2' + ) THEN CONCAT( + COALESCE( + c0.token_symbol, + CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) + ), + '-', + COALESCE( + c1.token_symbol, + CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) + ), + ' ', + COALESCE( + fee, + 0 + ), + ' ', + COALESCE( + tick_spacing, + 0 + ), + CASE + WHEN platform = 'uniswap-v3' THEN ' UNI-V3 LP' + WHEN platform = 'kyberswap-v2' THEN '' + END + ) + WHEN pool_name IS NULL + AND platform IN ( + 'beethoven-x', + 'curve' + ) THEN CONCAT( + COALESCE(c0.token_symbol, SUBSTRING(token0, 1, 5) || '...' || SUBSTRING(token0, 39, 42)), + CASE + WHEN token1 IS NOT NULL THEN '-' || COALESCE(c1.token_symbol, SUBSTRING(token1, 1, 5) || '...' || SUBSTRING(token1, 39, 42)) + ELSE '' + END, + CASE + WHEN token2 IS NOT NULL THEN '-' || COALESCE(c2.token_symbol, SUBSTRING(token2, 1, 5) || '...' || SUBSTRING(token2, 39, 42)) + ELSE '' + END, + CASE + WHEN token3 IS NOT NULL THEN '-' || COALESCE(c3.token_symbol, SUBSTRING(token3, 1, 5) || '...' || SUBSTRING(token3, 39, 42)) + ELSE '' + END, + CASE + WHEN token4 IS NOT NULL THEN '-' || COALESCE(c4.token_symbol, SUBSTRING(token4, 1, 5) || '...' || SUBSTRING(token4, 39, 42)) + ELSE '' + END, + CASE + WHEN token5 IS NOT NULL THEN '-' || COALESCE(c5.token_symbol, SUBSTRING(token5, 1, 5) || '...' || SUBSTRING(token5, 39, 42)) + ELSE '' + END, + CASE + WHEN token6 IS NOT NULL THEN '-' || COALESCE(c6.token_symbol, SUBSTRING(token6, 1, 5) || '...' || SUBSTRING(token6, 39, 42)) + ELSE '' + END, + CASE + WHEN token7 IS NOT NULL THEN '-' || COALESCE(c7.token_symbol, SUBSTRING(token7, 1, 5) || '...' || SUBSTRING(token7, 39, 42)) + ELSE '' + END + ) + ELSE CONCAT( + COALESCE( + c0.token_symbol, + CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) + ), + '-', + COALESCE( + c1.token_symbol, + CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) + ) + ) + END AS pool_name_heal, + fee, + tick_spacing, + token0, + token1, + token2, + token3, + token4, + token5, + token6, + token7, + tokens, + OBJECT_CONSTRUCT( + 'token0', + c0.token_symbol, + 'token1', + c1.token_symbol, + 'token2', + c2.token_symbol, + 'token3', + c3.token_symbol, + 'token4', + c4.token_symbol, + 'token5', + c5.token_symbol, + 'token6', + c6.token_symbol, + 'token7', + c7.token_symbol + ) AS symbols_heal, + OBJECT_CONSTRUCT( + 'token0', + c0.token_decimals, + 'token1', + c1.token_decimals, + 'token2', + c2.token_decimals, + 'token3', + c3.token_decimals, + 'token4', + c4.token_decimals, + 'token5', + c5.token_decimals, + 'token6', + c6.token_decimals, + 'token7', + c7.token_decimals + ) AS decimals_heal, + platform, + version, + _id, + t0._inserted_timestamp + FROM + {{ this }} + t0 + LEFT JOIN contracts c0 + ON c0.contract_address = t0.token0 + LEFT JOIN contracts c1 + ON c1.contract_address = t0.token1 + LEFT JOIN contracts c2 + ON c2.contract_address = t0.token2 + LEFT JOIN contracts c3 + ON c3.contract_address = t0.token3 + LEFT JOIN contracts c4 + ON c4.contract_address = t0.token4 + LEFT JOIN contracts c5 + ON c5.contract_address = t0.token5 + LEFT JOIN contracts c6 + ON c6.contract_address = t0.token6 + LEFT JOIN contracts c7 + ON c7.contract_address = t0.token7 + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform, + '-', + t1.version + ) + FROM + {{ this }} + t1 + WHERE + t1.decimals :token0 :: INT IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t1.tokens :token0 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t2.block_number, + '-', + t2.platform, + '-', + t2.version + ) + FROM + {{ this }} + t2 + WHERE + t2.decimals :token1 :: INT IS NULL + AND t2._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t2.tokens :token1 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t3.block_number, + '-', + t3.platform, + '-', + t3.version + ) + FROM + {{ this }} + t3 + WHERE + t3.decimals :token2 :: INT IS NULL + AND t3._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t3.tokens :token2 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t4.block_number, + '-', + t4.platform, + '-', + t4.version + ) + FROM + {{ this }} + t4 + WHERE + t4.decimals :token3 :: INT IS NULL + AND t4._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t4.tokens :token3 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t5.block_number, + '-', + t5.platform, + '-', + t5.version + ) + FROM + {{ this }} + t5 + WHERE + t5.decimals :token4 :: INT IS NULL + AND t5._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t5.tokens :token4 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t6.block_number, + '-', + t6.platform, + '-', + t6.version + ) + FROM + {{ this }} + t6 + WHERE + t6.decimals :token5 :: INT IS NULL + AND t6._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t6.tokens :token5 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t7.block_number, + '-', + t7.platform, + '-', + t7.version + ) + FROM + {{ this }} + t7 + WHERE + t7.decimals :token6 :: INT IS NULL + AND t7._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t7.tokens :token6 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t8.block_number, + '-', + t8.platform, + '-', + t8.version + ) + FROM + {{ this }} + t8 + WHERE + t8.decimals :token7 :: INT IS NULL + AND t8._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t8.tokens :token7 :: STRING) + GROUP BY + 1 + ) + ), + {% endif %} + + FINAL AS ( + SELECT + * + FROM + complete_lps + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + pool_address, + pool_name_heal AS pool_name, + fee, + tick_spacing, + token0, + token1, + token2, + token3, + token4, + token5, + token6, + token7, + tokens, + symbols_heal AS symbols, + decimals_heal AS decimals, + platform, + version, + _id, + _inserted_timestamp +FROM + heal_model +{% endif %} ) SELECT block_number, @@ -681,6 +1197,16 @@ SELECT tokens, symbols, decimals, + fee, + tick_spacing, + token0, + token1, + token2, + token3, + token4, + token5, + token6, + token7, _id, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( diff --git a/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql b/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql index 1a686525..17380830 100644 --- a/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql +++ b/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql @@ -1,37 +1,14 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform','version'], cluster_by = ['block_timestamp::DATE'], - tags = ['curated','reorg'] + tags = ['curated','reorg','heal'] ) }} -WITH contracts AS ( +WITH univ3 AS ( - SELECT - contract_address AS address, - token_symbol AS symbol, - token_name AS NAME, - token_decimals AS decimals - FROM - {{ ref('silver__contracts') }} -), -prices AS ( - SELECT - HOUR, - token_address, - price - FROM - {{ ref('price__ez_prices_hourly') }} - WHERE - token_address IN ( - SELECT - DISTINCT address - FROM - contracts - ) -), -univ3_swaps AS ( SELECT block_number, block_timestamp, @@ -40,51 +17,15 @@ univ3_swaps AS ( origin_from_address, origin_to_address, pool_address AS contract_address, - NULL AS pool_name, 'Swap' AS event_name, - amount0_unadj / pow(10, COALESCE(c1.decimals, 18)) AS amount0_adjusted, - amount1_unadj / pow(10, COALESCE(c2.decimals, 18)) AS amount1_adjusted, - CASE - WHEN c1.decimals IS NOT NULL THEN ROUND( - p1.price * amount0_adjusted, - 2 - ) - END AS amount0_usd, - CASE - WHEN c2.decimals IS NOT NULL THEN ROUND( - p2.price * amount1_adjusted, - 2 - ) - END AS amount1_usd, CASE WHEN amount0_unadj > 0 THEN ABS(amount0_unadj) ELSE ABS(amount1_unadj) END AS amount_in_unadj, - CASE - WHEN amount0_unadj > 0 THEN ABS(amount0_adjusted) - ELSE ABS(amount1_adjusted) - END AS amount_in, - CASE - WHEN amount0_unadj > 0 THEN ABS(amount0_usd) - ELSE ABS(amount1_usd) - END AS amount_in_usd, CASE WHEN amount0_unadj < 0 THEN ABS(amount0_unadj) ELSE ABS(amount1_unadj) END AS amount_out_unadj, - CASE - WHEN amount0_unadj < 0 THEN ABS(amount0_adjusted) - ELSE ABS(amount1_adjusted) - END AS amount_out, - CASE - WHEN amount0_unadj < 0 THEN ABS(amount0_usd) - ELSE ABS(amount1_usd) - END AS amount_out_usd, - sender, - recipient AS tx_to, - event_index, - 'uniswap-v3' AS platform, - 'v3' AS version, CASE WHEN amount0_unadj > 0 THEN token0_address ELSE token1_address @@ -93,47 +34,27 @@ univ3_swaps AS ( WHEN amount0_unadj < 0 THEN token0_address ELSE token1_address END AS token_out, - CASE - WHEN amount0_unadj > 0 THEN c1.symbol - ELSE c2.symbol - END AS symbol_in, - CASE - WHEN amount0_unadj < 0 THEN c1.symbol - ELSE c2.symbol - END AS symbol_out, + sender, + recipient AS tx_to, + event_index, + 'uniswap-v3' AS platform, + 'v3' AS version, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__univ3_swaps') }} - s - LEFT JOIN contracts c1 - ON c1.address = s.token0_address - LEFT JOIN contracts c2 - ON c2.address = s.token1_address - LEFT JOIN prices p1 - ON s.token0_address = p1.token_address - AND DATE_TRUNC( - 'hour', - block_timestamp - ) = p1.hour - LEFT JOIN prices p2 - ON s.token1_address = p2.token_address - AND DATE_TRUNC( - 'hour', - block_timestamp - ) = p2.hour -{% if is_incremental() and 'univ3_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'univ3' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -univ2_swaps AS ( +univ2 AS ( SELECT block_number, block_timestamp, @@ -143,50 +64,31 @@ univ2_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__univ2_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'univ2_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'univ2' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - -hashflow_swaps AS ( +hashflow AS ( SELECT block_number, block_timestamp, @@ -196,71 +98,31 @@ hashflow_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - CONCAT( - LEAST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ), - '-', - GREATEST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ) - ) AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__hashflow_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'hashflow_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'hashflow' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -hashflow_v3_swaps AS ( +hashflow_v3 AS ( SELECT block_number, block_timestamp, @@ -270,71 +132,31 @@ hashflow_v3_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v3' AS version, - token_in, - token_out, - CONCAT( - LEAST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ), - '-', - GREATEST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ) - ) AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__hashflow_v3_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'hashflow_v3_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'hashflow_v3' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -velodrome_swaps AS ( +velodrome AS ( SELECT block_number, block_timestamp, @@ -343,33 +165,25 @@ velodrome_swaps AS ( origin_from_address, origin_to_address, contract_address, - pool_name, CASE WHEN event_name IS NULL THEN 'Swap' ELSE event_name END AS event_name, amount_in_unadj, - amount_in, - amount_in_usd, amount_out_unadj, - amount_out, - amount_out_usd, + token_address_in AS token_in, + token_address_out AS token_out, sender_address AS sender, to_address AS tx_to, event_index, platform, 'v1' AS version, - token_address_in AS token_in, - token_address_out AS token_out, - symbol_in, - symbol_out, _log_id, '1970-01-01' :: DATE AS _inserted_timestamp FROM {{ ref('velodrome__ez_swaps') }} - s -{% if is_incremental() and 'velodrome_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'velodrome' not in var('HEAL_MODELS') %} WHERE block_timestamp >= ( SELECT @@ -379,7 +193,7 @@ WHERE ) {% endif %} ), -sushi_swaps AS ( +sushi AS ( SELECT block_number, block_timestamp, @@ -389,49 +203,31 @@ sushi_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__sushi_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'sushi_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'sushi' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -synthetix_swaps AS ( +synthetix AS ( SELECT block_number, block_timestamp, @@ -441,35 +237,10 @@ synthetix_swaps AS ( origin_to_address, contract_address, event_name, + amount_in_unadj, + amount_out_unadj, token_in, token_out, - symbol_in, - symbol_out, - CONCAT(LEAST(symbol_in, symbol_out), '-', GREATEST(symbol_in, symbol_out)) AS pool_name, - COALESCE( - decimals_in, - 18 - ) AS decimals_in, - COALESCE( - decimals_out, - 18 - ) AS decimals_out, - amount_in_unadj, - CASE - WHEN decimals_in IS NOT NULL THEN amount_in_unadj / pow( - 10, - decimals_in - ) - ELSE amount_in_unadj - END AS amount_in, - amount_out_unadj, - CASE - WHEN decimals_out IS NOT NULL THEN amount_out_unadj / pow( - 10, - decimals_out - ) - ELSE amount_out_unadj - END AS amount_out, sender, tx_to, event_index, @@ -479,37 +250,18 @@ synthetix_swaps AS ( _inserted_timestamp FROM {{ ref('silver_dex__synthetix_swaps') }} - s - LEFT JOIN ( - SELECT - synth_symbol AS synth_symbol_in, - synth_proxy_address AS token_in, - decimals AS decimals_in - FROM - {{ ref('silver__synthetix_synths_20230404') }} - ) synths_in - ON synths_in.synth_symbol_in = s.symbol_in - LEFT JOIN ( - SELECT - synth_symbol AS synth_symbol_out, - synth_proxy_address AS token_out, - decimals AS decimals_out - FROM - {{ ref('silver__synthetix_synths_20230404') }} - ) synths_out - ON synths_out.synth_symbol_out = s.symbol_out -{% if is_incremental() and 'synthetix_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'synthetix' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -curve_swaps AS ( +curve AS ( SELECT block_number, block_timestamp, @@ -519,66 +271,31 @@ curve_swaps AS ( origin_to_address, contract_address, event_name, - s.tokens_sold AS amount_in_unadj, - s.tokens_bought AS amount_out_unadj, + tokens_sold AS amount_in_unadj, + tokens_bought AS amount_out_unadj, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - COALESCE( - c1.symbol, - s.symbol_in - ) AS token_symbol_in, - COALESCE( - c2.symbol, - s.symbol_out - ) AS token_symbol_out, - NULL AS pool_name, - c1.decimals AS decimals_in, - CASE - WHEN decimals_in IS NOT NULL THEN s.tokens_sold / pow( - 10, - decimals_in - ) - ELSE s.tokens_sold - END AS amount_in, - c2.decimals AS decimals_out, - CASE - WHEN decimals_out IS NOT NULL THEN s.tokens_bought / pow( - 10, - decimals_out - ) - ELSE s.tokens_bought - END AS amount_out, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__curve_swaps') }} - s - LEFT JOIN contracts c1 - ON c1.address = s.token_in - LEFT JOIN contracts c2 - ON c2.address = s.token_out - WHERE - amount_out <> 0 - AND COALESCE( - token_symbol_in, - 'null' - ) <> COALESCE(token_symbol_out, 'null') -{% if is_incremental() and 'curve_swaps' not in var('HEAL_CURATED_MODEL') %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' - FROM - {{ this }} -) +{% if is_incremental() and 'curve' not in var('HEAL_MODELS') %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) {% endif %} ), -beethovenx_swaps AS ( +beethovenx AS ( SELECT block_number, block_timestamp, @@ -587,53 +304,35 @@ beethovenx_swaps AS ( origin_from_address, origin_to_address, contract_address, - NULL AS pool_name, CASE WHEN event_name IS NULL THEN 'Swap' ELSE event_name END AS event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__beethovenx_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'beethovenx_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'beethovenx' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -kyberswap_v1_static_swaps AS ( +kyberswap_v1_static AS ( SELECT block_number, block_timestamp, @@ -643,49 +342,31 @@ kyberswap_v1_static_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1-static' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__kyberswap_v1_static_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'kyberswap_v1_static_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kyberswap_v1_static' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -kyberswap_v2_elastic_swaps AS ( +kyberswap_v2_elastic AS ( SELECT block_number, block_timestamp, @@ -695,49 +376,31 @@ kyberswap_v2_elastic_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__kyberswap_v2_elastic_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'kyberswap_v2_elastic_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kyberswap_v2_elastic' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -fraxswap_swaps AS ( +fraxswap AS ( SELECT block_number, block_timestamp, @@ -747,49 +410,31 @@ fraxswap_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__fraxswap_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'fraxswap_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'fraxswap' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -velodrome_v2_swaps AS ( +velodrome_v2 AS ( SELECT block_number, block_timestamp, @@ -799,49 +444,31 @@ velodrome_v2_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__velodrome_v2_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'velodrome_v2_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'velodrome_v2' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -woofi_swaps AS ( +woofi AS ( SELECT block_number, block_timestamp, @@ -851,71 +478,31 @@ woofi_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - CONCAT( - LEAST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ), - '-', - GREATEST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ) - ) AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__woofi_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'woofi_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'woofi' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -dodo_v2_swaps AS ( +dodo_v2 AS ( SELECT block_number, block_timestamp, @@ -925,644 +512,552 @@ dodo_v2_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__dodo_v2_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'dodo_v2_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dodo_v2' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), --union all standard dex CTEs here (excludes amount_usd) -all_dex_standard AS ( +all_dex AS ( SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - sushi_swaps + sushi UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - univ2_swaps + univ2 UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - synthetix_swaps + synthetix UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - token_symbol_in AS symbol_in, - token_symbol_out AS symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - curve_swaps + curve UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - beethovenx_swaps + beethovenx UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - fraxswap_swaps + fraxswap UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - velodrome_v2_swaps + velodrome_v2 UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - hashflow_swaps + hashflow UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - hashflow_v3_swaps + hashflow_v3 UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - CASE - WHEN pool_name = 'WETH-WETH' THEN 'ETH-WETH' - ELSE pool_name - END AS pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - woofi_swaps + woofi UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - kyberswap_v1_static_swaps + kyberswap_v1_static UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - kyberswap_v2_elastic_swaps + kyberswap_v2_elastic UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, - amount_out_unadj, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - dodo_v2_swaps + dodo_v2 + UNION ALL + SELECT + * + FROM + velodrome + UNION ALL + SELECT + * + FROM + univ3 ), ---union all non-standard dex CTEs here (excludes amount_usd) -all_dex_custom AS ( +complete_dex_swaps AS ( SELECT - block_number, - block_timestamp, - tx_hash, + s.block_number, + s.block_timestamp, + s.tx_hash, origin_function_signature, origin_from_address, origin_to_address, - contract_address, - pool_name, + s.contract_address, event_name, - amount_in_unadj, - amount_in, - amount_in_usd, - amount_out_unadj, - amount_out, - amount_out_usd, - sender, - tx_to, - event_index, - platform, - version, token_in, - token_out, - symbol_in, - symbol_out, - _log_id, - _inserted_timestamp - FROM - velodrome_swaps - UNION ALL - SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, + c1.token_decimals AS decimals_in, + c1.token_symbol AS symbol_in, amount_in_unadj, - amount_in, - amount_in_usd, - amount_out_unadj, - amount_out, - amount_out_usd, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - _log_id, - _inserted_timestamp - FROM - univ3_swaps -), -FINAL AS ( - SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, CASE - WHEN s.decimals_in IS NOT NULL THEN ROUND( - amount_in * p1.price, - 2 - ) + WHEN decimals_in IS NULL THEN amount_in_unadj + ELSE (amount_in_unadj / pow(10, decimals_in)) + END AS amount_in, + CASE + WHEN decimals_in IS NOT NULL THEN amount_in * p1.price ELSE NULL END AS amount_in_usd, + token_out, + c2.token_decimals AS decimals_out, + c2.token_symbol AS symbol_out, amount_out_unadj, - amount_out, CASE - WHEN s.decimals_out IS NOT NULL THEN ROUND( - amount_out * p2.price, - 2 - ) + WHEN decimals_out IS NULL THEN amount_out_unadj + ELSE (amount_out_unadj / pow(10, decimals_out)) + END AS amount_out, + CASE + WHEN decimals_out IS NOT NULL THEN amount_out * p2.price ELSE NULL END AS amount_out_usd, + CASE + WHEN lp.pool_name IS NULL THEN CONCAT( + LEAST( + COALESCE( + symbol_in, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + symbol_out, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) + ), + '-', + GREATEST( + COALESCE( + symbol_in, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + symbol_out, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) + ) + ) + ELSE lp.pool_name + END AS pool_name, sender, tx_to, event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - _log_id, - _inserted_timestamp + s.platform, + s.version, + s._log_id, + s._inserted_timestamp FROM - all_dex_standard s - LEFT JOIN prices p1 + all_dex s + LEFT JOIN {{ ref('silver__contracts') }} + c1 + ON s.token_in = c1.contract_address + LEFT JOIN {{ ref('silver__contracts') }} + c2 + ON s.token_out = c2.contract_address + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p1 ON s.token_in = p1.token_address AND DATE_TRUNC( 'hour', block_timestamp ) = p1.hour - LEFT JOIN prices p2 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p2 ON s.token_out = p2.token_address AND DATE_TRUNC( 'hour', block_timestamp ) = p2.hour - UNION ALL + LEFT JOIN {{ ref('silver_dex__complete_dex_liquidity_pools') }} + lp + ON s.contract_address = lp.pool_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( SELECT - block_number, - block_timestamp, - tx_hash, + t0.block_number, + t0.block_timestamp, + t0.tx_hash, origin_function_signature, origin_from_address, origin_to_address, - contract_address, - pool_name, + t0.contract_address, event_name, + token_in, + c1.token_decimals AS decimals_in, + c1.token_symbol AS symbol_in, amount_in_unadj, - amount_in, - ROUND( - amount_in_usd, - 2 - ) AS amount_in_usd, + CASE + WHEN c1.token_decimals IS NULL THEN amount_in_unadj + ELSE (amount_in_unadj / pow(10, c1.token_decimals)) + END AS amount_in_heal, + CASE + WHEN c1.token_decimals IS NOT NULL THEN amount_in_heal * p1.price + ELSE NULL + END AS amount_in_usd_heal, + token_out, + c2.token_decimals AS decimals_out, + c2.token_symbol AS symbol_out, amount_out_unadj, - amount_out, - ROUND( - amount_out_usd, - 2 - ) AS amount_out_usd, + CASE + WHEN c2.token_decimals IS NULL THEN amount_out_unadj + ELSE (amount_out_unadj / pow(10, c2.token_decimals)) + END AS amount_out_heal, + CASE + WHEN c2.token_decimals IS NOT NULL THEN amount_out_heal * p2.price + ELSE NULL + END AS amount_out_usd_heal, + CASE + WHEN lp.pool_name IS NULL THEN CONCAT( + LEAST( + COALESCE( + c1.token_symbol, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + c2.token_symbol, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) + ), + '-', + GREATEST( + COALESCE( + c1.token_symbol, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + c2.token_symbol, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) + ) + ) + ELSE lp.pool_name + END AS pool_name_heal, sender, tx_to, event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - _log_id, - _inserted_timestamp + t0.platform, + t0.version, + t0._log_id, + t0._inserted_timestamp FROM - all_dex_custom C -) + {{ this }} + t0 + LEFT JOIN {{ ref('silver__contracts') }} + c1 + ON t0.token_in = c1.contract_address + LEFT JOIN {{ ref('silver__contracts') }} + c2 + ON t0.token_out = c2.contract_address + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p1 + ON t0.token_in = p1.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p1.hour + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p2 + ON t0.token_out = p2.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p2.hour + LEFT JOIN {{ ref('silver_dex__complete_dex_liquidity_pools') }} + lp + ON t0.contract_address = lp.pool_address + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform, + '-', + t1.version + ) + FROM + {{ this }} + t1 + WHERE + t1.decimals_in IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t1.token_in) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t2.block_number, + '-', + t2.platform, + '-', + t2.version + ) + FROM + {{ this }} + t2 + WHERE + t2.decimals_out IS NULL + AND t2._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t2.token_out) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t3.block_number, + '-', + t3.platform, + '-', + t3.version + ) + FROM + {{ this }} + t3 + WHERE + t3.amount_in_usd IS NULL + AND t3._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t3.token_in + AND p.hour = DATE_TRUNC( + 'hour', + t3.block_timestamp + ) + ) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t4.block_number, + '-', + t4.platform, + '-', + t4.version + ) + FROM + {{ this }} + t4 + WHERE + t4.amount_out_usd IS NULL + AND t4._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t4.token_out + AND p.hour = DATE_TRUNC( + 'hour', + t4.block_timestamp + ) + ) + GROUP BY + 1 + ) + ), + {% endif %} + + FINAL AS ( + SELECT + * + FROM + complete_dex_swaps + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL SELECT - f.block_number, - f.block_timestamp, - f.tx_hash, + block_number, + block_timestamp, + tx_hash, origin_function_signature, origin_from_address, origin_to_address, - f.contract_address, - CASE - WHEN f.pool_name IS NULL THEN p.pool_name - ELSE f.pool_name - END AS pool_name, + contract_address, event_name, + token_in, + decimals_in, + symbol_in, amount_in_unadj, - amount_in, - CASE - WHEN amount_out_usd IS NULL - OR ABS((amount_in_usd - amount_out_usd) / NULLIF(amount_out_usd, 0)) > 0.75 - OR ABS((amount_in_usd - amount_out_usd) / NULLIF(amount_in_usd, 0)) > 0.75 THEN NULL - ELSE amount_in_usd - END AS amount_in_usd, + amount_in_heal AS amount_in, + amount_in_usd_heal AS amount_in_usd, + token_out, + decimals_out, + symbol_out, amount_out_unadj, - amount_out, - CASE - WHEN amount_in_usd IS NULL - OR ABS((amount_out_usd - amount_in_usd) / NULLIF(amount_in_usd, 0)) > 0.75 - OR ABS((amount_out_usd - amount_in_usd) / NULLIF(amount_out_usd, 0)) > 0.75 THEN NULL - ELSE amount_out_usd - END AS amount_out_usd, + amount_out_heal AS amount_out, + amount_out_usd_heal AS amount_out_usd, + pool_name_heal AS pool_name, sender, tx_to, event_index, - f.platform, - f.version, + platform, + version, + _log_id, + _inserted_timestamp +FROM + heal_model +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + pool_name, + event_name, + amount_in_unadj, + amount_in, + amount_in_usd, + amount_out_unadj, + amount_out, + amount_out_usd, + sender, + tx_to, + event_index, + platform, + version, token_in, token_out, symbol_in, symbol_out, - f._log_id, - f._inserted_timestamp, + decimals_in, + decimals_out, + _log_id, + _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( - ['f.tx_hash','f.event_index'] + ['tx_hash','event_index'] ) }} AS complete_dex_swaps_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - FINAL f - LEFT JOIN {{ ref('silver_dex__complete_dex_liquidity_pools') }} - p - ON f.contract_address = p.pool_address + FINAL qualify (ROW_NUMBER() over (PARTITION BY _log_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/defi/dex/sushi/silver_dex__sushi_pools.sql b/models/silver/defi/dex/sushi/silver_dex__sushi_pools.sql index 54a0977f..c3fce622 100644 --- a/models/silver/defi/dex/sushi/silver_dex__sushi_pools.sql +++ b/models/silver/defi/dex/sushi/silver_dex__sushi_pools.sql @@ -27,6 +27,7 @@ WITH pool_creation AS ( '0x1b02da8cb0d097eb8d57a175b88c7d8b47997506' ) AND topics [0] :: STRING = '0xe469f9471ac1d98222517eb2cdff1ef4df5f7880269173bb782bb78e499d9de3' --DeployPool + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/synthetix/silver_dex__synthetix_swaps.sql b/models/silver/defi/dex/synthetix/silver_dex__synthetix_swaps.sql index ec3f0e86..b00cf1f3 100644 --- a/models/silver/defi/dex/synthetix/silver_dex__synthetix_swaps.sql +++ b/models/silver/defi/dex/synthetix/silver_dex__synthetix_swaps.sql @@ -4,7 +4,6 @@ unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], tags = ['curated','reorg'] - ) }} WITH swaps_base AS ( @@ -26,16 +25,26 @@ WITH swaps_base AS ( utils.udf_hex_to_int( segmented_data [3] :: STRING ) :: INTEGER AS amount_out_unadj, - REGEXP_REPLACE(utils.udf_hex_to_string( - segmented_data [0] :: STRING - ),'[^a-zA-Z0-9]+') AS symbol_in, - REGEXP_REPLACE(utils.udf_hex_to_string( - segmented_data [2] :: STRING - ),'[^a-zA-Z0-9]+') AS symbol_out, - CONCAT('0x', SUBSTR(segmented_data [4] :: STRING, 25, 40))AS tx_to, + REGEXP_REPLACE( + utils.udf_hex_to_string( + segmented_data [0] :: STRING + ), + '[^a-zA-Z0-9]+' + ) AS symbol_in, + REGEXP_REPLACE( + utils.udf_hex_to_string( + segmented_data [2] :: STRING + ), + '[^a-zA-Z0-9]+' + ) AS symbol_out, + CONCAT('0x', SUBSTR(segmented_data [4] :: STRING, 25, 40)) AS tx_to, event_index, 'synthetix' AS platform, - CONCAT(tx_hash,'-',event_index) AS _log_id, + CONCAT( + tx_hash, + '-', + event_index + ) AS _log_id, _inserted_timestamp FROM {{ ref('silver__logs') }} @@ -43,7 +52,8 @@ WITH swaps_base AS ( contract_address IN ( '0x8700daec35af8ff88c16bdf0418774cb3d7599b4' ) - AND topics[0] = '0x65b6972c94204d84cffd3a95615743e31270f04fdf251f3dccc705cfbad44776' + AND topics [0] = '0x65b6972c94204d84cffd3a95615743e31270f04fdf251f3dccc705cfbad44776' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -53,11 +63,9 @@ AND _inserted_timestamp >= ( {{ this }} ) {% endif %} - ) - SELECT - block_number, + block_number, block_timestamp, tx_hash, origin_function_signature, @@ -68,6 +76,8 @@ SELECT sender, amount_in_unadj, amount_out_unadj, + token_in, + token_out, symbol_in, symbol_out, tx_to, @@ -76,4 +86,22 @@ SELECT _log_id, _inserted_timestamp FROM - swaps_base + swaps_base s + LEFT JOIN ( + SELECT + synth_symbol AS synth_symbol_in, + synth_proxy_address AS token_in, + decimals AS decimals_in + FROM + {{ ref('silver__synthetix_synths_20230404') }} + ) sc1 + ON sc1.synth_symbol_in = s.symbol_in + LEFT JOIN ( + SELECT + synth_symbol AS synth_symbol_out, + synth_proxy_address AS token_out, + decimals AS decimals_out + FROM + {{ ref('silver__synthetix_synths_20230404') }} + ) sc2 + ON sc2.synth_symbol_out = s.symbol_out diff --git a/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql b/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql index 1a3748ef..333c2ce2 100644 --- a/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql +++ b/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql @@ -27,6 +27,7 @@ WITH pool_creation AS ( WHERE contract_address = LOWER('0x0c3c1c532F1e39EdF36BE9Fe0bE1410313E074Bf') AND topics [0] :: STRING = '0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9' --PairCreated + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql b/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql index 40b9c6c4..1d8d1405 100644 --- a/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql +++ b/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql @@ -32,6 +32,7 @@ WITH created_pools AS ( WHERE topics [0] = '0x783cca1c0412dd0d695e784568c96da2e9c22ff989357a2e8b1d9b2b4e6b7118' AND contract_address = '0x1f98431c8ad98523631ae4a59f267346ea31f984' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -60,6 +61,7 @@ initial_info AS ( {{ ref('silver__logs') }} WHERE topics [0] :: STRING = '0x98636036cb66a9c19a37435efc1e90142190214e8abeb821bdba3f2990dd4c95' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/velodrome/silver_dex__velodrome_v2_pools.sql b/models/silver/defi/dex/velodrome/silver_dex__velodrome_v2_pools.sql index 8d7c1c49..5b80083a 100644 --- a/models/silver/defi/dex/velodrome/silver_dex__velodrome_v2_pools.sql +++ b/models/silver/defi/dex/velodrome/silver_dex__velodrome_v2_pools.sql @@ -31,6 +31,7 @@ WHERE AND contract_address = '0xf1046053aa5682b4f9a81b5481394da16be5ff5a' AND pool_address <> '0x585af0b397ac42dbef7f18395426bf878634f18d' -- velo v1/v2 converter AND block_number > 100000000 + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql b/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql index a95c698c..d560e8b6 100644 --- a/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql +++ b/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql @@ -63,6 +63,7 @@ WITH router_swaps_base AS ( '0xeaf1ac8e89ea0ae13e0f03634a4ff23502527024' --v2 ) AND topics [0] :: STRING = '0x27c98e911efdd224f4002f6cd831c3ad0d2759ee176f9ee8466d95826af22a1c' --WooRouterSwap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -127,6 +128,7 @@ swaps_base AS ( FROM router_swaps_base ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql index 543027f9..3564d45b 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql @@ -1,9 +1,10 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} WITH aave AS ( @@ -30,13 +31,13 @@ WITH aave AS ( FROM {{ ref('silver__aave_borrows') }} A -{% if is_incremental() and 'aave' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'aave' not in var('HEAL_MODELS') %} WHERE A._inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -65,13 +66,13 @@ granary AS ( FROM {{ ref('silver__granary_borrows') }} A -{% if is_incremental() and 'granary' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'granary' not in var('HEAL_MODELS') %} WHERE A._inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -100,13 +101,13 @@ exactly AS ( FROM {{ ref('silver__exactly_borrows') }} A -{% if is_incremental() and 'exactly' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'exactly' not in var('HEAL_MODELS') %} WHERE A._inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -135,13 +136,13 @@ sonne AS ( FROM {{ ref('silver__sonne_borrows') }} A -{% if is_incremental() and 'sonne' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'sonne' not in var('HEAL_MODELS') %} WHERE A._inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -170,13 +171,13 @@ tarot AS ( FROM {{ ref('silver__tarot_borrows') }} A -{% if is_incremental() and 'tarot' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'tarot' not in var('HEAL_MODELS') %} WHERE A._inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -208,7 +209,7 @@ borrow_union AS ( FROM tarot ), -FINAL AS ( +complete_lending_borrows AS ( SELECT tx_hash, block_number, @@ -242,8 +243,126 @@ FINAL AS ( 'hour', block_timestamp ) = p.hour - LEFT JOIN {{ ref('silver__contracts') }} C - ON b.token_address = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + borrower, + protocol_market, + t0.token_address, + t0.token_symbol, + amount_unadj, + amount, + ROUND( + amount * p.price, + 2 + ) AS amount_usd_heal, + platform, + t0.blockchain, + t0._LOG_ID, + t0._INSERTED_TIMESTAMP + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.token_address = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.token_address + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_borrows + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + borrower, + protocol_market, + token_address, + token_symbol, + amount_unadj, + amount, + amount_usd_heal AS amount_usd, + platform, + blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP +FROM + heal_model +{% endif %} ) SELECT *, diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql index e09e263a..b87c80bd 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql @@ -1,9 +1,10 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} WITH aave AS ( @@ -30,11 +31,11 @@ WITH aave AS ( FROM {{ ref('silver__aave_deposits') }} -{% if is_incremental() and 'aave' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'aave' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -63,11 +64,11 @@ granary AS ( FROM {{ ref('silver__granary_deposits') }} -{% if is_incremental() and 'granary' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'granary' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -96,11 +97,11 @@ exactly AS ( FROM {{ ref('silver__exactly_deposits') }} -{% if is_incremental() and 'exactly' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'exactly' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -129,11 +130,11 @@ sonne AS ( FROM {{ ref('silver__sonne_deposits') }} -{% if is_incremental() and 'sonne' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'sonne' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -162,11 +163,11 @@ tarot AS ( FROM {{ ref('silver__tarot_deposits') }} -{% if is_incremental() and 'tarot' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'tarot' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -198,7 +199,7 @@ deposit_union AS ( FROM tarot ), -FINAL AS ( +complete_lending_deposits AS ( SELECT tx_hash, block_number, @@ -239,8 +240,126 @@ FINAL AS ( 'hour', block_timestamp ) = p.hour - LEFT JOIN {{ ref('silver__contracts') }} C - ON A.token_address = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + protocol_market, + depositor, + t0.token_address, + t0.token_symbol, + amount_unadj, + amount, + ROUND( + amount * p.price, + 2 + ) AS amount_usd_heal, + platform, + t0.blockchain, + t0._LOG_ID, + t0._INSERTED_TIMESTAMP + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.token_address = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.token_address + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_deposits + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + protocol_market, + depositor, + token_address, + token_symbol, + amount_unadj, + amount, + amount_usd_heal AS amount_usd, + platform, + blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP +FROM + heal_model +{% endif %} ) SELECT *, diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql index 9780f6b0..240679e4 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql @@ -1,12 +1,13 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} -WITH aave as ( +WITH aave AS ( SELECT tx_hash, @@ -33,18 +34,17 @@ WITH aave as ( FROM {{ ref('silver__aave_flashloans') }} -{% if is_incremental() and 'aave' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'aave' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - -granary as ( +granary AS ( SELECT tx_hash, block_number, @@ -63,25 +63,24 @@ granary as ( initiator_address, target_address, platform, - symbol AS token_symbol, + symbol, blockchain, _LOG_ID, _INSERTED_TIMESTAMP FROM {{ ref('silver__granary_flashloans') }} -{% if is_incremental() and 'granary' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'granary' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - -flashloan_union as ( +flashloan_union AS ( SELECT * FROM @@ -92,8 +91,7 @@ flashloan_union as ( FROM granary ), - -FINAL AS ( +complete_lending_flashloans AS ( SELECT tx_hash, block_number, @@ -108,7 +106,7 @@ FINAL AS ( initiator_address AS initiator, target_address AS target, f.token_address AS flashloan_token, - token_symbol AS flashloan_token_symbol, + f.symbol AS flashloan_token_symbol, flashloan_amount_unadj, flashloan_amount, ROUND( @@ -134,8 +132,179 @@ FINAL AS ( 'hour', block_timestamp ) = p.hour - LEFT JOIN {{ ref('silver__contracts') }} C - ON f.token_address = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + protocol_market, + initiator, + target, + flashloan_token, + flashloan_token_symbol, + flashloan_amount_unadj, + flashloan_amount, + ROUND( + flashloan_amount * p.price, + 2 + ) AS flashloan_amount_usd_heal, + premium_amount_unadj, + premium_amount, + ROUND( + premium_amount * p.price, + 2 + ) AS premium_amount_usd_heal, + platform, + t0.blockchain, + t0._LOG_ID, + t0._INSERTED_TIMESTAMP + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.flashloan_token = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.flashloan_amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.flashloan_token + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t2.block_number, + '-', + t2.platform + ) + FROM + {{ this }} + t2 + WHERE + t2.premium_amount_usd IS NULL + AND t2._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t2.flashloan_token + AND p.hour = DATE_TRUNC( + 'hour', + t2.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_flashloans + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + protocol_market, + initiator, + target, + flashloan_token, + flashloan_token_symbol, + flashloan_amount_unadj, + flashloan_amount, + flashloan_amount_usd_heal AS flashloan_amount_usd, + premium_amount_unadj, + premium_amount, + premium_amount_usd_heal AS premium_amount_usd, + platform, + blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP +FROM + heal_model +{% endif %} ) SELECT *, diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql index c84add53..823f82f3 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql @@ -1,89 +1,89 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} -WITH aave as ( -SELECT - tx_hash, - block_number, - block_timestamp, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, - contract_address, - liquidator, - borrower, - amount_unadj, - amount AS liquidated_amount, - NULL AS liquidated_amount_usd, - collateral_aave_token AS protocol_collateral_asset, - collateral_asset, - collateral_token_symbol AS collateral_asset_symbol, - debt_asset, - debt_token_symbol AS debt_asset_symbol, - 'Aave V3' AS platform, - 'optimism' AS blockchain, - _LOG_ID, - _INSERTED_TIMESTAMP -FROM - {{ ref('silver__aave_liquidations') }} +WITH aave AS ( -{% if is_incremental() and 'aave' not in var('HEAL_CURATED_MODEL') %} + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + liquidator, + borrower, + amount_unadj, + amount AS liquidated_amount, + NULL AS liquidated_amount_usd, + collateral_aave_token AS protocol_collateral_asset, + collateral_asset, + collateral_token_symbol AS collateral_asset_symbol, + debt_asset, + debt_token_symbol AS debt_asset_symbol, + 'Aave V3' AS platform, + 'optimism' AS blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__aave_liquidations') }} + +{% if is_incremental() and 'aave' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), +granary AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + liquidator, + borrower, + amount_unadj, + amount AS liquidated_amount, + NULL AS liquidated_amount_usd, + collateral_granary_token AS protocol_collateral_asset, + collateral_asset, + collateral_token_symbol AS collateral_asset_symbol, + debt_asset, + debt_token_symbol AS debt_asset_symbol, + 'Granary' AS platform, + 'optimism' AS blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__granary_liquidations') }} -granary as ( -SELECT - tx_hash, - block_number, - block_timestamp, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, - contract_address, - liquidator, - borrower, - amount_unadj, - amount AS liquidated_amount, - NULL AS liquidated_amount_usd, - collateral_granary_token AS protocol_collateral_asset, - collateral_asset, - collateral_token_symbol AS collateral_asset_symbol, - debt_asset, - debt_token_symbol AS debt_asset_symbol, - 'Granary' AS platform, - 'optimism' AS blockchain, - _LOG_ID, - _INSERTED_TIMESTAMP -FROM - {{ ref('silver__granary_liquidations') }} - -{% if is_incremental() and 'granary' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'granary' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - -exactly as ( - SELECT +exactly AS ( + SELECT tx_hash, block_number, block_timestamp, @@ -110,18 +110,17 @@ exactly as ( {{ ref('silver__exactly_liquidations') }} l -{% if is_incremental() and 'exactly' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'exactly' not in var('HEAL_MODELS') %} WHERE l._inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - -sonne as ( +sonne AS ( SELECT tx_hash, block_number, @@ -149,18 +148,17 @@ sonne as ( {{ ref('silver__sonne_liquidations') }} l -{% if is_incremental() and 'sonne' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'sonne' not in var('HEAL_MODELS') %} WHERE l._inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - -tarot as ( +tarot AS ( SELECT tx_hash, block_number, @@ -188,78 +186,43 @@ tarot as ( {{ ref('silver__tarot_liquidations') }} l -{% if is_incremental() and 'tarot' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'tarot' not in var('HEAL_MODELS') %} WHERE l._inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - -liquidation_union as ( - SELECT - * - FROM - aave - UNION ALL - SELECT - * - FROM - granary - UNION ALL - SELECT - * - FROM - exactly - UNION ALL - SELECT - * - FROM - sonne - UNION ALL - SELECT - * - FROM - tarot -), - -contracts AS ( +liquidation_union AS ( SELECT * FROM - {{ ref('silver__contracts') }} C - WHERE - C.contract_address IN ( - SELECT - DISTINCT(collateral_asset) AS asset - FROM - liquidation_union - ) -), -prices AS ( + aave + UNION ALL SELECT * FROM - {{ ref('price__ez_prices_hourly') }} - p - WHERE - token_address IN ( - SELECT - DISTINCT(collateral_asset) AS asset - FROM - liquidation_union - ) - AND HOUR > ( - SELECT - MIN(block_timestamp) - FROM - liquidation_union - ) + granary + UNION ALL + SELECT + * + FROM + exactly + UNION ALL + SELECT + * + FROM + sonne + UNION ALL + SELECT + * + FROM + tarot ), -FINAL AS ( +complete_lending_liquidations AS ( SELECT tx_hash, block_number, @@ -271,7 +234,10 @@ FINAL AS ( A.contract_address, CASE WHEN platform = 'Sonne' THEN 'LiquidateBorrow' - WHEN platform IN ('Tarot','Exactly') THEN 'Liquidate' + WHEN platform IN ( + 'Tarot', + 'Exactly' + ) THEN 'Liquidate' ELSE 'LiquidationCall' END AS event_name, liquidator, @@ -282,9 +248,9 @@ FINAL AS ( amount_unadj, liquidated_amount AS amount, ROUND( - liquidated_amount * p.price, - 2 - ) AS amount_usd, + liquidated_amount * p.price, + 2 + ) AS amount_usd, debt_asset AS debt_token, debt_asset_symbol AS debt_token_symbol, platform, @@ -293,23 +259,148 @@ FINAL AS ( A._INSERTED_TIMESTAMP FROM liquidation_union A - LEFT JOIN prices p + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p ON collateral_asset = p.token_address AND DATE_TRUNC( 'hour', block_timestamp ) = p.hour - LEFT JOIN contracts C - ON collateral_asset = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + liquidator, + borrower, + protocol_market, + collateral_token, + collateral_token_symbol, + amount_unadj, + amount, + ROUND( + amount * p.price, + 2 + ) AS amount_usd_heal, + debt_token, + debt_token_symbol, + platform, + t0.blockchain, + t0._LOG_ID, + t0._INSERTED_TIMESTAMP + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.collateral_token = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.collateral_token + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_liquidations + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + liquidator, + borrower, + protocol_market, + collateral_token, + collateral_token_symbol, + amount_unadj, + amount, + amount_usd_heal AS amount_usd, + debt_token, + debt_token_symbol, + platform, + blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP +FROM + heal_model +{% endif %} ) SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash','event_index'] - ) }} AS complete_lending_liquidations_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index'] + ) }} AS complete_lending_liquidations_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id ORDER BY diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql index 9e6a8833..b9aa5b55 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql @@ -1,12 +1,13 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} -WITH aave as ( +WITH aave AS ( SELECT tx_hash, @@ -30,19 +31,18 @@ WITH aave as ( _INSERTED_TIMESTAMP FROM {{ ref('silver__aave_repayments') }} -{% if is_incremental() and 'aave' not in var('HEAL_CURATED_MODEL') %} + +{% if is_incremental() and 'aave' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - -granary as ( - +granary AS ( SELECT tx_hash, block_number, @@ -66,18 +66,17 @@ granary as ( FROM {{ ref('silver__granary_repayments') }} - {% if is_incremental() and 'granary' not in var('HEAL_CURATED_MODEL') %} - WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' - FROM - {{ this }} - ) - {% endif %} +{% if is_incremental() and 'granary' not in var('HEAL_MODELS') %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) +{% endif %} ), - -exactly as ( +exactly AS ( SELECT tx_hash, block_number, @@ -101,18 +100,17 @@ exactly as ( FROM {{ ref('silver__exactly_repayments') }} - {% if is_incremental() and 'exactly' not in var('HEAL_CURATED_MODEL') %} - WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' - FROM - {{ this }} - ) - {% endif %} +{% if is_incremental() and 'exactly' not in var('HEAL_MODELS') %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) +{% endif %} ), - -sonne as ( +sonne AS ( SELECT tx_hash, block_number, @@ -136,81 +134,77 @@ sonne as ( FROM {{ ref('silver__sonne_repayments') }} - {% if is_incremental() and 'sonne' not in var('HEAL_CURATED_MODEL') %} - WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' - FROM - {{ this }} - ) - {% endif %} +{% if is_incremental() and 'sonne' not in var('HEAL_MODELS') %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) +{% endif %} ), - -tarot as ( - +tarot AS ( SELECT - tx_hash, - block_number, - block_timestamp, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, - contract_address, - repay_contract_address AS token_address, - token_address AS protocol_market, - amount_unadj, - amount, - repay_contract_symbol AS token_symbol, - payer AS payer_address, - borrower, - platform, - 'optimism' AS blockchain, - _LOG_ID, - _INSERTED_TIMESTAMP + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + repay_contract_address AS token_address, + token_address AS protocol_market, + amount_unadj, + amount, + repay_contract_symbol AS token_symbol, + payer AS payer_address, + borrower, + platform, + 'optimism' AS blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP FROM {{ ref('silver__tarot_repayments') }} -{% if is_incremental() and 'tarot' not in var('HEAL_CURATED_MODEL') %} - WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' - FROM - {{ this }} - ) - {% endif %} - +{% if is_incremental() and 'tarot' not in var('HEAL_MODELS') %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) +{% endif %} ), - -repayments_union as ( - SELECT - * - FROM - aave - UNION ALL - SELECT - * - FROM - granary - UNION ALL - SELECT - * - FROM - exactly - UNION ALL - SELECT - * - FROM - sonne - UNION ALL - SELECT - * - FROM - tarot +repayments_union AS ( + SELECT + * + FROM + aave + UNION ALL + SELECT + * + FROM + granary + UNION ALL + SELECT + * + FROM + exactly + UNION ALL + SELECT + * + FROM + sonne + UNION ALL + SELECT + * + FROM + tarot ), -FINAL AS ( +complete_lending_repayments AS ( SELECT tx_hash, block_number, @@ -248,17 +242,137 @@ FINAL AS ( 'hour', block_timestamp ) = p.hour - LEFT JOIN {{ ref('silver__contracts') }} C - ON A.token_address = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + protocol_market, + payer, + borrower, + t0.token_address, + t0.token_symbol, + amount_unadj, + amount, + ROUND( + amount * p.price, + 2 + ) AS amount_usd_heal, + platform, + t0.blockchain, + t0._LOG_ID, + t0._INSERTED_TIMESTAMP + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.token_address = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.token_address + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_repayments + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + protocol_market, + payer, + borrower, + token_address, + token_symbol, + amount_unadj, + amount, + amount_usd_heal AS amount_usd, + platform, + blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP +FROM + heal_model +{% endif %} ) SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash','event_index'] - ) }} AS complete_lending_repayments_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index'] + ) }} AS complete_lending_repayments_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id ORDER BY diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql index 212dde9c..3165e365 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql @@ -1,12 +1,13 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} -WITH aave as ( +WITH aave AS ( SELECT tx_hash, @@ -30,21 +31,19 @@ WITH aave as ( FROM {{ ref('silver__aave_withdraws') }} -{% if is_incremental() and 'aave' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'aave' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - -granary as ( - +granary AS ( SELECT tx_hash, block_number, @@ -67,20 +66,19 @@ granary as ( FROM {{ ref('silver__granary_withdraws') }} - {% if is_incremental() and 'granary' not in var('HEAL_CURATED_MODEL') %} - WHERE - _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) - INTERVAL '36 hours' - FROM - {{ this }} - ) - {% endif %} +{% if is_incremental() and 'granary' not in var('HEAL_MODELS') %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) +{% endif %} ), - -exactly as ( +exactly AS ( SELECT tx_hash, block_number, @@ -103,21 +101,19 @@ exactly as ( FROM {{ ref('silver__exactly_withdraws') }} - {% if is_incremental() and 'exactly' not in var('HEAL_CURATED_MODEL') %} - WHERE - _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) - INTERVAL '36 hours' - FROM - {{ this }} - ) - {% endif %} +{% if is_incremental() and 'exactly' not in var('HEAL_MODELS') %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) +{% endif %} ), - -sonne as ( - +sonne AS ( SELECT tx_hash, block_number, @@ -140,21 +136,19 @@ sonne as ( FROM {{ ref('silver__sonne_withdraws') }} - {% if is_incremental() and 'sonne' not in var('HEAL_CURATED_MODEL') %} - WHERE - _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) - INTERVAL '36 hours' - FROM - {{ this }} - ) - {% endif %} +{% if is_incremental() and 'sonne' not in var('HEAL_MODELS') %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) +{% endif %} ), - -tarot as ( - +tarot AS ( SELECT tx_hash, block_number, @@ -177,21 +171,19 @@ tarot as ( FROM {{ ref('silver__tarot_withdraws') }} -{% if is_incremental() and 'tarot' not in var('HEAL_CURATED_MODEL') %} - WHERE - _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) - INTERVAL '36 hours' - FROM - {{ this }} - ) - {% endif %} - +{% if is_incremental() and 'tarot' not in var('HEAL_MODELS') %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) +{% endif %} ), - -withdraws_union as ( +withdraws_union AS ( SELECT * FROM @@ -217,8 +209,7 @@ withdraws_union as ( FROM tarot ), - -FINAL AS ( +complete_lending_withdraws AS ( SELECT tx_hash, block_number, @@ -229,7 +220,10 @@ FINAL AS ( origin_function_signature, A.contract_address, CASE - WHEN platform IN ('Tarot','Sonne') THEN 'Redeem' + WHEN platform IN ( + 'Tarot', + 'Sonne' + ) THEN 'Redeem' ELSE 'Withdraw' END AS event_name, protocol_market, @@ -255,8 +249,126 @@ FINAL AS ( 'hour', block_timestamp ) = p.hour - LEFT JOIN {{ ref('silver__contracts') }} C - ON A.token_address = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + protocol_market, + depositor, + t0.token_address, + t0.token_symbol, + amount_unadj, + amount, + ROUND( + amount * p.price, + 2 + ) AS amount_usd_heal, + platform, + t0.blockchain, + t0._log_id, + t0._inserted_timestamp + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.token_address = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.token_address + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_withdraws + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + protocol_market, + depositor, + token_address, + token_symbol, + amount_unadj, + amount, + amount_usd_heal AS amount_usd, + platform, + blockchain, + _log_id, + _inserted_timestamp +FROM + heal_model +{% endif %} ) SELECT *, diff --git a/models/silver/defi/lending/exactly/silver__exactly_borrows.sql b/models/silver/defi/lending/exactly/silver__exactly_borrows.sql index 23dd71bb..972ce998 100644 --- a/models/silver/defi/lending/exactly/silver__exactly_borrows.sql +++ b/models/silver/defi/lending/exactly/silver__exactly_borrows.sql @@ -51,6 +51,7 @@ sonne_borrows AS ( asset_details ) AND topics [0] :: STRING = '0x96558a334f4759f0e7c423d68c84721860bd8fbf94ddc4e55158ecb125ad04b5' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/exactly/silver__exactly_deposits.sql b/models/silver/defi/lending/exactly/silver__exactly_deposits.sql index f3b7588c..f276f1b4 100644 --- a/models/silver/defi/lending/exactly/silver__exactly_deposits.sql +++ b/models/silver/defi/lending/exactly/silver__exactly_deposits.sql @@ -52,6 +52,7 @@ exactly_deposits AS ( asset_details ) AND topics [0] :: STRING = '0xdcbc1c05240f31ff3ad067ef1ee35ce4997762752e3a095284754544f4c709d7' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/exactly/silver__exactly_liquidations.sql b/models/silver/defi/lending/exactly/silver__exactly_liquidations.sql index b18fc705..ba0ce974 100644 --- a/models/silver/defi/lending/exactly/silver__exactly_liquidations.sql +++ b/models/silver/defi/lending/exactly/silver__exactly_liquidations.sql @@ -55,6 +55,7 @@ exactly_liquidations AS ( asset_details ) AND topics [0] :: STRING = '0x67bb48f97d82192848c24158abf58ec614777328e19655e0a219652b773fd1db' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/exactly/silver__exactly_repayments.sql b/models/silver/defi/lending/exactly/silver__exactly_repayments.sql index 133711f8..a9408a2a 100644 --- a/models/silver/defi/lending/exactly/silver__exactly_repayments.sql +++ b/models/silver/defi/lending/exactly/silver__exactly_repayments.sql @@ -50,6 +50,7 @@ exactly_repayments AS ( asset_details ) AND topics [0] :: STRING = '0xe4a1ae657f49cb1fb1c7d3a94ae6093565c4c8c0e03de488f79c377c3c3a24e0' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/exactly/silver__exactly_withdraws.sql b/models/silver/defi/lending/exactly/silver__exactly_withdraws.sql index d942e43c..86a821b6 100644 --- a/models/silver/defi/lending/exactly/silver__exactly_withdraws.sql +++ b/models/silver/defi/lending/exactly/silver__exactly_withdraws.sql @@ -50,6 +50,7 @@ exactly_redemptions AS ( asset_details ) AND topics [0] :: STRING = '0xfbde797d201c681b91056529119e0b02407c7bb96a4a2c75c01fc9667232c8db' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/sonne/silver__sonne_borrows.sql b/models/silver/defi/lending/sonne/silver__sonne_borrows.sql index 50c993a7..e9997998 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_borrows.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_borrows.sql @@ -55,6 +55,7 @@ sonne_borrows AS ( asset_details ) AND topics [0] :: STRING = '0x13ed6866d4e1ee6da46f845c46d7e54120883d75c5ea9a2dacc1c4ca8984ab80' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/sonne/silver__sonne_deposits.sql b/models/silver/defi/lending/sonne/silver__sonne_deposits.sql index 7bd4aece..a36d1401 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_deposits.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_deposits.sql @@ -52,6 +52,7 @@ sonne_deposits AS ( asset_details ) AND topics [0] :: STRING = '0x4c209b5fc8ad50758f13e2e1088ba56a560dff690a1c6fef26394f4c03821c4f' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/sonne/silver__sonne_liquidations.sql b/models/silver/defi/lending/sonne/silver__sonne_liquidations.sql index 32f5b730..842985b7 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_liquidations.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_liquidations.sql @@ -54,6 +54,7 @@ sonne_liquidations AS ( asset_details ) AND topics [0] :: STRING = '0x298637f684da70674f26509b10f07ec2fbc77a335ab1e7d6215a4b2484d8bb52' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/sonne/silver__sonne_repayments.sql b/models/silver/defi/lending/sonne/silver__sonne_repayments.sql index 4b7f3247..a9dc7897 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_repayments.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_repayments.sql @@ -50,6 +50,7 @@ sonne_repayments AS ( asset_details ) AND topics [0] :: STRING = '0x1a2a22cb034d26d1854bdc6666a5b91fe25efbbb5dcad3b0355478d6f5c362a1' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/sonne/silver__sonne_withdraws.sql b/models/silver/defi/lending/sonne/silver__sonne_withdraws.sql index 4adb221f..c846c6a0 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_withdraws.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_withdraws.sql @@ -52,6 +52,7 @@ sonne_redemptions AS ( asset_details ) AND topics [0] :: STRING = '0xe5b754fb1abb7f01b499791d0b820ae3b6af3424ac1c59768edb53f4ec31a929' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/tarot/silver__tarot_borrows.sql b/models/silver/defi/lending/tarot/silver__tarot_borrows.sql index 14c5b79d..30823d89 100644 --- a/models/silver/defi/lending/tarot/silver__tarot_borrows.sql +++ b/models/silver/defi/lending/tarot/silver__tarot_borrows.sql @@ -58,6 +58,7 @@ log_pull AS ( asset_details ) AND topics [0] :: STRING = '0x33f3048bd4e6af45e53afb722adfd57dbde82da7e93e44db921fb4b8c6a70c4b' + AND tx_status = 'SUCCESS' AND loan_amount_raw > 0 --borrow and repay in same log event, value in segmented data determines what kind of event {% if is_incremental() %} diff --git a/models/silver/defi/lending/tarot/silver__tarot_deposits.sql b/models/silver/defi/lending/tarot/silver__tarot_deposits.sql index c089db2c..3276c98f 100644 --- a/models/silver/defi/lending/tarot/silver__tarot_deposits.sql +++ b/models/silver/defi/lending/tarot/silver__tarot_deposits.sql @@ -57,6 +57,7 @@ log_pull AS ( asset_details ) AND topics [0] :: STRING = '0x2f00e3cdd69a77be7ed215ec7b2a36784dd158f921fca79ac29deffa353fe6ee' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/tarot/silver__tarot_liquidations.sql b/models/silver/defi/lending/tarot/silver__tarot_liquidations.sql index 9be5977e..a61234ac 100644 --- a/models/silver/defi/lending/tarot/silver__tarot_liquidations.sql +++ b/models/silver/defi/lending/tarot/silver__tarot_liquidations.sql @@ -65,6 +65,7 @@ log_pull as ( contract_address IN (SELECT TOKEN_ADDRESS FROM asset_details) AND topics [0] :: STRING = '0xb0dbe18c6ffdf0da655dd690e77211d379205c497be44c64447c3f5f021b5167' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( SELECT diff --git a/models/silver/defi/lending/tarot/silver__tarot_repayments.sql b/models/silver/defi/lending/tarot/silver__tarot_repayments.sql index 8b75ce60..b1ed017f 100644 --- a/models/silver/defi/lending/tarot/silver__tarot_repayments.sql +++ b/models/silver/defi/lending/tarot/silver__tarot_repayments.sql @@ -59,6 +59,7 @@ log_pull AS ( asset_details ) AND topics [0] :: STRING = '0x33f3048bd4e6af45e53afb722adfd57dbde82da7e93e44db921fb4b8c6a70c4b' + AND tx_status = 'SUCCESS' AND repay_amount_raw > 0 --borrow and repay in same log event, value in segmented data determines what kind of event {% if is_incremental() %} diff --git a/models/silver/defi/lending/tarot/silver__tarot_withdraws.sql b/models/silver/defi/lending/tarot/silver__tarot_withdraws.sql index 10a96fc5..8f31438f 100644 --- a/models/silver/defi/lending/tarot/silver__tarot_withdraws.sql +++ b/models/silver/defi/lending/tarot/silver__tarot_withdraws.sql @@ -51,6 +51,7 @@ log_pull as ( contract_address IN (SELECT TOKEN_ADDRESS FROM asset_details) AND topics [0] :: STRING = '0x3f693fff038bb8a046aa76d9516190ac7444f7d69cf952c4cbdc086fdef2d6fc' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( SELECT diff --git a/models/silver/nft/sales/silver__complete_nft_sales.sql b/models/silver/nft/sales/silver__complete_nft_sales.sql index 4de7c829..09022f84 100644 --- a/models/silver/nft/sales/silver__complete_nft_sales.sql +++ b/models/silver/nft/sales/silver__complete_nft_sales.sql @@ -38,13 +38,13 @@ WITH nft_base_models AS ( FROM {{ ref('silver__quix_sales') }} -{% if is_incremental() and 'quix' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'quix' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -80,13 +80,13 @@ SELECT FROM {{ ref('silver__quix_seaport_sales') }} -{% if is_incremental() and 'quix_seaport' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'quix_seaport' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -122,13 +122,13 @@ SELECT FROM {{ ref('silver__seaport_1_1_sales') }} -{% if is_incremental() and 'seaport_1_1' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'seaport_1_1' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -164,13 +164,13 @@ SELECT FROM {{ ref('silver__seaport_1_4_sales') }} -{% if is_incremental() and 'seaport_1_4' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'seaport_1_4' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -206,13 +206,13 @@ SELECT FROM {{ ref('silver__seaport_1_5_sales') }} -{% if is_incremental() and 'seaport_1_5' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'seaport_1_5' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -248,13 +248,13 @@ SELECT FROM {{ ref('silver__seaport_1_6_sales') }} -{% if is_incremental() and 'seaport_1_6' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'seaport_1_6' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -557,7 +557,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -585,7 +585,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -624,7 +624,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) diff --git a/models/silver/nft/silver__nft_transfers.sql b/models/silver/nft/silver__nft_transfers.sql index 21c86f95..2cf888c6 100644 --- a/models/silver/nft/silver__nft_transfers.sql +++ b/models/silver/nft/silver__nft_transfers.sql @@ -362,7 +362,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} )