diff --git a/README.md b/README.md index 134a8af..da1a51a 100644 --- a/README.md +++ b/README.md @@ -35,35 +35,43 @@ bsc: query_tag: ``` -### Variables +## Variables To control the creation of UDF or SP macros with dbt run: * UPDATE_UDFS_AND_SPS -When True, executes all macros included in the on-run-start hooks within dbt_project.yml on model run as normal -When False, none of the on-run-start macros are executed on model run + * Default values are False + * When True, executes all macros included in the on-run-start hooks within dbt_project.yml on model run as normal + * When False, none of the on-run-start macros are executed on model run -Default values are False + * Usage: `dbt run --vars '{"UPDATE_UDFS_AND_SPS":True}' -m ...` -* Usage: -dbt run --var '{"UPDATE_UDFS_AND_SPS":True}' -m ... +Use a variable to heal a model incrementally: +* HEAL_MODEL + * Default is FALSE (Boolean) + * When FALSE, logic will be negated + * When TRUE, heal logic will apply + * Include `heal` in model tags within the config block for inclusion in the `dbt_run_heal_models` workflow, e.g. `tags = 'heal'` -To reload records in a curated complete table without a full-refresh, such as `silver_bridge.complete_bridge_activity`: -* HEAL_CURATED_MODEL -Default is an empty array [] -When item is included in var array [], incremental logic will be skipped for that CTE / code block -When item is not included in var array [] or does not match specified item in model, incremental logic will apply -Example set up: `{% if is_incremental() and 'axelar' not in var('HEAL_CURATED_MODEL') %}` + * Usage: `dbt run --vars '{"HEAL_MODEL":True}' -m ...` -* Usage: -Single CTE: dbt run --var '{"HEAL_CURATED_MODEL":"axelar"}' -m ... -Multiple CTEs: dbt run --var '{"HEAL_CURATED_MODEL":["axelar","across","celer_cbridge"]}' -m ... +Use a variable to negate incremental logic: +* Example use case: reload records in a curated complete table without a full-refresh, such as `silver_bridge.complete_bridge_activity`: +* HEAL_MODELS + * Default is an empty array [] + * When item is included in var array [], incremental logic will be skipped for that CTE / code block + * When item is not included in var array [] or does not match specified item in model, incremental logic will apply + * Example set up: `{% if is_incremental() and 'axelar' not in var('HEAL_MODELS') %}` -### Resources: -- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) -- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers -- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support -- Find [dbt events](https://events.getdbt.com) near you -- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices + * Usage: + * Single CTE: `dbt run --vars '{"HEAL_MODELS":"axelar"}' -m ...` + * Multiple CTEs: `dbt run --vars '{"HEAL_MODELS":["axelar","across","celer_cbridge"]}' -m ...` + +Use a variable to extend the incremental lookback period: +* LOOKBACK + * Default is a string representing the specified time interval e.g. '12 hours', '7 days' etc. + * Example set up: `SELECT MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}'` + + * Usage: `dbt run --vars '{"LOOKBACK":"36 hours"}' -m ...` ## Applying Model Tags @@ -97,7 +105,7 @@ To add/update a model's snowflake tags, add/modify the `meta` model property und By default, model tags are pushed to Snowflake on each load. You can disable this by setting the `UPDATE_SNOWFLAKE_TAGS` project variable to `False` during a run. ``` -dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":False}' -s models/core/core__fact_blocks.sql +dbt run --vars '{"UPDATE_SNOWFLAKE_TAGS":False}' -s models/core/core__fact_blocks.sql ``` ### Querying for existing tags on a model in snowflake @@ -105,4 +113,11 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":False}' -s models/core/core__fact_blocks ``` select * from table(bsc.information_schema.tag_references('bsc.core.fact_blocks', 'table')); -``` \ No newline at end of file +``` + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 27a3055..4bdbe63 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -70,7 +70,7 @@ vars: BALANCES_START: 0 BALANCES_END: 10000000 HEAL_MODEL: False - HEAL_CURATED_MODEL: [] + HEAL_MODELS: [] START_GHA_TASKS: False API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}' diff --git a/models/gold/defi/defi__ez_bridge_activity.sql b/models/gold/defi/defi__ez_bridge_activity.sql index cba4521..f7803e6 100644 --- a/models/gold/defi/defi__ez_bridge_activity.sql +++ b/models/gold/defi/defi__ez_bridge_activity.sql @@ -33,7 +33,13 @@ SELECT token_symbol, amount_unadj, amount, - amount_usd, + ROUND( + CASE + WHEN amount_usd < 1e+15 THEN amount_usd + ELSE NULL + END, + 2 + ) AS amount_usd, COALESCE ( complete_bridge_activity_id, {{ dbt_utils.generate_surrogate_key( diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql index b0775e2..2a32f9e 100644 --- a/models/gold/defi/defi__ez_dex_swaps.sql +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -24,10 +24,26 @@ SELECT event_name, amount_in_unadj, amount_in, - amount_in_usd, + ROUND( + CASE + WHEN amount_out_usd IS NULL + OR ABS((amount_in_usd - amount_out_usd) / NULLIF(amount_out_usd, 0)) > 0.75 + OR ABS((amount_in_usd - amount_out_usd) / NULLIF(amount_in_usd, 0)) > 0.75 THEN NULL + ELSE amount_in_usd + END, + 2 + ) AS amount_in_usd, amount_out_unadj, amount_out, - amount_out_usd, + ROUND( + CASE + WHEN amount_in_usd IS NULL + OR ABS((amount_out_usd - amount_in_usd) / NULLIF(amount_in_usd, 0)) > 0.75 + OR ABS((amount_out_usd - amount_in_usd) / NULLIF(amount_out_usd, 0)) > 0.75 THEN NULL + ELSE amount_out_usd + END, + 2 + ) AS amount_out_usd, sender, tx_to, event_index, diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index 7db266f..0592226 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -35,7 +35,7 @@ AND _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -169,7 +169,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -195,7 +195,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) diff --git a/models/silver/defi/bridge/allbridge/silver_bridge__allbridge_sent.sql b/models/silver/defi/bridge/allbridge/silver_bridge__allbridge_sent.sql index 63803a9..9407a59 100644 --- a/models/silver/defi/bridge/allbridge/silver_bridge__allbridge_sent.sql +++ b/models/silver/defi/bridge/allbridge/silver_bridge__allbridge_sent.sql @@ -53,6 +53,7 @@ WITH base_evt AS ( WHERE topics [0] :: STRING = '0x884a8def17f0d5bbb3fef53f3136b5320c9b39f75afb8985eeab9ea1153ee56d' AND contract_address = '0xbbbd1bbb4f9b936c3604906d7592a644071de884' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql b/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql index 94fe2f3..8f4dc7a 100644 --- a/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql +++ b/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql @@ -42,6 +42,7 @@ WITH base_evt AS ( WHERE topics [0] :: STRING = '0x7e50569d26be643bda7757722291ec66b1be66d8283474ae3fab5a98f878a7a2' AND contract_address = '0x304acf330bbe08d1e512eefaa92f6a57871fd895' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -89,6 +90,7 @@ native_gas_paid AS ( WHERE topics [0] :: STRING = '0x999d431b58761213cf53af96262b67a069cbd963499fd8effd1e21556217b841' AND contract_address = '0x2d5d7d31f671f86c782533cc367f14109a082712' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql b/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql index f288f85..7f2dd9c 100644 --- a/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql +++ b/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql @@ -51,6 +51,7 @@ WITH base_evt AS ( '0x265b25e22bcd7f10a5bd6e6410f10537cc7567e8', '0xdd90e5e87a2081dcf0391920868ebc2ffb81a1af' ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql b/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql index 6ed583b..fee21c4 100644 --- a/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql +++ b/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql @@ -70,6 +70,7 @@ WITH base_evt AS ( WHERE topics [0] :: STRING = '0xfc8703fd57380f9dd234a89dce51333782d49c5902f307b02f03e014d18fe471' --CreatedOrder AND contract_address = '0xef4fb24ad0916217251f553c0596f8edc630eb66' --Dln: Source + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql b/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql index 4a466bb..124dfba 100644 --- a/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql +++ b/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql @@ -40,6 +40,7 @@ WITH base_evt AS ( --PortalV2 '0xbf0b5d561b986809924f88099c4ff0e6bcce60c9' --PortalV2 ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql b/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql index 6381b25..dfa762d 100644 --- a/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql +++ b/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql @@ -126,6 +126,7 @@ dst_info AS ( WHERE contract_address = '0x25ab3efd52e6470681ce037cd546dc60726948d3' AND topics [0] :: STRING = '0x5ce4019f772fda6cb703b26bce3ec3006eb36b73f1d3a0eb441213317d9f5e9d' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql b/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql index ddccfcf..ca64050 100644 --- a/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql +++ b/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql @@ -1,12 +1,14 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform','version'], cluster_by = ['block_timestamp::DATE'], - tags = ['curated','reorg'] + tags = ['curated','reorg','heal'] ) }} WITH allbridge AS ( + SELECT block_number, block_timestamp, @@ -32,11 +34,11 @@ WITH allbridge AS ( FROM {{ ref('silver_bridge__allbridge_sent') }} -{% if is_incremental() and 'allbridge' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'allbridge' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -68,11 +70,11 @@ axelar AS ( FROM {{ ref('silver_bridge__axelar_contractcallwithtoken') }} -{% if is_incremental() and 'axelar' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'axelar' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -104,11 +106,11 @@ celer_cbridge AS ( FROM {{ ref('silver_bridge__celer_cbridge_send') }} -{% if is_incremental() and 'celer_cbridge' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'celer_cbridge' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -140,11 +142,11 @@ dln_debridge AS ( FROM {{ ref('silver_bridge__dln_debridge_createdorder') }} -{% if is_incremental() and 'dln_debridge' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dln_debridge' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -176,11 +178,11 @@ eywa AS ( FROM {{ ref('silver_bridge__eywa_requestsent') }} -{% if is_incremental() and 'eywa' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'eywa' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -212,11 +214,11 @@ meson AS ( FROM {{ ref('silver_bridge__meson_transfers') }} -{% if is_incremental() and 'meson' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'meson' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -248,11 +250,11 @@ stargate AS ( FROM {{ ref('silver_bridge__stargate_swap') }} -{% if is_incremental() and 'stargate' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'stargate' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -284,11 +286,11 @@ symbiosis AS ( FROM {{ ref('silver_bridge__symbiosis_synthesizerequest') }} -{% if is_incremental() and 'symbiosis' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'symbiosis' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -320,11 +322,11 @@ synapse_tb AS ( FROM {{ ref('silver_bridge__synapse_token_bridge') }} -{% if is_incremental() and 'synapse_tb' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'synapse_tb' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -356,11 +358,11 @@ synapse_tbs AS ( FROM {{ ref('silver_bridge__synapse_tokenbridgeandswap') }} -{% if is_incremental() and 'synapse_tbs' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'synapse_tbs' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -392,11 +394,11 @@ wormhole AS ( FROM {{ ref('silver_bridge__wormhole_transfers') }} -{% if is_incremental() and 'wormhole' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'wormhole' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -419,7 +421,7 @@ all_protocols AS ( celer_cbridge UNION ALL SELECT - * + * FROM dln_debridge UNION ALL @@ -458,7 +460,7 @@ all_protocols AS ( FROM wormhole ), -FINAL AS ( +complete_bridge_activity AS ( SELECT block_number, block_timestamp, @@ -514,7 +516,7 @@ FINAL AS ( 2 ) ELSE NULL - END AS amount_usd_unadj, + END AS amount_usd, _id, b._inserted_timestamp FROM @@ -539,6 +541,183 @@ FINAL AS ( ) = LOWER( b.destination_chain ) +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + block_number, + block_timestamp, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_hash, + event_index, + bridge_address, + event_name, + platform, + version, + sender, + receiver, + destination_chain_receiver, + destination_chain_id, + destination_chain, + t0.token_address, + C.token_symbol AS token_symbol, + C.token_decimals AS token_decimals, + amount_unadj, + CASE + WHEN C.token_decimals IS NOT NULL THEN (amount_unadj / pow(10, C.token_decimals)) + ELSE amount_unadj + END AS amount_heal, + CASE + WHEN C.token_decimals IS NOT NULL THEN amount_heal * p.price + ELSE NULL + END AS amount_usd_heal, + _id, + t0._inserted_timestamp + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('silver__contracts') }} C + ON t0.token_address = C.contract_address + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.token_address = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform, + '-', + t1.version + ) + FROM + {{ this }} + t1 + WHERE + t1.token_decimals IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t1.token_address) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t2.block_number, + '-', + t2.platform, + '-', + t2.version + ) + FROM + {{ this }} + t2 + WHERE + t2.amount_usd IS NULL + AND t2._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t2.token_address + AND p.hour = DATE_TRUNC( + 'hour', + t2.block_timestamp + ) + ) + GROUP BY + 1 + ) + ), + {% endif %} + + FINAL AS ( + SELECT + * + FROM + complete_bridge_activity + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + block_number, + block_timestamp, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_hash, + event_index, + bridge_address, + event_name, + platform, + version, + sender, + receiver, + destination_chain_receiver, + destination_chain_id, + destination_chain, + token_address, + token_symbol, + token_decimals, + amount_unadj, + amount_heal AS amount, + amount_usd_heal AS amount_usd, + _id, + _inserted_timestamp +FROM + heal_model +{% endif %} ) SELECT block_number, @@ -562,21 +741,18 @@ SELECT token_decimals, amount_unadj, amount, - CASE - WHEN amount_usd_unadj < 1e+15 THEN amount_usd_unadj - ELSE NULL - END AS amount_usd, + amount_usd, _id, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( - ['_id'] + ['_id'] ) }} AS complete_bridge_activity_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - FINAL -WHERE destination_chain <> 'bsc' -qualify (ROW_NUMBER() over (PARTITION BY _id + FINAL +WHERE + destination_chain <> 'bsc' qualify (ROW_NUMBER() over (PARTITION BY _id ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql b/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql index 66d0fad..089824c 100644 --- a/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql +++ b/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql @@ -20,7 +20,8 @@ WITH base_contracts AS ( WHERE from_address = LOWER('0xe7Ec689f432f29383f217e36e680B5C855051f25') AND TYPE ILIKE 'create%' - AND tx_status ILIKE 'success' + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql b/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql index 9da04f6..509a6d4 100644 --- a/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql +++ b/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql @@ -66,6 +66,7 @@ base_evt AS ( ON d.contract_address = p.pool_address WHERE topics [0] :: STRING = '0x34660fc8af304464529f48a778e03d03e4d34bcd5f9b6f0cfbf3cd238c642f7f' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql b/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql index 7c8091f..97a44da 100644 --- a/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql +++ b/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql @@ -44,6 +44,7 @@ WITH base_evt AS ( '0xb91d3060c90aac7c4c706aef2b37997b3b2a1dcf', '0x5aa5f7f84ed0e5db0a4a85c3947ea16b53352fd4' ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql b/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql index b44c328..dc2a4a0 100644 --- a/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql +++ b/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql @@ -46,6 +46,7 @@ WITH base_evt AS ( '0xd123f70ae324d34a9e76b67a27bf77593ba8749f' ) AND origin_to_address IS NOT NULL + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql b/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql index 4c90dd3..7a4b49e 100644 --- a/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql +++ b/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql @@ -53,6 +53,7 @@ WITH base_evt AS ( '0x0efc29e196da2e81afe96edd041bedcdf9e74893', '0xd123f70ae324d34a9e76b67a27bf77593ba8749f' ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/biswap/silver_dex__biswap_pools.sql b/models/silver/defi/dex/biswap/silver_dex__biswap_pools.sql index f6dfb8b..7da5184 100644 --- a/models/silver/defi/dex/biswap/silver_dex__biswap_pools.sql +++ b/models/silver/defi/dex/biswap/silver_dex__biswap_pools.sql @@ -27,6 +27,7 @@ WITH pool_creation AS ( WHERE contract_address = '0x858e3312ed3a876947ea49d572a7c42de08af7ee' AND topics [0] :: STRING = '0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9' --PairCreated + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/dodo/silver_dex__dodo_v1_pools.sql b/models/silver/defi/dex/dodo/silver_dex__dodo_v1_pools.sql index b6f3adc..5ac055d 100644 --- a/models/silver/defi/dex/dodo/silver_dex__dodo_v1_pools.sql +++ b/models/silver/defi/dex/dodo/silver_dex__dodo_v1_pools.sql @@ -24,6 +24,7 @@ WITH pool_events AS ( WHERE contract_address = '0xca459456a45e300aa7ef447dbb60f87cccb42828' --DODOZoo AND topics [0] :: STRING = '0x5c428a2e12ecaa744a080b25b4cda8b86359c82d726575d7d747e07708071f93' --DODOBirth + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/dodo/silver_dex__dodo_v1_swaps.sql b/models/silver/defi/dex/dodo/silver_dex__dodo_v1_swaps.sql index 550f44d..4c4cbab 100644 --- a/models/silver/defi/dex/dodo/silver_dex__dodo_v1_swaps.sql +++ b/models/silver/defi/dex/dodo/silver_dex__dodo_v1_swaps.sql @@ -71,6 +71,7 @@ sell_base_token AS ( FROM proxies ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -124,6 +125,7 @@ buy_base_token AS ( FROM proxies ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql index bf96335..7ad7c89 100644 --- a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql +++ b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql @@ -39,6 +39,7 @@ WITH pools AS ( --NewDSP '0xaf5c5f12a80fc937520df6fcaed66262a4cc775e0f3fceaf7a7cfe476d9a751d' --NewDVM ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql index 616b19d..774c247 100644 --- a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql +++ b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql @@ -94,6 +94,7 @@ swaps_base AS ( FROM proxies ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_pools.sql b/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_pools.sql index 78c853d..48459ee 100644 --- a/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_pools.sql +++ b/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_pools.sql @@ -31,6 +31,7 @@ WITH pool_creation AS ( '0xf89e6ca06121b6d4370f4b196ae458e8b969a011' --v2 factory ) AND topics [0] :: STRING = '0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9' --pairCreated + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_swaps.sql b/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_swaps.sql index f1a65f4..a561e38 100644 --- a/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_swaps.sql +++ b/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_swaps.sql @@ -59,6 +59,7 @@ swaps_base AS ( ON l.contract_address = pool_address WHERE l.topics [0] :: STRING = '0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822' --Swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql index bc155b6..1a39c66 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql @@ -23,7 +23,8 @@ WITH contract_deployments AS ( '0xb5574750a786a37e300a916974ecd63f93fc6754' ) AND TYPE ILIKE 'create%' - AND tx_status ILIKE 'success' + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql index 4074283..826fdb6 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql @@ -46,6 +46,7 @@ router_swaps_base AS ( ON l.contract_address = p.pool_address WHERE l.topics [0] :: STRING = '0xb709ddcc6550418e9b89df1f4938071eeaa3f6376309904c77e15d46b16066f5' --swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -89,6 +90,7 @@ swaps_base AS ( ON l.contract_address = p.pool_address WHERE l.topics [0] :: STRING = '0x8cf3dec1929508e5677d7db003124e74802bfba7250a572205a9986d86ca9f1e' --swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql index 65f473a..c313947 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql @@ -21,6 +21,7 @@ WITH contract_deployments AS ( WHERE contract_address = '0xde828fdc3f497f16416d1bb645261c7c6a62dab5' AND topics [0] :: STRING = '0xdbd2a1ea6808362e6adbec4db4969cbc11e3b0b28fb6c74cb342defaaf1daada' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql index 2bae43b..d1bc7ec 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql @@ -51,6 +51,7 @@ swaps AS ( ON l.contract_address = p.pool_address WHERE l.topics [0] :: STRING = '0x34f57786fb01682fb4eec88d340387ef01a168fe345ea5b76f709d4e560c10eb' --Trade + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql index 15085d8..2e4e73f 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql @@ -34,6 +34,7 @@ WITH pool_creation AS ( WHERE contract_address = '0x878dfe971d44e9122048308301f540910bbd934c' --dynamic fee factory AND topics [0] :: STRING = '0xfc574402c445e75f2b79b67884ff9c662244dce454c5ae68935fcd0bebb7c8ff' --created pool + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql index a13e440..d3af4aa 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql @@ -64,6 +64,7 @@ swaps_base AS ( ON p.pool_address = l.contract_address WHERE l.topics [0] :: STRING = '0x606ecd02b3e3b4778f8e97b2e03351de14224efaa5fa64e62200afc9395c2499' --Dynamic Swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql index fccdd17..55a046d 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql @@ -39,6 +39,7 @@ WITH pool_creation AS ( WHERE contract_address = LOWER('0x1c758aF0688502e49140230F6b0EBd376d429be5') --static pool factory AND topics [0] :: STRING = '0xb6bce363b712c921bead4bcc977289440eb6172eb89e258e3a25bd49ca806de6' --create pool + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql index 2b9f331..eeaa466 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql @@ -64,6 +64,7 @@ swaps_base AS ( ON p.pool_address = l.contract_address WHERE l.topics [0] :: STRING = '0x606ecd02b3e3b4778f8e97b2e03351de14224efaa5fa64e62200afc9395c2499' -- static swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql index f20cb9f..7e4add3 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql @@ -30,6 +30,7 @@ WITH pool_creation AS ( WHERE contract_address = '0x5f1dddbf348ac2fbe22a163e30f99f9ece3dd50a' --Elastic Pool Deployer AND topics [0] :: STRING = '0x783cca1c0412dd0d695e784568c96da2e9c22ff989357a2e8b1d9b2b4e6b7118' --Create pool + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql index 216b204..1536e76 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql @@ -77,6 +77,7 @@ swaps_base AS ( ON p.pool_address = l.contract_address WHERE topics [0] :: STRING = '0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67' -- elastic swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/level/silver_dex__levelfi_swaps.sql b/models/silver/defi/dex/level/silver_dex__levelfi_swaps.sql index 3f81c9e..dd1e8b0 100644 --- a/models/silver/defi/dex/level/silver_dex__levelfi_swaps.sql +++ b/models/silver/defi/dex/level/silver_dex__levelfi_swaps.sql @@ -74,6 +74,7 @@ WITH swaps_base AS ( WHERE contract_address = '0xa5abfb56a78d2bd4689b25b8a77fd49bb0675874' --router AND topics [0] :: STRING = '0xd6d34547c69c5ee3d2667625c188acf1006abb93e0ee7cf03925c67cf7760413' --swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_pools.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_pools.sql index dbaed1f..0ae5b71 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_pools.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_pools.sql @@ -27,6 +27,7 @@ WITH pool_creation AS ( WHERE contract_address = '0xbcfccbde45ce874adcb698cc183debcf17952812' --factory AND topics [0] :: STRING = '0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9' --PairCreated + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_pools.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_pools.sql index c4d75bd..bd126a9 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_pools.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_pools.sql @@ -31,6 +31,7 @@ WITH pool_creation AS ( '0x1f830fb91094a0e87c0a80150aa0af3805456090' ) --factory AND topics [0] :: STRING = '0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9' --PairCreated + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_mm_swaps.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_mm_swaps.sql index a5cb72d..b34defd 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_mm_swaps.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_mm_swaps.sql @@ -58,6 +58,7 @@ WITH swaps_base AS ( WHERE contract_address = '0xfeacb05b373f1a08e68235ba7fc92636b92ced01' --router AND topics [0] :: STRING = '0xe7d6f812e1a54298ddef0b881cd08a4d452d9de35eb18b5145aa580fdda18b26' --swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_pools.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_pools.sql index e5bd7a0..6046f53 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_pools.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_pools.sql @@ -44,6 +44,7 @@ WITH pools AS ( '0xa9551fb056fc743efe2a0a34e39f9769ad10166520df7843c09a66f82e148b97', '0x48dc7a1b156fe3e70ed5ed0afcb307661905edf536f15bb5786e327ea1933532' ) -- swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_pools.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_pools.sql index 81379e9..77a3b54 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_pools.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_pools.sql @@ -36,6 +36,7 @@ WITH created_pools AS ( WHERE contract_address = '0x0bfbcf9fa4f9c56b0f40a671ad40e0805a091865' --factory AND topics [0] :: STRING = '0x783cca1c0412dd0d695e784568c96da2e9c22ff989357a2e8b1d9b2b4e6b7118' --paircreated + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql b/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql index f8374f4..fac0688 100644 --- a/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql +++ b/models/silver/defi/dex/silver_dex__complete_dex_liquidity_pools.sql @@ -3,15 +3,15 @@ incremental_strategy = 'delete+insert', unique_key = ['block_number','platform','version'], cluster_by = ['block_timestamp::DATE'], - tags = ['curated','reorg'] + tags = ['curated','reorg','heal'] ) }} WITH contracts AS ( SELECT - contract_address AS address, - token_symbol AS symbol, - token_decimals AS decimals, + contract_address, + token_symbol, + token_decimals, _inserted_timestamp FROM {{ ref('silver__contracts') }} @@ -24,8 +24,16 @@ biswap AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'biswap' AS platform, 'v1' AS version, _log_id AS _id, @@ -33,11 +41,11 @@ biswap AS ( FROM {{ ref('silver_dex__biswap_pools') }} -{% if is_incremental() and 'biswap' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'biswap' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -51,8 +59,16 @@ dodo_v1 AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, base_token AS token0, quote_token AS token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'dodo-v1' AS platform, 'v1' AS version, _id, @@ -60,11 +76,11 @@ dodo_v1 AS ( FROM {{ ref('silver_dex__dodo_v1_pools') }} -{% if is_incremental() and 'dodo_v1' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dodo_v1' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -78,8 +94,16 @@ dodo_v2 AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, base_token AS token0, quote_token AS token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'dodo-v2' AS platform, 'v2' AS version, _log_id AS _id, @@ -89,10 +113,10 @@ dodo_v2 AS ( WHERE token0 IS NOT NULL -{% if is_incremental() and 'dodo_v2' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dodo_v2' not in var('HEAL_MODELS') %} AND _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -106,8 +130,16 @@ frax AS ( factory_address AS contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'fraxswap' AS platform, 'v1' AS version, _log_id AS _id, @@ -115,11 +147,11 @@ frax AS ( FROM {{ ref('silver_dex__fraxswap_pools') }} -{% if is_incremental() and 'frax' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'frax' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -133,8 +165,16 @@ kyberswap_v1_dynamic AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'kyberswap-v1' AS platform, 'v1-dynamic' AS version, _log_id AS _id, @@ -142,11 +182,11 @@ kyberswap_v1_dynamic AS ( FROM {{ ref('silver_dex__kyberswap_v1_dynamic_pools') }} -{% if is_incremental() and 'kyberswap_v1_dynamic' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kyberswap_v1_dynamic' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -160,8 +200,16 @@ kyberswap_v1_static AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'kyberswap-v1' AS platform, 'v1-static' AS version, _log_id AS _id, @@ -169,11 +217,11 @@ kyberswap_v1_static AS ( FROM {{ ref('silver_dex__kyberswap_v1_static_pools') }} -{% if is_incremental() and 'kyberswap_v1_static' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kyberswap_v1_static' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -186,10 +234,17 @@ kyberswap_v2_elastic AS ( tx_hash, contract_address, pool_address, + NULL AS pool_name, swap_fee_units AS fee, tick_distance AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'kyberswap-v2' AS platform, 'v2' AS version, _log_id AS _id, @@ -197,11 +252,11 @@ kyberswap_v2_elastic AS ( FROM {{ ref('silver_dex__kyberswap_v2_elastic_pools') }} -{% if is_incremental() and 'kyberswap_v2_elastic' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kyberswap_v2_elastic' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -215,8 +270,16 @@ pancakeswap_v1 AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'pancakeswap-v1' AS platform, 'v1' AS version, _log_id AS _id, @@ -224,11 +287,11 @@ pancakeswap_v1 AS ( FROM {{ ref('silver_dex__pancakeswap_v1_pools') }} -{% if is_incremental() and 'pancakeswap_v1' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'pancakeswap_v1' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -242,8 +305,16 @@ pancakeswap_v2_amm AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'pancakeswap-v2' AS platform, 'v2-amm' AS version, _log_id AS _id, @@ -251,11 +322,11 @@ pancakeswap_v2_amm AS ( FROM {{ ref('silver_dex__pancakeswap_v2_amm_pools') }} -{% if is_incremental() and 'pancakeswap_v2_amm' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'pancakeswap_v2_amm' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -269,21 +340,28 @@ pancakeswap_v2_ss AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, + tokenA AS token0, + tokenB AS token1, + tokenC AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'pancakeswap-v2' AS platform, 'v2-ss' AS version, _log_id AS _id, - _inserted_timestamp, - tokenA AS token0, - tokenB AS token1, - tokenC AS token2 + _inserted_timestamp FROM {{ ref('silver_dex__pancakeswap_v2_ss_pools') }} -{% if is_incremental() and 'pancakeswap_v2_ss' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'pancakeswap_v2_ss' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -296,10 +374,17 @@ pancakeswap_v3 AS ( tx_hash, contract_address, pool_address, + NULL AS pool_name, fee, tick_spacing, token0_address AS token0, token1_address AS token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'pancakeswap-v3' AS platform, 'v3' AS version, _log_id AS _id, @@ -307,11 +392,11 @@ pancakeswap_v3 AS ( FROM {{ ref('silver_dex__pancakeswap_v3_pools') }} -{% if is_incremental() and 'pancakeswap_v3' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'pancakeswap_v3' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -325,8 +410,16 @@ sushi AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'sushiswap' AS platform, 'v1' AS version, _log_id AS _id, @@ -334,11 +427,11 @@ sushi AS ( FROM {{ ref('silver_dex__sushi_pools') }} -{% if is_incremental() and 'sushi' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'sushi' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -352,8 +445,16 @@ trader_joe_v1 AS ( contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'trader-joe-v1' AS platform, 'v1' AS version, _log_id AS _id, @@ -361,11 +462,11 @@ trader_joe_v1 AS ( FROM {{ ref('silver_dex__trader_joe_v1_pools') }} -{% if is_incremental() and 'trader_joe_v1' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'trader_joe_v1' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -379,8 +480,16 @@ trader_joe_v2 AS ( contract_address, lb_pair AS pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, tokenX AS token0, tokenY AS token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'trader-joe-v2' AS platform, 'v2' AS version, _log_id AS _id, @@ -388,11 +497,11 @@ trader_joe_v2 AS ( FROM {{ ref('silver_dex__trader_joe_v2_pools') }} -{% if is_incremental() and 'trader_joe_v2' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'trader_joe_v2' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -405,10 +514,17 @@ uni_v3 AS ( tx_hash, contract_address, pool_address, + NULL AS pool_name, fee, tick_spacing, token0_address AS token0, token1_address AS token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'uniswap-v3' AS platform, 'v3' AS version, _log_id AS _id, @@ -416,44 +532,52 @@ uni_v3 AS ( FROM {{ ref('silver_dex__univ3_pools') }} -{% if is_incremental() and 'uni_v3' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'uni_v3' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), uni_v2 AS ( - -SELECT + SELECT block_number, block_timestamp, tx_hash, contract_address, pool_address, NULL AS pool_name, + NULL AS fee, + NULL AS tick_spacing, token0, token1, + NULL AS token2, + NULL AS token3, + NULL AS token4, + NULL AS token5, + NULL AS token6, + NULL AS token7, 'uniswap-v2' AS platform, 'v2' AS version, _log_id AS _id, _inserted_timestamp -FROM + FROM {{ ref('silver_dex__univ2_pools') }} -{% if is_incremental() and 'uni_v2' not in var('HEAL_CURATED_MODEL') %} + +{% if is_incremental() and 'uni_v2' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -all_pools_standard AS ( +all_pools AS ( SELECT * FROM @@ -513,8 +637,7 @@ all_pools_standard AS ( * FROM trader_joe_v2 -), -all_pools_v3 AS ( + UNION ALL SELECT * FROM @@ -529,99 +652,34 @@ all_pools_v3 AS ( * FROM kyberswap_v2_elastic -), -all_pools_other AS ( + UNION ALL SELECT * FROM pancakeswap_v2_ss ), -FINAL AS ( +complete_lps AS ( SELECT block_number, block_timestamp, tx_hash, - contract_address, + p.contract_address, pool_address, CASE - WHEN pool_name IS NULL THEN CONCAT( + WHEN pool_name IS NOT NULL THEN pool_name + WHEN pool_name IS NULL + AND platform IN ( + 'uniswap-v3', + 'kyberswap-v2', + 'pancakeswap-v3' + ) THEN CONCAT( COALESCE( - c0.symbol, + c0.token_symbol, CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) ), '-', COALESCE( - c1.symbol, - CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) - ) - ) - ELSE pool_name - END AS pool_name, - OBJECT_CONSTRUCT( - 'token0', - token0, - 'token1', - token1 - ) AS tokens, - OBJECT_CONSTRUCT( - 'token0', - c0.symbol, - 'token1', - c1.symbol - ) AS symbols, - OBJECT_CONSTRUCT( - 'token0', - c0.decimals, - 'token1', - c1.decimals - ) AS decimals, - platform, - version, - _id, - p._inserted_timestamp - FROM - all_pools_standard p - LEFT JOIN contracts c0 - ON c0.address = p.token0 - LEFT JOIN contracts c1 - ON c1.address = p.token1 - UNION ALL - SELECT - block_number, - block_timestamp, - tx_hash, - contract_address, - pool_address, - CASE - WHEN platform = 'kyberswap-v2' THEN CONCAT( - COALESCE( - c0.symbol, - CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) - ), - '-', - COALESCE( - c1.symbol, - CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) - ), - ' ', - COALESCE( - fee, - 0 - ), - ' ', - COALESCE( - tick_spacing, - 0 - ) - ) - WHEN platform = 'uniswap-v3' THEN CONCAT( - COALESCE( - c0.symbol, - CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) - ), - '-', - COALESCE( - c1.symbol, + c1.token_symbol, CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) ), ' ', @@ -634,116 +692,624 @@ FINAL AS ( tick_spacing, 0 ), - ' UNI-V3 LP' - ) - WHEN platform = 'pancakeswap-v3' THEN CONCAT( - COALESCE( - c0.symbol, - CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) - ), - '-', - COALESCE( - c1.symbol, - CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) - ), - ' ', - COALESCE( - fee, - 0 - ), - ' ', - COALESCE( - tick_spacing, - 0 - ), - ' PCS-V3 LP' - ) - END AS pool_name, - OBJECT_CONSTRUCT( - 'token0', - token0, - 'token1', - token1 - ) AS tokens, - OBJECT_CONSTRUCT( - 'token0', - c0.symbol, - 'token1', - c1.symbol - ) AS symbols, - OBJECT_CONSTRUCT( - 'token0', - c0.decimals, - 'token1', - c1.decimals - ) AS decimals, - platform, - version, - _id, - p._inserted_timestamp - FROM - all_pools_v3 p - LEFT JOIN contracts c0 - ON c0.address = p.token0 - LEFT JOIN contracts c1 - ON c1.address = p.token1 - UNION ALL - SELECT - block_number, - block_timestamp, - tx_hash, - contract_address, - pool_address, - CASE - WHEN pool_name IS NULL THEN CONCAT( - COALESCE(c0.symbol, SUBSTRING(token0, 1, 5) || '...' || SUBSTRING(token0, 39, 42)), CASE - WHEN token1 IS NOT NULL THEN '-' || COALESCE(c1.symbol, SUBSTRING(token1, 1, 5) || '...' || SUBSTRING(token1, 39, 42)) + WHEN platform = 'uniswap-v3' THEN ' UNI-V3 LP' + WHEN platform = 'kyberswap-v2' THEN '' + WHEN platform = 'pancakeswap-v3' THEN ' PCS-V3 LP' + END + ) + WHEN pool_name IS NULL + AND platform = 'pancakeswap-v2' + AND version = 'v2-ss' THEN CONCAT( + COALESCE(c0.token_symbol, SUBSTRING(token0, 1, 5) || '...' || SUBSTRING(token0, 39, 42)), + CASE + WHEN token1 IS NOT NULL THEN '-' || COALESCE(c1.token_symbol, SUBSTRING(token1, 1, 5) || '...' || SUBSTRING(token1, 39, 42)) ELSE '' END, CASE - WHEN token2 IS NOT NULL THEN '-' || COALESCE(c2.symbol, SUBSTRING(token2, 1, 5) || '...' || SUBSTRING(token2, 39, 42)) + WHEN token2 IS NOT NULL THEN '-' || COALESCE(c2.token_symbol, SUBSTRING(token2, 1, 5) || '...' || SUBSTRING(token2, 39, 42)) ELSE '' END ) - ELSE pool_name + ELSE CONCAT( + COALESCE( + c0.token_symbol, + CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) + ), + '-', + COALESCE( + c1.token_symbol, + CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) + ) + ) END AS pool_name, + fee, + tick_spacing, + token0, + token1, + token2, + token3, + token4, + token5, + token6, + token7, OBJECT_CONSTRUCT( 'token0', token0, 'token1', token1, 'token2', - token2 + token2, + 'token3', + token3, + 'token4', + token4, + 'token5', + token5, + 'token6', + token6, + 'token7', + token7 ) AS tokens, OBJECT_CONSTRUCT( 'token0', - c0.symbol, + c0.token_symbol, 'token1', - c1.symbol, + c1.token_symbol, 'token2', - c2.symbol + c2.token_symbol, + 'token3', + c3.token_symbol, + 'token4', + c4.token_symbol, + 'token5', + c5.token_symbol, + 'token6', + c6.token_symbol, + 'token7', + c7.token_symbol ) AS symbols, OBJECT_CONSTRUCT( 'token0', - c0.decimals, + c0.token_decimals, 'token1', - c1.decimals, + c1.token_decimals, 'token2', - c2.decimals + c2.token_decimals, + 'token3', + c3.token_decimals, + 'token4', + c4.token_decimals, + 'token5', + c5.token_decimals, + 'token6', + c6.token_decimals, + 'token7', + c7.token_decimals ) AS decimals, platform, version, _id, p._inserted_timestamp FROM - all_pools_other p + all_pools p LEFT JOIN contracts c0 - ON c0.address = p.token0 + ON c0.contract_address = p.token0 LEFT JOIN contracts c1 - ON c1.address = p.token1 + ON c1.contract_address = p.token1 LEFT JOIN contracts c2 - ON c2.address = p.token2 + ON c2.contract_address = p.token2 + LEFT JOIN contracts c3 + ON c3.contract_address = p.token3 + LEFT JOIN contracts c4 + ON c4.contract_address = p.token4 + LEFT JOIN contracts c5 + ON c5.contract_address = p.token5 + LEFT JOIN contracts c6 + ON c6.contract_address = p.token6 + LEFT JOIN contracts c7 + ON c7.contract_address = p.token7 +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + t0.contract_address, + pool_address, + CASE + WHEN pool_name IS NOT NULL THEN pool_name + WHEN pool_name IS NULL + AND platform IN ( + 'uniswap-v3', + 'kyberswap-v2', + 'pancakeswap-v3' + ) THEN CONCAT( + COALESCE( + c0.token_symbol, + CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) + ), + '-', + COALESCE( + c1.token_symbol, + CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) + ), + ' ', + COALESCE( + fee, + 0 + ), + ' ', + COALESCE( + tick_spacing, + 0 + ), + CASE + WHEN platform = 'uniswap-v3' THEN ' UNI-V3 LP' + WHEN platform = 'kyberswap-v2' THEN '' + WHEN platform = 'pancakeswap-v3' THEN ' PCS-V3 LP' + END + ) + WHEN pool_name IS NULL + AND platform = 'pancakeswap-v2' + AND version = 'v2-ss' THEN CONCAT( + COALESCE(c0.token_symbol, SUBSTRING(token0, 1, 5) || '...' || SUBSTRING(token0, 39, 42)), + CASE + WHEN token1 IS NOT NULL THEN '-' || COALESCE(c1.token_symbol, SUBSTRING(token1, 1, 5) || '...' || SUBSTRING(token1, 39, 42)) + ELSE '' + END, + CASE + WHEN token2 IS NOT NULL THEN '-' || COALESCE(c2.token_symbol, SUBSTRING(token2, 1, 5) || '...' || SUBSTRING(token2, 39, 42)) + ELSE '' + END + ) + ELSE CONCAT( + COALESCE( + c0.token_symbol, + CONCAT(SUBSTRING(token0, 1, 5), '...', SUBSTRING(token0, 39, 42)) + ), + '-', + COALESCE( + c1.token_symbol, + CONCAT(SUBSTRING(token1, 1, 5), '...', SUBSTRING(token1, 39, 42)) + ) + ) + END AS pool_name_heal, + fee, + tick_spacing, + token0, + token1, + token2, + token3, + token4, + token5, + token6, + token7, + tokens, + OBJECT_CONSTRUCT( + 'token0', + c0.token_symbol, + 'token1', + c1.token_symbol, + 'token2', + c2.token_symbol, + 'token3', + c3.token_symbol, + 'token4', + c4.token_symbol, + 'token5', + c5.token_symbol, + 'token6', + c6.token_symbol, + 'token7', + c7.token_symbol + ) AS symbols_heal, + OBJECT_CONSTRUCT( + 'token0', + c0.token_decimals, + 'token1', + c1.token_decimals, + 'token2', + c2.token_decimals, + 'token3', + c3.token_decimals, + 'token4', + c4.token_decimals, + 'token5', + c5.token_decimals, + 'token6', + c6.token_decimals, + 'token7', + c7.token_decimals + ) AS decimals_heal, + platform, + version, + _id, + t0._inserted_timestamp + FROM + {{ this }} + t0 + LEFT JOIN contracts c0 + ON c0.contract_address = t0.token0 + LEFT JOIN contracts c1 + ON c1.contract_address = t0.token1 + LEFT JOIN contracts c2 + ON c2.contract_address = t0.token2 + LEFT JOIN contracts c3 + ON c3.contract_address = t0.token3 + LEFT JOIN contracts c4 + ON c4.contract_address = t0.token4 + LEFT JOIN contracts c5 + ON c5.contract_address = t0.token5 + LEFT JOIN contracts c6 + ON c6.contract_address = t0.token6 + LEFT JOIN contracts c7 + ON c7.contract_address = t0.token7 + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform, + '-', + t1.version + ) + FROM + {{ this }} + t1 + WHERE + t1.decimals :token0 :: INT IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t1.tokens :token0 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t2.block_number, + '-', + t2.platform, + '-', + t2.version + ) + FROM + {{ this }} + t2 + WHERE + t2.decimals :token1 :: INT IS NULL + AND t2._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t2.tokens :token1 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t3.block_number, + '-', + t3.platform, + '-', + t3.version + ) + FROM + {{ this }} + t3 + WHERE + t3.decimals :token2 :: INT IS NULL + AND t3._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t3.tokens :token2 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t4.block_number, + '-', + t4.platform, + '-', + t4.version + ) + FROM + {{ this }} + t4 + WHERE + t4.decimals :token3 :: INT IS NULL + AND t4._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t4.tokens :token3 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t5.block_number, + '-', + t5.platform, + '-', + t5.version + ) + FROM + {{ this }} + t5 + WHERE + t5.decimals :token4 :: INT IS NULL + AND t5._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t5.tokens :token4 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t6.block_number, + '-', + t6.platform, + '-', + t6.version + ) + FROM + {{ this }} + t6 + WHERE + t6.decimals :token5 :: INT IS NULL + AND t6._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t6.tokens :token5 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t7.block_number, + '-', + t7.platform, + '-', + t7.version + ) + FROM + {{ this }} + t7 + WHERE + t7.decimals :token6 :: INT IS NULL + AND t7._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t7.tokens :token6 :: STRING) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t8.block_number, + '-', + t8.platform, + '-', + t8.version + ) + FROM + {{ this }} + t8 + WHERE + t8.decimals :token7 :: INT IS NULL + AND t8._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t8.tokens :token7 :: STRING) + GROUP BY + 1 + ) + ), + {% endif %} + + FINAL AS ( + SELECT + * + FROM + complete_lps + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + pool_address, + pool_name_heal AS pool_name, + fee, + tick_spacing, + token0, + token1, + token2, + token3, + token4, + token5, + token6, + token7, + tokens, + symbols_heal AS symbols, + decimals_heal AS decimals, + platform, + version, + _id, + _inserted_timestamp +FROM + heal_model +{% endif %} ) SELECT block_number, @@ -757,6 +1323,16 @@ SELECT tokens, symbols, decimals, + fee, + tick_spacing, + token0, + token1, + token2, + token3, + token4, + token5, + token6, + token7, _id, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( diff --git a/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql b/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql index 527f7f2..cb51022 100644 --- a/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql +++ b/models/silver/defi/dex/silver_dex__complete_dex_swaps.sql @@ -1,37 +1,14 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform','version'], cluster_by = ['block_timestamp::DATE'], - tags = ['curated','reorg'] + tags = ['curated','reorg','heal'] ) }} -WITH contracts AS ( +WITH univ3 AS ( - SELECT - contract_address AS address, - token_symbol AS symbol, - token_name AS NAME, - token_decimals AS decimals - FROM - {{ ref('silver__contracts') }} -), -prices AS ( - SELECT - HOUR, - token_address, - price - FROM - {{ ref('price__ez_prices_hourly') }} - WHERE - token_address IN ( - SELECT - DISTINCT address - FROM - contracts - ) -), -univ3_swaps AS ( SELECT block_number, block_timestamp, @@ -40,51 +17,15 @@ univ3_swaps AS ( origin_from_address, origin_to_address, pool_address AS contract_address, - NULL AS pool_name, 'Swap' AS event_name, - amount0_unadj / pow(10, COALESCE(c1.decimals, 18)) AS amount0_adjusted, - amount1_unadj / pow(10, COALESCE(c2.decimals, 18)) AS amount1_adjusted, - CASE - WHEN c1.decimals IS NOT NULL THEN ROUND( - p1.price * amount0_adjusted, - 2 - ) - END AS amount0_usd, - CASE - WHEN c2.decimals IS NOT NULL THEN ROUND( - p2.price * amount1_adjusted, - 2 - ) - END AS amount1_usd, CASE WHEN amount0_unadj > 0 THEN ABS(amount0_unadj) ELSE ABS(amount1_unadj) END AS amount_in_unadj, - CASE - WHEN amount0_unadj > 0 THEN ABS(amount0_adjusted) - ELSE ABS(amount1_adjusted) - END AS amount_in, - CASE - WHEN amount0_unadj > 0 THEN ABS(amount0_usd) - ELSE ABS(amount1_usd) - END AS amount_in_usd, CASE WHEN amount0_unadj < 0 THEN ABS(amount0_unadj) ELSE ABS(amount1_unadj) END AS amount_out_unadj, - CASE - WHEN amount0_unadj < 0 THEN ABS(amount0_adjusted) - ELSE ABS(amount1_adjusted) - END AS amount_out, - CASE - WHEN amount0_unadj < 0 THEN ABS(amount0_usd) - ELSE ABS(amount1_usd) - END AS amount_out_usd, - sender, - recipient AS tx_to, - event_index, - 'uniswap-v3' AS platform, - 'v3' AS version, CASE WHEN amount0_unadj > 0 THEN token0_address ELSE token1_address @@ -93,47 +34,27 @@ univ3_swaps AS ( WHEN amount0_unadj < 0 THEN token0_address ELSE token1_address END AS token_out, - CASE - WHEN amount0_unadj > 0 THEN c1.symbol - ELSE c2.symbol - END AS symbol_in, - CASE - WHEN amount0_unadj < 0 THEN c1.symbol - ELSE c2.symbol - END AS symbol_out, + sender, + recipient AS tx_to, + event_index, + 'uniswap-v3' AS platform, + 'v3' AS version, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__univ3_swaps') }} - s - LEFT JOIN contracts c1 - ON c1.address = s.token0_address - LEFT JOIN contracts c2 - ON c2.address = s.token1_address - LEFT JOIN prices p1 - ON s.token0_address = p1.token_address - AND DATE_TRUNC( - 'hour', - block_timestamp - ) = p1.hour - LEFT JOIN prices p2 - ON s.token1_address = p2.token_address - AND DATE_TRUNC( - 'hour', - block_timestamp - ) = p2.hour -{% if is_incremental() and 'univ3_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'univ3' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -univ2_swaps AS ( +univ2 AS ( SELECT block_number, block_timestamp, @@ -143,49 +64,31 @@ univ2_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__univ2_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'univ2_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'univ2' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -woofi_swaps AS ( +woofi AS ( SELECT block_number, block_timestamp, @@ -195,65 +98,25 @@ woofi_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - CONCAT( - LEAST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ), - '-', - GREATEST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ) - ) AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__woofi_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'woofi_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'woofi' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -269,43 +132,25 @@ kyberswap_v1_dynamic AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1-dynamic' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__kyberswap_v1_dynamic_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'kyberswap_v1_dynamic' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kyberswap_v1_dynamic' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -321,43 +166,25 @@ kyberswap_v1_static AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1-static' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__kyberswap_v1_static_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'kyberswap_v1_static' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kyberswap_v1_static' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -373,49 +200,31 @@ kyberswap_v2_elastic AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__kyberswap_v2_elastic_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'kyberswap_v2_elastic' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kyberswap_v2_elastic' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -fraxswap_swaps AS ( +fraxswap AS ( SELECT block_number, block_timestamp, @@ -425,49 +234,31 @@ fraxswap_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__fraxswap_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'fraxswap_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'fraxswap' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -sushi_swaps AS ( +sushi AS ( SELECT block_number, block_timestamp, @@ -477,49 +268,31 @@ sushi_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__sushi_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'sushi_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'sushi' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -dodo_v1_swaps AS ( +dodo_v1 AS ( SELECT block_number, block_timestamp, @@ -529,49 +302,31 @@ dodo_v1_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__dodo_v1_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'dodo_v1_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dodo_v1' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -dodo_v2_swaps AS ( +dodo_v2 AS ( SELECT block_number, block_timestamp, @@ -581,49 +336,31 @@ dodo_v2_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__dodo_v2_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'dodo_v2_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dodo_v2' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -hashflow_swaps AS ( +hashflow AS ( SELECT block_number, block_timestamp, @@ -633,71 +370,31 @@ hashflow_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - CONCAT( - LEAST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ), - '-', - GREATEST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ) - ) AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__hashflow_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'hashflow_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'hashflow' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -hashflow_v3_swaps AS ( +hashflow_v3 AS ( SELECT block_number, block_timestamp, @@ -707,71 +404,31 @@ hashflow_v3_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v3' AS version, - token_in, - token_out, - CONCAT( - LEAST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ), - '-', - GREATEST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ) - ) AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__hashflow_v3_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'hashflow_v3_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'hashflow_v3' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -trader_joe_v1_swaps AS ( +trader_joe_v1 AS ( SELECT block_number, block_timestamp, @@ -781,49 +438,31 @@ trader_joe_v1_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__trader_joe_v1_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'trader_joe_v1_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'trader_joe_v1' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -trader_joe_v2_swaps AS ( +trader_joe_v2 AS ( SELECT block_number, block_timestamp, @@ -833,49 +472,31 @@ trader_joe_v2_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__trader_joe_v2_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'trader_joe_v2_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'trader_joe_v2' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -trader_joe_v2_1_swaps AS ( +trader_joe_v2_1 AS ( SELECT block_number, block_timestamp, @@ -885,49 +506,31 @@ trader_joe_v2_1_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2.1' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__trader_joe_v2_1_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'trader_joe_v2_1_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'trader_joe_v2_1' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -biswap_swaps AS ( +biswap AS ( SELECT block_number, block_timestamp, @@ -937,49 +540,31 @@ biswap_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__biswap_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'biswap_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'biswap' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -levelfi_swaps AS ( +levelfi AS ( SELECT block_number, block_timestamp, @@ -989,71 +574,31 @@ levelfi_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - CONCAT( - LEAST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ), - '-', - GREATEST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ) - ) AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__levelfi_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'levelfi_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'levelfi' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -pancakeswap_v1_swaps AS ( +pancakeswap_v1 AS ( SELECT block_number, block_timestamp, @@ -1063,49 +608,31 @@ pancakeswap_v1_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v1' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__pancakeswap_v1_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'pancakeswap_v1_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'pancakeswap_v1' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -pancakeswap_v2_amm_swaps AS ( +pancakeswap_v2_amm AS ( SELECT block_number, block_timestamp, @@ -1115,49 +642,31 @@ pancakeswap_v2_amm_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2-amm' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__pancakeswap_v2_amm_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'pancakeswap_v2_amm_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'pancakeswap_v2_amm' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -pancakeswap_v2_mm_swaps AS ( +pancakeswap_v2_mm AS ( SELECT block_number, block_timestamp, @@ -1167,71 +676,31 @@ pancakeswap_v2_mm_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2-mm' AS version, - token_in, - token_out, - CONCAT( - LEAST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ), - '-', - GREATEST( - COALESCE( - symbol_in, - CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) - ), - COALESCE( - symbol_out, - CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) - ) - ) - ) AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__pancakeswap_v2_mm_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'pancakeswap_v2_mm_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'pancakeswap_v2_mm' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -pancakeswap_v2_ss_swaps AS ( +pancakeswap_v2_ss AS ( SELECT block_number, block_timestamp, @@ -1241,49 +710,31 @@ pancakeswap_v2_ss_swaps AS ( origin_to_address, contract_address, event_name, - c1.decimals AS decimals_in, - c1.symbol AS symbol_in, amount_in_unadj, - CASE - WHEN decimals_in IS NULL THEN amount_in_unadj - ELSE (amount_in_unadj / pow(10, decimals_in)) - END AS amount_in, - c2.decimals AS decimals_out, - c2.symbol AS symbol_out, amount_out_unadj, - CASE - WHEN decimals_out IS NULL THEN amount_out_unadj - ELSE (amount_out_unadj / pow(10, decimals_out)) - END AS amount_out, + token_in, + token_out, sender, tx_to, event_index, platform, 'v2-ss' AS version, - token_in, - token_out, - NULL AS pool_name, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__pancakeswap_v2_ss_swaps') }} - s - LEFT JOIN contracts c1 - ON s.token_in = c1.address - LEFT JOIN contracts c2 - ON s.token_out = c2.address -{% if is_incremental() and 'pancakeswap_v2_ss_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'pancakeswap_v2_ss' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -pancakeswap_v3_swaps AS ( +pancakeswap_v3 AS ( SELECT block_number, block_timestamp, @@ -1292,904 +743,599 @@ pancakeswap_v3_swaps AS ( origin_from_address, origin_to_address, pool_address AS contract_address, - NULL AS pool_name, 'Swap' AS event_name, - amount0 AS amount0_unadj, - amount1 AS amount1_unadj, - amount0_unadj / pow(10, COALESCE(c1.decimals, 18)) AS amount0_adjusted, - amount1_unadj / pow(10, COALESCE(c2.decimals, 18)) AS amount1_adjusted, CASE - WHEN c1.decimals IS NOT NULL THEN ROUND( - p1.price * amount0_adjusted, - 2 - ) - END AS amount0_usd, - CASE - WHEN c2.decimals IS NOT NULL THEN ROUND( - p2.price * amount1_adjusted, - 2 - ) - END AS amount1_usd, - CASE - WHEN amount0_unadj > 0 THEN ABS(amount0_unadj) - ELSE ABS(amount1_unadj) + WHEN amount0 > 0 THEN ABS(amount0) + ELSE ABS(amount1) END AS amount_in_unadj, CASE - WHEN amount0_unadj > 0 THEN ABS(amount0_adjusted) - ELSE ABS(amount1_adjusted) - END AS amount_in, - CASE - WHEN amount0_unadj > 0 THEN ABS(amount0_usd) - ELSE ABS(amount1_usd) - END AS amount_in_usd, - CASE - WHEN amount0_unadj < 0 THEN ABS(amount0_unadj) - ELSE ABS(amount1_unadj) + WHEN amount0 < 0 THEN ABS(amount0) + ELSE ABS(amount1) END AS amount_out_unadj, CASE - WHEN amount0_unadj < 0 THEN ABS(amount0_adjusted) - ELSE ABS(amount1_adjusted) - END AS amount_out, + WHEN amount0 > 0 THEN token0_address + ELSE token1_address + END AS token_in, CASE - WHEN amount0_unadj < 0 THEN ABS(amount0_usd) - ELSE ABS(amount1_usd) - END AS amount_out_usd, + WHEN amount0 < 0 THEN token0_address + ELSE token1_address + END AS token_out, sender_address AS sender, recipient_address AS tx_to, event_index, 'pancakeswap-v3' AS platform, 'v3' AS version, - CASE - WHEN amount0_unadj > 0 THEN token0_address - ELSE token1_address - END AS token_in, - CASE - WHEN amount0_unadj < 0 THEN token0_address - ELSE token1_address - END AS token_out, - CASE - WHEN amount0_unadj > 0 THEN c1.symbol - ELSE c2.symbol - END AS symbol_in, - CASE - WHEN amount0_unadj < 0 THEN c1.symbol - ELSE c2.symbol - END AS symbol_out, _log_id, _inserted_timestamp FROM {{ ref('silver_dex__pancakeswap_v3_swaps') }} - s - LEFT JOIN contracts c1 - ON c1.address = s.token0_address - LEFT JOIN contracts c2 - ON c2.address = s.token1_address - LEFT JOIN prices p1 - ON s.token0_address = p1.token_address - AND DATE_TRUNC( - 'hour', - block_timestamp - ) = p1.hour - LEFT JOIN prices p2 - ON s.token1_address = p2.token_address - AND DATE_TRUNC( - 'hour', - block_timestamp - ) = p2.hour -{% if is_incremental() and 'pancakeswap_v3_swaps' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'pancakeswap_v3' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), ---union all standard dex CTEs here (excludes amount_usd) -all_dex_standard AS ( +all_dex AS ( SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - sushi_swaps + sushi UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - univ2_swaps + univ2 UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - fraxswap_swaps + fraxswap UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - woofi_swaps + woofi UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM kyberswap_v1_dynamic UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM kyberswap_v1_static UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM kyberswap_v2_elastic UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - dodo_v1_swaps + dodo_v1 UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - dodo_v2_swaps + dodo_v2 UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - hashflow_swaps + hashflow UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - hashflow_v3_swaps + hashflow_v3 UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - trader_joe_v1_swaps + trader_joe_v1 UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - trader_joe_v2_swaps + trader_joe_v2 UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - trader_joe_v2_1_swaps + trader_joe_v2_1 UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - biswap_swaps + biswap UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - levelfi_swaps + levelfi UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - pancakeswap_v1_swaps + pancakeswap_v1 UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - pancakeswap_v2_amm_swaps + pancakeswap_v2_amm UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - pancakeswap_v2_mm_swaps + pancakeswap_v2_mm UNION ALL SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_out_unadj, - amount_in, - amount_out, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - decimals_in, - decimals_out, - _log_id, - _inserted_timestamp + * FROM - pancakeswap_v2_ss_swaps + pancakeswap_v2_ss + UNION ALL + SELECT + * + FROM + univ3 + UNION ALL + SELECT + * + FROM + pancakeswap_v3 ), ---union all non-standard dex CTEs here (excludes amount_usd) -all_dex_custom AS ( +complete_dex_swaps AS ( SELECT - block_number, - block_timestamp, - tx_hash, + s.block_number, + s.block_timestamp, + s.tx_hash, origin_function_signature, origin_from_address, origin_to_address, - contract_address, - pool_name, + s.contract_address, event_name, - amount_in_unadj, - amount_in, - amount_in_usd, - amount_out_unadj, - amount_out, - amount_out_usd, - sender, - tx_to, - event_index, - platform, - version, token_in, - token_out, - symbol_in, - symbol_out, - _log_id, - _inserted_timestamp - FROM - univ3_swaps - UNION ALL - SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, + c1.token_decimals AS decimals_in, + c1.token_symbol AS symbol_in, amount_in_unadj, - amount_in, - amount_in_usd, - amount_out_unadj, - amount_out, - amount_out_usd, - sender, - tx_to, - event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - _log_id, - _inserted_timestamp - FROM - pancakeswap_v3_swaps -), -FINAL AS ( - SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address, - pool_name, - event_name, - amount_in_unadj, - amount_in, CASE - WHEN s.decimals_in IS NOT NULL THEN ROUND( - amount_in * p1.price, - 2 - ) + WHEN decimals_in IS NULL THEN amount_in_unadj + ELSE (amount_in_unadj / pow(10, decimals_in)) + END AS amount_in, + CASE + WHEN decimals_in IS NOT NULL THEN amount_in * p1.price ELSE NULL END AS amount_in_usd, + token_out, + c2.token_decimals AS decimals_out, + c2.token_symbol AS symbol_out, amount_out_unadj, - amount_out, CASE - WHEN s.decimals_out IS NOT NULL THEN ROUND( - amount_out * p2.price, - 2 - ) + WHEN decimals_out IS NULL THEN amount_out_unadj + ELSE (amount_out_unadj / pow(10, decimals_out)) + END AS amount_out, + CASE + WHEN decimals_out IS NOT NULL THEN amount_out * p2.price ELSE NULL END AS amount_out_usd, + CASE + WHEN lp.pool_name IS NULL THEN CONCAT( + LEAST( + COALESCE( + symbol_in, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + symbol_out, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) + ), + '-', + GREATEST( + COALESCE( + symbol_in, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + symbol_out, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) + ) + ) + ELSE lp.pool_name + END AS pool_name, sender, tx_to, event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - _log_id, - _inserted_timestamp + s.platform, + s.version, + s._log_id, + s._inserted_timestamp FROM - all_dex_standard s - LEFT JOIN prices p1 + all_dex s + LEFT JOIN {{ ref('silver__contracts') }} + c1 + ON s.token_in = c1.contract_address + LEFT JOIN {{ ref('silver__contracts') }} + c2 + ON s.token_out = c2.contract_address + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p1 ON s.token_in = p1.token_address AND DATE_TRUNC( 'hour', block_timestamp ) = p1.hour - LEFT JOIN prices p2 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p2 ON s.token_out = p2.token_address AND DATE_TRUNC( 'hour', block_timestamp ) = p2.hour - UNION ALL + LEFT JOIN {{ ref('silver_dex__complete_dex_liquidity_pools') }} + lp + ON s.contract_address = lp.pool_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( SELECT - block_number, - block_timestamp, - tx_hash, + t0.block_number, + t0.block_timestamp, + t0.tx_hash, origin_function_signature, origin_from_address, origin_to_address, - contract_address, - pool_name, + t0.contract_address, event_name, + token_in, + c1.token_decimals AS decimals_in, + c1.token_symbol AS symbol_in, amount_in_unadj, - amount_in, - ROUND( - amount_in_usd, - 2 - ) AS amount_in_usd, + CASE + WHEN c1.token_decimals IS NULL THEN amount_in_unadj + ELSE (amount_in_unadj / pow(10, c1.token_decimals)) + END AS amount_in_heal, + CASE + WHEN c1.token_decimals IS NOT NULL THEN amount_in_heal * p1.price + ELSE NULL + END AS amount_in_usd_heal, + token_out, + c2.token_decimals AS decimals_out, + c2.token_symbol AS symbol_out, amount_out_unadj, - amount_out, - ROUND( - amount_out_usd, - 2 - ) AS amount_out_usd, + CASE + WHEN c2.token_decimals IS NULL THEN amount_out_unadj + ELSE (amount_out_unadj / pow(10, c2.token_decimals)) + END AS amount_out_heal, + CASE + WHEN c2.token_decimals IS NOT NULL THEN amount_out_heal * p2.price + ELSE NULL + END AS amount_out_usd_heal, + CASE + WHEN lp.pool_name IS NULL THEN CONCAT( + LEAST( + COALESCE( + c1.token_symbol, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + c2.token_symbol, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) + ), + '-', + GREATEST( + COALESCE( + c1.token_symbol, + CONCAT(SUBSTRING(token_in, 1, 5), '...', SUBSTRING(token_in, 39, 42)) + ), + COALESCE( + c2.token_symbol, + CONCAT(SUBSTRING(token_out, 1, 5), '...', SUBSTRING(token_out, 39, 42)) + ) + ) + ) + ELSE lp.pool_name + END AS pool_name_heal, sender, tx_to, event_index, - platform, - version, - token_in, - token_out, - symbol_in, - symbol_out, - _log_id, - _inserted_timestamp + t0.platform, + t0.version, + t0._log_id, + t0._inserted_timestamp FROM - all_dex_custom C -) + {{ this }} + t0 + LEFT JOIN {{ ref('silver__contracts') }} + c1 + ON t0.token_in = c1.contract_address + LEFT JOIN {{ ref('silver__contracts') }} + c2 + ON t0.token_out = c2.contract_address + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p1 + ON t0.token_in = p1.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p1.hour + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p2 + ON t0.token_out = p2.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p2.hour + LEFT JOIN {{ ref('silver_dex__complete_dex_liquidity_pools') }} + lp + ON t0.contract_address = lp.pool_address + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform, + '-', + t1.version + ) + FROM + {{ this }} + t1 + WHERE + t1.decimals_in IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t1.token_in) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t2.block_number, + '-', + t2.platform, + '-', + t2.version + ) + FROM + {{ this }} + t2 + WHERE + t2.decimals_out IS NULL + AND t2._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__contracts') }} C + WHERE + C._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND C.token_decimals IS NOT NULL + AND C.contract_address = t2.token_out) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t3.block_number, + '-', + t3.platform, + '-', + t3.version + ) + FROM + {{ this }} + t3 + WHERE + t3.amount_in_usd IS NULL + AND t3._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t3.token_in + AND p.hour = DATE_TRUNC( + 'hour', + t3.block_timestamp + ) + ) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform, + '-', + t0.version + ) IN ( + SELECT + CONCAT( + t4.block_number, + '-', + t4.platform, + '-', + t4.version + ) + FROM + {{ this }} + t4 + WHERE + t4.amount_out_usd IS NULL + AND t4._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t4.token_out + AND p.hour = DATE_TRUNC( + 'hour', + t4.block_timestamp + ) + ) + GROUP BY + 1 + ) + ), + {% endif %} + + FINAL AS ( + SELECT + * + FROM + complete_dex_swaps + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL SELECT - f.block_number, - f.block_timestamp, - f.tx_hash, + block_number, + block_timestamp, + tx_hash, origin_function_signature, origin_from_address, origin_to_address, - f.contract_address, - CASE - WHEN f.pool_name IS NULL THEN p.pool_name - ELSE f.pool_name - END AS pool_name, + contract_address, event_name, + token_in, + decimals_in, + symbol_in, amount_in_unadj, - amount_in, - CASE - WHEN amount_out_usd IS NULL - OR ABS((amount_in_usd - amount_out_usd) / NULLIF(amount_out_usd, 0)) > 0.75 - OR ABS((amount_in_usd - amount_out_usd) / NULLIF(amount_in_usd, 0)) > 0.75 THEN NULL - ELSE amount_in_usd - END AS amount_in_usd, + amount_in_heal AS amount_in, + amount_in_usd_heal AS amount_in_usd, + token_out, + decimals_out, + symbol_out, amount_out_unadj, - amount_out, - CASE - WHEN amount_in_usd IS NULL - OR ABS((amount_out_usd - amount_in_usd) / NULLIF(amount_in_usd, 0)) > 0.75 - OR ABS((amount_out_usd - amount_in_usd) / NULLIF(amount_out_usd, 0)) > 0.75 THEN NULL - ELSE amount_out_usd - END AS amount_out_usd, + amount_out_heal AS amount_out, + amount_out_usd_heal AS amount_out_usd, + pool_name_heal AS pool_name, sender, tx_to, event_index, - f.platform, - f.version, + platform, + version, + _log_id, + _inserted_timestamp +FROM + heal_model +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + pool_name, + event_name, + amount_in_unadj, + amount_in, + amount_in_usd, + amount_out_unadj, + amount_out, + amount_out_usd, + sender, + tx_to, + event_index, + platform, + version, token_in, token_out, symbol_in, symbol_out, - f._log_id, - f._inserted_timestamp, + decimals_in, + decimals_out, + _log_id, + _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( - ['f.tx_hash','f.event_index'] + ['tx_hash','event_index'] ) }} AS complete_dex_swaps_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - FINAL f - LEFT JOIN {{ ref('silver_dex__complete_dex_liquidity_pools') }} - p - ON f.contract_address = p.pool_address + FINAL qualify (ROW_NUMBER() over (PARTITION BY _log_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/defi/dex/sushi/silver_dex__sushi_pools.sql b/models/silver/defi/dex/sushi/silver_dex__sushi_pools.sql index d076c54..a364d88 100644 --- a/models/silver/defi/dex/sushi/silver_dex__sushi_pools.sql +++ b/models/silver/defi/dex/sushi/silver_dex__sushi_pools.sql @@ -27,6 +27,7 @@ WITH pool_creation AS ( WHERE contract_address = LOWER('0xc35DADB65012eC5796536bD9864eD8773aBc74C4') AND topics [0] :: STRING = '0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9' --PairCreated + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_pools.sql b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_pools.sql index f7a70c7..c3f8492 100644 --- a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_pools.sql +++ b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_pools.sql @@ -27,6 +27,7 @@ WITH pool_creation AS ( WHERE contract_address = LOWER('0x4f8bdc85E3eec5b9dE67097c3f59B6Db025d9986') AND topics [0] :: STRING = '0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9' --PairCreated + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_swaps.sql b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_swaps.sql index c608eef..0bdc6ed 100644 --- a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_swaps.sql +++ b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_swaps.sql @@ -59,6 +59,7 @@ swaps_base AS ( ON p.pool_address = l.contract_address WHERE l.topics [0] :: STRING = '0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822' --Swap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_1_swaps.sql b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_1_swaps.sql index 56d7045..ef097be 100644 --- a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_1_swaps.sql +++ b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_1_swaps.sql @@ -81,6 +81,7 @@ swaps_base AS ( WHERE topics [0] :: STRING = '0xad7d6f97abf51ce18e17a38f4d70e975be9c0708474987bb3e26ad21bd93ca70' --Swap AND version = 'v2.1' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_pools.sql b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_pools.sql index 654fcb9..33b5e36 100644 --- a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_pools.sql +++ b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_pools.sql @@ -37,6 +37,7 @@ WITH pool_creation AS ( '0x43646a8e839b2f2766392c1bf8f60f6e587b6960' ) AND topics [0] :: STRING = '0x2c8d104b27c6b7f4492017a6f5cf3803043688934ebcaa6a03540beeaf976aff' --LB PairCreated + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_swaps.sql b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_swaps.sql index e4215be..53fffe5 100644 --- a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_swaps.sql +++ b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_swaps.sql @@ -80,6 +80,7 @@ swaps_base AS ( WHERE topics [0] :: STRING = '0xc528cda9e500228b16ce84fadae290d9a49aecb17483110004c5af0a07f6fd73' --Swap AND version = 'v2' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql b/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql index 1ef11aa..cf63f39 100644 --- a/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql +++ b/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql @@ -27,6 +27,7 @@ WITH pool_creation AS ( WHERE contract_address = LOWER('0x8909Dc15e40173Ff4699343b6eB8132c65e18eC6') AND topics [0] :: STRING = '0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9' --PairCreated + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql b/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql index 74446d9..f1574cf 100644 --- a/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql +++ b/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql @@ -36,13 +36,14 @@ WITH created_pools AS ( WHERE topics [0] = '0x783cca1c0412dd0d695e784568c96da2e9c22ff989357a2e8b1d9b2b4e6b7118' AND contract_address = '0xdb1d10011ad0ff90774d0c6bb92e5c5c8b4461f7' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) :: DATE - 2 + ) - INTERVAL '12 hours' FROM {{ this }} ) @@ -62,13 +63,14 @@ initial_info AS ( {{ ref('silver__logs') }} WHERE topics [0] :: STRING = '0x98636036cb66a9c19a37435efc1e90142190214e8abeb821bdba3f2990dd4c95' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) :: DATE - 2 + ) - INTERVAL '12 hours' FROM {{ this }} ) diff --git a/models/silver/defi/dex/uniswap/silver_dex__univ3_swaps.sql b/models/silver/defi/dex/uniswap/silver_dex__univ3_swaps.sql index efdd347..7b96246 100644 --- a/models/silver/defi/dex/uniswap/silver_dex__univ3_swaps.sql +++ b/models/silver/defi/dex/uniswap/silver_dex__univ3_swaps.sql @@ -46,7 +46,7 @@ AND _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) :: DATE + ) - INTERVAL '12 hours' FROM {{ this }} ) diff --git a/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql b/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql index 7c64040..0028dc0 100644 --- a/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql +++ b/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql @@ -64,6 +64,7 @@ WITH router_swaps_base AS ( '0xcef5be73ae943b77f9bc08859367d923c030a269' --v2 ) AND topics [0] :: STRING = '0x27c98e911efdd224f4002f6cd831c3ad0d2759ee176f9ee8466d95826af22a1c' --WooRouterSwap + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -136,6 +137,7 @@ swaps_base AS ( FROM router_swaps_base ) + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql index 9a0b869..bc2ad5c 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql @@ -1,9 +1,10 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} WITH kinza AS ( @@ -30,19 +31,19 @@ WITH kinza AS ( FROM {{ ref('silver__kinza_borrows') }} A -{% if is_incremental() and 'kinza' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kinza' not in var('HEAL_MODELS') %} WHERE A._inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -radiant as ( +radiant AS ( SELECT tx_hash, block_number, @@ -65,19 +66,19 @@ radiant as ( FROM {{ ref('silver__radiant_borrows') }} A -{% if is_incremental() and 'radiant' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'radiant' not in var('HEAL_MODELS') %} WHERE A._inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -liqee as ( +liqee AS ( SELECT tx_hash, block_number, @@ -100,19 +101,19 @@ liqee as ( FROM {{ ref('silver__liqee_borrows') }} A -{% if is_incremental() and 'liqee' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'liqee' not in var('HEAL_MODELS') %} WHERE A._inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM - {{ this }} -) + {{ this }} + ) {% endif %} ), -dforce as ( +dforce AS ( SELECT tx_hash, block_number, @@ -135,19 +136,19 @@ dforce as ( FROM {{ ref('silver__dforce_borrows') }} A -{% if is_incremental() and 'dforce' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dforce' not in var('HEAL_MODELS') %} WHERE A._inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -venus as ( +venus AS ( SELECT tx_hash, block_number, @@ -171,19 +172,19 @@ venus as ( {{ ref('silver__venus_borrows') }} l -{% if is_incremental() and 'venus' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'venus' not in var('HEAL_MODELS') %} WHERE l._inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -borrow_union as ( +borrow_union AS ( SELECT * FROM @@ -209,7 +210,7 @@ borrow_union as ( FROM radiant ), -FINAL AS ( +complete_lending_borrows AS ( SELECT tx_hash, block_number, @@ -243,8 +244,126 @@ FINAL AS ( 'hour', block_timestamp ) = p.hour - LEFT JOIN {{ ref('silver__contracts') }} C - ON b.token_address = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + borrower, + protocol_market, + t0.token_address, + t0.token_symbol, + amount_unadj, + amount, + ROUND( + amount * p.price, + 2 + ) AS amount_usd_heal, + platform, + t0.blockchain, + t0._LOG_ID, + t0._INSERTED_TIMESTAMP + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.token_address = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.token_address + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_borrows + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + borrower, + protocol_market, + token_address, + token_symbol, + amount_unadj, + amount, + amount_usd_heal AS amount_usd, + platform, + blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP +FROM + heal_model +{% endif %} ) SELECT *, diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql index d30e378..47777ac 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_deposits.sql @@ -1,9 +1,10 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} WITH kinza AS ( @@ -30,17 +31,17 @@ WITH kinza AS ( FROM {{ ref('silver__kinza_deposits') }} -{% if is_incremental() and 'kinza' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kinza' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -radiant as ( +radiant AS ( SELECT tx_hash, block_number, @@ -63,18 +64,17 @@ radiant as ( FROM {{ ref('silver__radiant_deposits') }} -{% if is_incremental() and 'radiant' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'radiant' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), - -venus as ( +venus AS ( SELECT tx_hash, block_number, @@ -97,17 +97,17 @@ venus as ( FROM {{ ref('silver__venus_deposits') }} -{% if is_incremental() and 'venus' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'venus' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -liqee as ( +liqee AS ( SELECT tx_hash, block_number, @@ -130,17 +130,17 @@ liqee as ( FROM {{ ref('silver__liqee_deposits') }} -{% if is_incremental() and 'liqee' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'liqee' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -dforce as ( +dforce AS ( SELECT tx_hash, block_number, @@ -163,43 +163,43 @@ dforce as ( FROM {{ ref('silver__dforce_deposits') }} -{% if is_incremental() and 'dforce' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dforce' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -deposits as ( - SELECT - * - FROM - venus - UNION ALL - SELECT - * - FROM - liqee - UNION ALL - SELECT - * - FROM - kinza - UNION ALL - SELECT - * - FROM - dforce - UNION ALL - SELECT - * - FROM - radiant +deposits AS ( + SELECT + * + FROM + venus + UNION ALL + SELECT + * + FROM + liqee + UNION ALL + SELECT + * + FROM + kinza + UNION ALL + SELECT + * + FROM + dforce + UNION ALL + SELECT + * + FROM + radiant ), -FINAL AS ( +complete_lending_deposits AS ( SELECT tx_hash, block_number, @@ -210,8 +210,15 @@ FINAL AS ( origin_function_signature, A.contract_address, CASE - WHEN platform in ('dForce','Liqee','Venus') THEN 'Mint' - WHEN platform in ('Radiant','Kinza') THEN 'Supply' + WHEN platform IN ( + 'dForce', + 'Liqee', + 'Venus' + ) THEN 'Mint' + WHEN platform IN ( + 'Radiant', + 'Kinza' + ) THEN 'Supply' ELSE 'Deposit' END AS event_name, protocol_market, @@ -237,17 +244,135 @@ FINAL AS ( 'hour', block_timestamp ) = p.hour - LEFT JOIN {{ ref('silver__contracts') }} C - ON A.token_address = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + protocol_market, + depositor, + t0.token_address, + t0.token_symbol, + amount_unadj, + amount, + ROUND( + amount * p.price, + 2 + ) AS amount_usd_heal, + platform, + t0.blockchain, + t0._LOG_ID, + t0._INSERTED_TIMESTAMP + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.token_address = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.token_address + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_deposits + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + protocol_market, + depositor, + token_address, + token_symbol, + amount_unadj, + amount, + amount_usd_heal AS amount_usd, + platform, + blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP +FROM + heal_model +{% endif %} ) SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash','event_index'] - ) }} AS complete_lending_deposits_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index'] + ) }} AS complete_lending_deposits_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id ORDER BY diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql index 313a712..8104b42 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_flashloans.sql @@ -1,9 +1,10 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} WITH kinza AS ( @@ -33,17 +34,17 @@ WITH kinza AS ( FROM {{ ref('silver__kinza_flashloans') }} -{% if is_incremental() and 'kinza' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kinza' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -radiant as ( +radiant AS ( SELECT tx_hash, block_number, @@ -62,36 +63,35 @@ radiant as ( initiator_address, target_address, platform, - symbol AS token_symbol, + symbol, blockchain, _LOG_ID, _INSERTED_TIMESTAMP FROM {{ ref('silver__radiant_flashloans') }} -{% if is_incremental() and 'radiant' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'radiant' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -flashloans as ( - - SELECT - * - FROM - kinza - UNION ALL - SELECT - * - FROM - radiant +flashloans AS ( + SELECT + * + FROM + kinza + UNION ALL + SELECT + * + FROM + radiant ), -FINAL AS ( +complete_lending_flashloans AS ( SELECT tx_hash, block_number, @@ -106,7 +106,7 @@ FINAL AS ( initiator_address AS initiator, target_address AS target, f.token_address AS flashloan_token, - token_symbol AS flashloan_token_symbol, + f.symbol AS flashloan_token_symbol, amount_unadj AS flashloan_amount_unadj, flashloan_amount, ROUND( @@ -132,8 +132,179 @@ FINAL AS ( 'hour', block_timestamp ) = p.hour - LEFT JOIN {{ ref('silver__contracts') }} C - ON f.token_address = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + protocol_market, + initiator, + target, + flashloan_token, + flashloan_token_symbol, + flashloan_amount_unadj, + flashloan_amount, + ROUND( + flashloan_amount * p.price, + 2 + ) AS flashloan_amount_usd_heal, + premium_amount_unadj, + premium_amount, + ROUND( + premium_amount * p.price, + 2 + ) AS premium_amount_usd_heal, + platform, + t0.blockchain, + t0._LOG_ID, + t0._INSERTED_TIMESTAMP + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.flashloan_token = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.flashloan_amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.flashloan_token + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) + OR CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t2.block_number, + '-', + t2.platform + ) + FROM + {{ this }} + t2 + WHERE + t2.premium_amount_usd IS NULL + AND t2._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t2.flashloan_token + AND p.hour = DATE_TRUNC( + 'hour', + t2.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_flashloans + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + protocol_market, + initiator, + target, + flashloan_token, + flashloan_token_symbol, + flashloan_amount_unadj, + flashloan_amount, + flashloan_amount_usd_heal AS flashloan_amount_usd, + premium_amount_unadj, + premium_amount, + premium_amount_usd_heal AS premium_amount_usd, + platform, + blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP +FROM + heal_model +{% endif %} ) SELECT *, diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql index d0d0f05..d8ac6fd 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_liquidations.sql @@ -1,9 +1,10 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} WITH venus AS ( @@ -34,17 +35,17 @@ WITH venus AS ( {{ ref('silver__venus_liquidations') }} l -{% if is_incremental() and 'venus' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'venus' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -dforce as ( +dforce AS ( SELECT tx_hash, block_number, @@ -71,17 +72,17 @@ dforce as ( {{ ref('silver__dforce_liquidations') }} l -{% if is_incremental() and 'dforce' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dforce' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -kinza as ( +kinza AS ( SELECT tx_hash, block_number, @@ -107,17 +108,17 @@ kinza as ( FROM {{ ref('silver__kinza_liquidations') }} -{% if is_incremental() and 'kinza' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kinza' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -radiant as ( +radiant AS ( SELECT tx_hash, block_number, @@ -143,17 +144,17 @@ radiant as ( FROM {{ ref('silver__radiant_liquidations') }} -{% if is_incremental() and 'radaint' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'radaint' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -venus as ( +venus AS ( SELECT tx_hash, block_number, @@ -180,17 +181,17 @@ venus as ( {{ ref('silver__venus_liquidations') }} l -{% if is_incremental() and 'venus' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'venus' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -liqee as ( +liqee AS ( SELECT tx_hash, block_number, @@ -215,45 +216,44 @@ liqee as ( _INSERTED_TIMESTAMP FROM {{ ref('silver__liqee_liquidations') }} - -{% if is_incremental() and 'liqee' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'liqee' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -liquidation_union as ( - SELECT - * - FROM - venus - UNION ALL - SELECT - * - FROM - liqee - UNION ALL - SELECT - * - FROM - kinza - UNION ALL - SELECT - * - FROM - dforce - UNION ALL - SELECT - * - FROM - radiant +liquidation_union AS ( + SELECT + * + FROM + venus + UNION ALL + SELECT + * + FROM + liqee + UNION ALL + SELECT + * + FROM + kinza + UNION ALL + SELECT + * + FROM + dforce + UNION ALL + SELECT + * + FROM + radiant ), -FINAL AS ( +complete_lending_liquidations AS ( SELECT tx_hash, block_number, @@ -264,7 +264,11 @@ FINAL AS ( origin_function_signature, A.contract_address, CASE - WHEN platform in ('dForce','Liqee','Venus') THEN 'LiquidateBorrow' + WHEN platform IN ( + 'dForce', + 'Liqee', + 'Venus' + ) THEN 'LiquidateBorrow' ELSE 'LiquidationCall' END AS event_name, liquidator, @@ -286,14 +290,139 @@ FINAL AS ( A._INSERTED_TIMESTAMP FROM liquidation_union A - LEFT JOIN {{ ref('price__ez_prices_hourly') }} P + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p ON collateral_asset = p.token_address AND DATE_TRUNC( 'hour', block_timestamp ) = p.hour - LEFT JOIN {{ ref('silver__contracts') }} C - ON collateral_asset = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + liquidator, + borrower, + protocol_market, + collateral_token, + collateral_token_symbol, + amount_unadj, + amount, + ROUND( + amount * p.price, + 2 + ) AS amount_usd_heal, + debt_token, + debt_token_symbol, + platform, + t0.blockchain, + t0._LOG_ID, + t0._INSERTED_TIMESTAMP + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.collateral_token = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.collateral_token + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_liquidations + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + liquidator, + borrower, + protocol_market, + collateral_token, + collateral_token_symbol, + amount_unadj, + amount, + amount_usd_heal AS amount_usd, + debt_token, + debt_token_symbol, + platform, + blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP +FROM + heal_model +{% endif %} ) SELECT *, diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql index 2a4a1ec..1dd9f15 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_repayments.sql @@ -1,9 +1,10 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} WITH kinza AS ( @@ -31,17 +32,17 @@ WITH kinza AS ( FROM {{ ref('silver__kinza_repayments') }} -{% if is_incremental() and 'kinza' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kinza' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -radiant as ( +radiant AS ( SELECT tx_hash, block_number, @@ -65,17 +66,17 @@ radiant as ( FROM {{ ref('silver__radiant_repayments') }} -{% if is_incremental() and 'radiant' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'radiant' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -venus as ( +venus AS ( SELECT tx_hash, block_number, @@ -99,17 +100,17 @@ venus as ( FROM {{ ref('silver__venus_repayments') }} -{% if is_incremental() and 'venus' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'venus' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -dforce as ( +dforce AS ( SELECT tx_hash, block_number, @@ -133,17 +134,17 @@ dforce as ( FROM {{ ref('silver__dforce_repayments') }} -{% if is_incremental() and 'dforce' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dforce' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -liqee as ( +liqee AS ( SELECT tx_hash, block_number, @@ -167,43 +168,43 @@ liqee as ( FROM {{ ref('silver__liqee_repayments') }} -{% if is_incremental() and 'liqee' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'liqee' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -repayments as ( - SELECT - * - FROM - venus - UNION ALL - SELECT - * - FROM - liqee - UNION ALL - SELECT - * - FROM - kinza - UNION ALL - SELECT - * - FROM - dforce - UNION ALL - SELECT - * - FROM - radiant +repayments AS ( + SELECT + * + FROM + venus + UNION ALL + SELECT + * + FROM + liqee + UNION ALL + SELECT + * + FROM + kinza + UNION ALL + SELECT + * + FROM + dforce + UNION ALL + SELECT + * + FROM + radiant ), -FINAL AS ( +complete_lending_repayments AS ( SELECT tx_hash, block_number, @@ -214,7 +215,11 @@ FINAL AS ( origin_function_signature, A.contract_address, CASE - WHEN platform in ('dForce','Liqee','Venus') THEN 'RepayBorrow' + WHEN platform IN ( + 'dForce', + 'Liqee', + 'Venus' + ) THEN 'RepayBorrow' ELSE 'Repay' END AS event_name, protocol_market, @@ -241,17 +246,137 @@ FINAL AS ( 'hour', block_timestamp ) = p.hour - LEFT JOIN {{ ref('silver__contracts') }} C - ON A.token_address = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + protocol_market, + payer, + borrower, + t0.token_address, + t0.token_symbol, + amount_unadj, + amount, + ROUND( + amount * p.price, + 2 + ) AS amount_usd_heal, + platform, + t0.blockchain, + t0._LOG_ID, + t0._INSERTED_TIMESTAMP + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.token_address = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.token_address + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_repayments + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + protocol_market, + payer, + borrower, + token_address, + token_symbol, + amount_unadj, + amount, + amount_usd_heal AS amount_usd, + platform, + blockchain, + _LOG_ID, + _INSERTED_TIMESTAMP +FROM + heal_model +{% endif %} ) SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash','event_index'] - ) }} AS complete_lending_repayments_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index'] + ) }} AS complete_lending_repayments_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id ORDER BY diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql index f5c6562..c6057f6 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_withdraws.sql @@ -1,9 +1,10 @@ +-- depends_on: {{ ref('silver__complete_token_prices') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = ['block_number','platform'], cluster_by = ['block_timestamp::DATE'], - tags = ['reorg','curated'] + tags = ['reorg','curated','heal'] ) }} WITH kinza AS ( @@ -30,19 +31,19 @@ WITH kinza AS ( FROM {{ ref('silver__kinza_withdraws') }} -{% if is_incremental() and 'kinza' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'kinza' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -radiant as ( +radiant AS ( SELECT tx_hash, block_number, @@ -65,19 +66,19 @@ radiant as ( FROM {{ ref('silver__radiant_withdraws') }} -{% if is_incremental() and 'radiant' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'radiant' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM - {{ this }} -) + {{ this }} + ) {% endif %} ), -liqee as ( +liqee AS ( SELECT tx_hash, block_number, @@ -100,19 +101,19 @@ liqee as ( FROM {{ ref('silver__liqee_withdraws') }} -{% if is_incremental() and 'liqee' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'liqee' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -venus as ( +venus AS ( SELECT tx_hash, block_number, @@ -135,19 +136,19 @@ venus as ( FROM {{ ref('silver__venus_withdraws') }} -{% if is_incremental() and 'venus' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'venus' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -dforce as ( +dforce AS ( SELECT tx_hash, block_number, @@ -170,19 +171,19 @@ dforce as ( FROM {{ ref('silver__dforce_withdraws') }} -{% if is_incremental() and 'dforce' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'dforce' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) {% endif %} ), -withdraws as ( +withdraws AS ( SELECT * FROM @@ -208,7 +209,7 @@ withdraws as ( FROM radiant ), -FINAL AS ( +complete_lending_withdraws AS ( SELECT tx_hash, block_number, @@ -219,7 +220,11 @@ FINAL AS ( origin_function_signature, A.contract_address, CASE - WHEN platform in ('dForce','Liqee','Venus') THEN 'Redeem' + WHEN platform IN ( + 'dForce', + 'Liqee', + 'Venus' + ) THEN 'Redeem' ELSE 'Withdraw' END AS event_name, protocol_market, @@ -245,8 +250,126 @@ FINAL AS ( 'hour', block_timestamp ) = p.hour - LEFT JOIN {{ ref('silver__contracts') }} C - ON A.token_address = C.contract_address +), + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +heal_model AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + t0.contract_address, + event_name, + protocol_market, + depositor, + t0.token_address, + t0.token_symbol, + amount_unadj, + amount, + ROUND( + amount * p.price, + 2 + ) AS amount_usd_heal, + platform, + t0.blockchain, + t0._log_id, + t0._inserted_timestamp + FROM + {{ this }} + t0 + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON t0.token_address = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + WHERE + CONCAT( + t0.block_number, + '-', + t0.platform + ) IN ( + SELECT + CONCAT( + t1.block_number, + '-', + t1.platform + ) + FROM + {{ this }} + t1 + WHERE + t1.amount_usd IS NULL + AND t1._inserted_timestamp < ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('silver__complete_token_prices') }} + p + WHERE + p._inserted_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND p.price IS NOT NULL + AND p.token_address = t1.token_address + AND p.hour = DATE_TRUNC( + 'hour', + t1.block_timestamp + ) + ) + GROUP BY + 1 + ) +), +{% endif %} + +FINAL AS ( + SELECT + * + FROM + complete_lending_withdraws + +{% if is_incremental() and var( + 'HEAL_MODEL' +) %} +UNION ALL +SELECT + tx_hash, + block_number, + block_timestamp, + event_index, + origin_from_address, + origin_to_address, + origin_function_signature, + contract_address, + event_name, + protocol_market, + depositor, + token_address, + token_symbol, + amount_unadj, + amount, + amount_usd_heal AS amount_usd, + platform, + blockchain, + _log_id, + _inserted_timestamp +FROM + heal_model +{% endif %} ) SELECT *, diff --git a/models/silver/defi/lending/dforce/silver__dforce_borrows.sql b/models/silver/defi/lending/dforce/silver__dforce_borrows.sql index 9d1bc46..c72fed0 100644 --- a/models/silver/defi/lending/dforce/silver__dforce_borrows.sql +++ b/models/silver/defi/lending/dforce/silver__dforce_borrows.sql @@ -56,6 +56,7 @@ dforce_borrows AS ( WHERE token_address <> '0xf51422c47c6c3e40cfca4a7b04232aedb7f49948' --excludes qDOT edge case ) AND topics [0] :: STRING = '0x2dd79f4fccfd18c360ce7f9132f3621bf05eee18f995224badb32d17f172df73' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/dforce/silver__dforce_deposits.sql b/models/silver/defi/lending/dforce/silver__dforce_deposits.sql index 2de45fa..a301b3c 100644 --- a/models/silver/defi/lending/dforce/silver__dforce_deposits.sql +++ b/models/silver/defi/lending/dforce/silver__dforce_deposits.sql @@ -53,6 +53,7 @@ dforce_deposits AS ( WHERE token_address <> '0xf51422c47c6c3e40cfca4a7b04232aedb7f49948' --excludes qDOT edge case ) AND topics [0] :: STRING = '0x2f00e3cdd69a77be7ed215ec7b2a36784dd158f921fca79ac29deffa353fe6ee' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/dforce/silver__dforce_liquidations.sql b/models/silver/defi/lending/dforce/silver__dforce_liquidations.sql index 5a9cd13..3ade9ee 100644 --- a/models/silver/defi/lending/dforce/silver__dforce_liquidations.sql +++ b/models/silver/defi/lending/dforce/silver__dforce_liquidations.sql @@ -56,6 +56,7 @@ dforce_liquidations AS ( WHERE token_address <> '0xf51422c47c6c3e40cfca4a7b04232aedb7f49948' --excludes qDOT edge case ) AND topics [0] :: STRING = '0x298637f684da70674f26509b10f07ec2fbc77a335ab1e7d6215a4b2484d8bb52' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/dforce/silver__dforce_repayments.sql b/models/silver/defi/lending/dforce/silver__dforce_repayments.sql index 010b422..2ac836e 100644 --- a/models/silver/defi/lending/dforce/silver__dforce_repayments.sql +++ b/models/silver/defi/lending/dforce/silver__dforce_repayments.sql @@ -51,6 +51,7 @@ dforce_repayments AS ( WHERE token_address <> '0xf51422c47c6c3e40cfca4a7b04232aedb7f49948' --excludes qDOT edge case ) AND topics [0] :: STRING = '0x6fadbf7329d21f278e724fa0d4511001a158f2a97ee35c5bc4cf8b64417399ef' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/dforce/silver__dforce_withdraws.sql b/models/silver/defi/lending/dforce/silver__dforce_withdraws.sql index 9a7b66f..420f329 100644 --- a/models/silver/defi/lending/dforce/silver__dforce_withdraws.sql +++ b/models/silver/defi/lending/dforce/silver__dforce_withdraws.sql @@ -53,6 +53,7 @@ dforce_redemptions AS ( WHERE token_address <> '0xf51422c47c6c3e40cfca4a7b04232aedb7f49948' --excludes qDOT edge case ) AND topics [0] :: STRING = '0x3f693fff038bb8a046aa76d9516190ac7444f7d69cf952c4cbdc086fdef2d6fc' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/liqee/silver__liqee_borrows.sql b/models/silver/defi/lending/liqee/silver__liqee_borrows.sql index 1c6095d..136aea7 100644 --- a/models/silver/defi/lending/liqee/silver__liqee_borrows.sql +++ b/models/silver/defi/lending/liqee/silver__liqee_borrows.sql @@ -55,6 +55,7 @@ liqee_borrows AS ( asset_details ) AND topics [0] :: STRING = '0x2dd79f4fccfd18c360ce7f9132f3621bf05eee18f995224badb32d17f172df73' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/liqee/silver__liqee_deposits.sql b/models/silver/defi/lending/liqee/silver__liqee_deposits.sql index 8f3e2f2..8bd1761 100644 --- a/models/silver/defi/lending/liqee/silver__liqee_deposits.sql +++ b/models/silver/defi/lending/liqee/silver__liqee_deposits.sql @@ -52,6 +52,7 @@ liqee_deposits AS ( asset_details ) AND topics [0] :: STRING = '0x2f00e3cdd69a77be7ed215ec7b2a36784dd158f921fca79ac29deffa353fe6ee' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/liqee/silver__liqee_liquidations.sql b/models/silver/defi/lending/liqee/silver__liqee_liquidations.sql index f6fae50..8e76477 100644 --- a/models/silver/defi/lending/liqee/silver__liqee_liquidations.sql +++ b/models/silver/defi/lending/liqee/silver__liqee_liquidations.sql @@ -55,6 +55,7 @@ liqee_liquidations AS ( asset_details ) AND topics [0] :: STRING = '0x298637f684da70674f26509b10f07ec2fbc77a335ab1e7d6215a4b2484d8bb52' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/liqee/silver__liqee_repayments.sql b/models/silver/defi/lending/liqee/silver__liqee_repayments.sql index 905aceb..0243ada 100644 --- a/models/silver/defi/lending/liqee/silver__liqee_repayments.sql +++ b/models/silver/defi/lending/liqee/silver__liqee_repayments.sql @@ -50,6 +50,7 @@ liqee_repayments AS ( asset_details ) AND topics [0] :: STRING = '0x6fadbf7329d21f278e724fa0d4511001a158f2a97ee35c5bc4cf8b64417399ef' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/liqee/silver__liqee_withdraws.sql b/models/silver/defi/lending/liqee/silver__liqee_withdraws.sql index e6918ac..0e19701 100644 --- a/models/silver/defi/lending/liqee/silver__liqee_withdraws.sql +++ b/models/silver/defi/lending/liqee/silver__liqee_withdraws.sql @@ -52,6 +52,7 @@ liqee_redemptions AS ( asset_details ) AND topics [0] :: STRING = '0x3f693fff038bb8a046aa76d9516190ac7444f7d69cf952c4cbdc086fdef2d6fc' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/venus/silver__venus_borrows.sql b/models/silver/defi/lending/venus/silver__venus_borrows.sql index 12f88a5..cfb3db9 100644 --- a/models/silver/defi/lending/venus/silver__venus_borrows.sql +++ b/models/silver/defi/lending/venus/silver__venus_borrows.sql @@ -55,6 +55,8 @@ venus_borrows AS ( asset_details ) AND topics [0] :: STRING = '0x13ed6866d4e1ee6da46f845c46d7e54120883d75c5ea9a2dacc1c4ca8984ab80' + AND tx_status = 'SUCCESS' + {% if is_incremental() %} AND _inserted_timestamp >= ( SELECT diff --git a/models/silver/defi/lending/venus/silver__venus_deposits.sql b/models/silver/defi/lending/venus/silver__venus_deposits.sql index ef58a2a..39956bc 100644 --- a/models/silver/defi/lending/venus/silver__venus_deposits.sql +++ b/models/silver/defi/lending/venus/silver__venus_deposits.sql @@ -52,6 +52,7 @@ venus_deposits AS ( asset_details ) AND topics [0] :: STRING = '0xb4c03061fb5b7fed76389d5af8f2e0ddb09f8c70d1333abbb62582835e10accb' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/venus/silver__venus_liquidations.sql b/models/silver/defi/lending/venus/silver__venus_liquidations.sql index f5a81cb..896d197 100644 --- a/models/silver/defi/lending/venus/silver__venus_liquidations.sql +++ b/models/silver/defi/lending/venus/silver__venus_liquidations.sql @@ -55,6 +55,7 @@ venus_liquidations AS ( asset_details ) AND topics [0] :: STRING = '0x298637f684da70674f26509b10f07ec2fbc77a335ab1e7d6215a4b2484d8bb52' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/defi/lending/venus/silver__venus_repayments.sql b/models/silver/defi/lending/venus/silver__venus_repayments.sql index 921587b..7dcefa4 100644 --- a/models/silver/defi/lending/venus/silver__venus_repayments.sql +++ b/models/silver/defi/lending/venus/silver__venus_repayments.sql @@ -50,6 +50,8 @@ venus_repayments AS ( asset_details ) AND topics [0] :: STRING = '0x1a2a22cb034d26d1854bdc6666a5b91fe25efbbb5dcad3b0355478d6f5c362a1' + AND tx_status = 'SUCCESS' + {% if is_incremental() %} AND _inserted_timestamp >= ( SELECT diff --git a/models/silver/defi/lending/venus/silver__venus_withdraws.sql b/models/silver/defi/lending/venus/silver__venus_withdraws.sql index 33df53d..3e8dc6e 100644 --- a/models/silver/defi/lending/venus/silver__venus_withdraws.sql +++ b/models/silver/defi/lending/venus/silver__venus_withdraws.sql @@ -52,6 +52,7 @@ venus_redemptions AS ( asset_details ) AND topics [0] :: STRING = '0xbd5034ffbd47e4e72a94baa2cdb74c6fad73cb3bcdc13036b72ec8306f5a7646' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/nft/sales/silver__complete_nft_sales.sql b/models/silver/nft/sales/silver__complete_nft_sales.sql index da801f1..1aa49f7 100644 --- a/models/silver/nft/sales/silver__complete_nft_sales.sql +++ b/models/silver/nft/sales/silver__complete_nft_sales.sql @@ -38,11 +38,11 @@ WITH nft_base_models AS ( FROM {{ ref('silver__seaport_1_1_sales') }} -{% if is_incremental() and 'seaport_1_1' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'seaport_1_1' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -78,11 +78,11 @@ SELECT FROM {{ ref('silver__seaport_1_4_sales') }} -{% if is_incremental() and 'seaport_1_4' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'seaport_1_4' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -118,11 +118,11 @@ SELECT FROM {{ ref('silver__seaport_1_5_sales') }} -{% if is_incremental() and 'seaport_1_5' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'seaport_1_5' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -158,11 +158,11 @@ SELECT FROM {{ ref('silver__tofunft_sales') }} -{% if is_incremental() and 'tofunft' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'tofunft' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -198,11 +198,11 @@ SELECT FROM {{ ref('silver__pancakeswap_sales') }} -{% if is_incremental() and 'pancakeswap' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'pancakeswap' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -238,11 +238,11 @@ SELECT FROM {{ ref('silver__seaport_1_6_sales') }} -{% if is_incremental() and 'seaport_1_6' not in var('HEAL_CURATED_MODEL') %} +{% if is_incremental() and 'seaport_1_6' not in var('HEAL_MODELS') %} WHERE _inserted_timestamp >= ( SELECT - MAX(_inserted_timestamp) - INTERVAL '36 hours' + MAX(_inserted_timestamp) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -541,7 +541,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -569,7 +569,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) @@ -608,7 +608,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} ) diff --git a/models/silver/nft/silver__nft_transfers.sql b/models/silver/nft/silver__nft_transfers.sql index 26f7051..2e4d9ae 100644 --- a/models/silver/nft/silver__nft_transfers.sql +++ b/models/silver/nft/silver__nft_transfers.sql @@ -362,7 +362,7 @@ heal_model AS ( SELECT MAX( _inserted_timestamp - ) - INTERVAL '36 hours' + ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}' FROM {{ this }} )