From 3f14ebf571e2c2e65e01573c1d2cac68d8a1f575 Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 17 Jan 2024 09:49:22 -0700 Subject: [PATCH] heal var --- README.md | 11 +++++++++++ dbt_project.yml | 1 + ...silver_bridge__complete_bridge_activity.sql | 18 +++++++++--------- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ae24509..134a8af 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,17 @@ Default values are False * Usage: dbt run --var '{"UPDATE_UDFS_AND_SPS":True}' -m ... +To reload records in a curated complete table without a full-refresh, such as `silver_bridge.complete_bridge_activity`: +* HEAL_CURATED_MODEL +Default is an empty array [] +When item is included in var array [], incremental logic will be skipped for that CTE / code block +When item is not included in var array [] or does not match specified item in model, incremental logic will apply +Example set up: `{% if is_incremental() and 'axelar' not in var('HEAL_CURATED_MODEL') %}` + +* Usage: +Single CTE: dbt run --var '{"HEAL_CURATED_MODEL":"axelar"}' -m ... +Multiple CTEs: dbt run --var '{"HEAL_CURATED_MODEL":["axelar","across","celer_cbridge"]}' -m ... + ### Resources: - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers diff --git a/dbt_project.yml b/dbt_project.yml index b018735..9c68cec 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -77,6 +77,7 @@ vars: BALANCES_START: 0 BALANCES_END: 10000000 HEAL_MODEL: False + HEAL_CURATED_MODEL: [] START_GHA_TASKS: False API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}' diff --git a/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql b/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql index 9157170..d8d0f26 100644 --- a/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql +++ b/models/silver/defi/bridge/silver_bridge__complete_bridge_activity.sql @@ -32,7 +32,7 @@ WITH allbridge AS ( FROM {{ ref('silver_bridge__allbridge_sent') }} -{% if is_incremental() %} +{% if is_incremental() and 'allbridge' not in var('HEAL_CURATED_MODEL') %} WHERE _inserted_timestamp >= ( SELECT @@ -68,7 +68,7 @@ axelar AS ( FROM {{ ref('silver_bridge__axelar_contractcallwithtoken') }} -{% if is_incremental() %} +{% if is_incremental() and 'axelar' not in var('HEAL_CURATED_MODEL') %} WHERE _inserted_timestamp >= ( SELECT @@ -104,7 +104,7 @@ celer_cbridge AS ( FROM {{ ref('silver_bridge__celer_cbridge_send') }} -{% if is_incremental() %} +{% if is_incremental() and 'celer_cbridge' not in var('HEAL_CURATED_MODEL') %} WHERE _inserted_timestamp >= ( SELECT @@ -140,7 +140,7 @@ meson AS ( FROM {{ ref('silver_bridge__meson_transfers') }} -{% if is_incremental() %} +{% if is_incremental() and 'meson' not in var('HEAL_CURATED_MODEL') %} WHERE _inserted_timestamp >= ( SELECT @@ -176,7 +176,7 @@ stargate AS ( FROM {{ ref('silver_bridge__stargate_swap') }} -{% if is_incremental() %} +{% if is_incremental() and 'stargate' not in var('HEAL_CURATED_MODEL') %} WHERE _inserted_timestamp >= ( SELECT @@ -212,7 +212,7 @@ symbiosis AS ( FROM {{ ref('silver_bridge__symbiosis_synthesizerequest') }} -{% if is_incremental() %} +{% if is_incremental() and 'symbiosis' not in var('HEAL_CURATED_MODEL') %} WHERE _inserted_timestamp >= ( SELECT @@ -248,7 +248,7 @@ synapse_tb AS ( FROM {{ ref('silver_bridge__synapse_token_bridge') }} -{% if is_incremental() %} +{% if is_incremental() and 'synapse_tb' not in var('HEAL_CURATED_MODEL') %} WHERE _inserted_timestamp >= ( SELECT @@ -284,7 +284,7 @@ synapse_tbs AS ( FROM {{ ref('silver_bridge__synapse_tokenbridgeandswap') }} -{% if is_incremental() %} +{% if is_incremental() and 'synapse_tbs' not in var('HEAL_CURATED_MODEL') %} WHERE _inserted_timestamp >= ( SELECT @@ -320,7 +320,7 @@ wormhole AS ( FROM {{ ref('silver_bridge__wormhole_transfers') }} -{% if is_incremental() %} +{% if is_incremental() and 'wormhole' not in var('HEAL_CURATED_MODEL') %} WHERE _inserted_timestamp >= ( SELECT