update to backfill loop till filled

models/defillama/bronze/bronze__defillama_historical_yields_100.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = 'defillama_historical_yields_id',
-    tags = ['defillama']
+    tags = ['100']
 ) }}
 
 WITH historical_yield AS (

models/defillama/bronze/bronze__defillama_historical_yields_101_200.sql

@@ -1,17 +1,10 @@
 {{ config(
     materialized = 'incremental',
     unique_key = 'defillama_historical_yields_id',
-    tags = ['defillama','200']
+    tags = ['200']
 ) }}
 
 WITH 
-{# upstream_complete_check as (
-    select
-        'check'
-    FROM
-        {{ ref('bronze__defillama_historical_yields_101_200') }}
-), #}
-
 historical_yield AS (
 
 {% for item in range(100) %}

models/defillama/bronze/bronze__defillama_historical_yields_201_300.sql

@@ -1,17 +1,10 @@
 {{ config(
     materialized = 'incremental',
     unique_key = 'defillama_historical_yields_id',
-    tags = ['defillama','300']
+    tags = ['300']
 ) }}
 
 WITH 
-{# upstream_complete_check as (
-    select
-        'check'
-    FROM
-        {{ ref('bronze__defillama_historical_yields_101_200') }}
-), #}
-
 historical_yield AS (
 
 {% for item in range(100) %}

models/defillama/bronze/bronze__defillama_historical_yields_301_400.sql

@@ -1,17 +1,10 @@
 {{ config(
     materialized = 'incremental',
     unique_key = 'defillama_historical_yields_id',
-    tags = ['defillama','400']
+    tags = ['400']
 ) }}
 
 WITH 
-{# upstream_complete_check as (
-    select
-        'check'
-    FROM
-        {{ ref('bronze__defillama_historical_yields_201_300') }}
-), #}
-
 historical_yield AS (
 
 {% for item in range(100) %}

models/defillama/bronze/bronze__defillama_historical_yields_401_500.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = 'defillama_historical_yields_id',
-    tags = ['defillama','500']
+    tags = ['500']
 ) }}
 
 WITH 

models/defillama/silver/silver__defillama_historical_yields.sql

@@ -1,5 +1,5 @@
 {{ config(
-    materialized = 'table',
+    materialized = 'incremental',
     unique_key = 'defillama_historical_yields_id',
     tags = ['defillama']
 ) }}
@@ -10,21 +10,61 @@ WITH yield_union AS(
         *
     FROM
         {{ ref('bronze__defillama_historical_yields_100') }}
+{% if is_incremental() %}
+WHERE _inserted_timestamp >= (
+    SELECT
+        MAX(
+            _inserted_timestamp
+        )
+    FROM
+        {{ this }}
+)
+{% endif %}
     UNION ALL
     SELECT
         *
     FROM
         {{ ref('bronze__defillama_historical_yields_101_200') }}
+{% if is_incremental() %}
+WHERE _inserted_timestamp >= (
+    SELECT
+        MAX(
+            _inserted_timestamp
+        )
+    FROM
+        {{ this }}
+)
+{% endif %}
     UNION ALL
     SELECT
         *
     FROM
         {{ ref('bronze__defillama_historical_yields_201_300') }}
+{% if is_incremental() %}
+WHERE _inserted_timestamp >= (
+    SELECT
+        MAX(
+            _inserted_timestamp
+        )
+    FROM
+        {{ this }}
+)
+{% endif %}
     UNION ALL
     SELECT
         *
     FROM
         {{ ref('bronze__defillama_historical_yields_301_400') }}
+{% if is_incremental() %}
+WHERE _inserted_timestamp >= (
+    SELECT
+        MAX(
+            _inserted_timestamp
+        )
+    FROM
+        {{ this }}
+)
+{% endif %}
     UNION ALL
     SELECT
         *
@@ -34,4 +74,7 @@ WITH yield_union AS(
 SELECT
     *
 FROM
-    yield_union
+    yield_union qualify(ROW_NUMBER() over(PARTITION BY defillama_historical_yields_id
+ORDER BY
+    _inserted_timestamp DESC)) = 1
+

models/defillama/silver/silver__defillama_historical_yields.yml

@@ -0,0 +1,7 @@
+version: 2
+models:
+  - name: silver__defillama_historical_yields
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+            combination_of_columns:
+              - defillama_historical_yields_id

snowflake_call.sh

@@ -1,11 +1,24 @@
-# Execute the query and capture the output
-q_result=$(dbt run-operation execute_and_print_query)
+while true; do
+    # Execute the query and capture the output
+    q_result=$(dbt run-operation execute_and_print_query)
 
-# Use grep and sed to extract numbers, assuming they represent tag identifiers
-numbers=($(echo "$q_result" | grep '<agate.Row:' | sed "s/.*<agate.Row: ('\(.*\)')>.*/\1/"))
+    # Use grep and sed to extract numbers, assuming they represent tag identifiers
+    numbers=($(echo "$q_result" | grep '<agate.Row:' | sed "s/.*<agate.Row: ('\(.*\)')>.*/\1/"))
 
-# Format the extracted numbers as dbt tag selectors
-tag_selectors=$(printf "tag:%s " "${numbers[@]}")
+    # Format the extracted numbers as dbt tag selectors
+    tag_selectors=$(printf "tag:%s " "${numbers[@]}")
+    tag_selectors=$(echo "$tag_selectors" | sed 's/ $//') # Remove trailing space
+    echo $tag_selectors
 
-# Trim any trailing whitespace and output
-echo "$tag_selectors" | sed 's/ $//'
+    # Check if tag_selectors is not empty and matches the pattern
+    if [[ -n "$tag_selectors" && "$tag_selectors" =~ ^tag:[0-9]+ ]]; then
+        echo "Valid tags received: $tag_selectors"
+        dbt run --select "$tag_selectors"
+        
+        # Run the dependent models
+        dbt run -m models/defillama/silver/silver__defillama_historical_yields.sql models/defillama/bronze/bronze__defillama_historical_backfill_list.sql
+    else
+        echo "No valid tags to run, exiting loop"
+        break
+    fi
+done