historical yields logic

macros/cli_query.sql

@@ -1,7 +1,7 @@
 -- macros/print_query_results.sql
 
 {% macro execute_and_print_query() %}
-    {% set results = run_query("select * from bronze.defillama_historical_backfill_list") %}
+    {% set results = run_query("select distinct(model) from bronze.defillama_historical_backfill_list") %}
     
     {% if execute %}
         {% for row in results %}

models/defillama/bronze/bronze__defillama_historical_backfill_list.sql

@@ -1,62 +1,99 @@
 {{ config(
     materialized = 'table',
-    unique_key = 'defillama_historical_yields_id',
+    unique_key = ['model','unfilled_row_count'],
     tags = ['defillama']
 ) }}
 
-WITH pools as (
-    SELECT 
+WITH pools AS (
+
+    SELECT
         DISTINCT pool_id,
-            MAX(_inserted_timestamp::DATE) AS max_timestamp
-    FROM 
+        MAX(
+            _inserted_timestamp :: DATE
+        ) AS max_timestamp
+    FROM
         {{ ref('silver__defillama_historical_yields') }}
-    GROUP BY 1
-    HAVING CURRENT_DATE = max_timestamp
+    GROUP BY
+        1
+    HAVING
+        CURRENT_DATE = max_timestamp
 ),
-max_row as (
-    select
-        max(row_num) as max_row
-    from
+max_row AS (
+    SELECT
+        MAX(row_num) AS max_row
+    FROM
         {{ ref('silver__defillama_yields') }}
 ),
-backfill_pools as (
-    SELECT 
+backfill_pools AS (
+    SELECT
         pool_id,
         symbol,
-        row_num
-    FROM external_DEV.silver.defillama_yields
-    WHERE row_num BETWEEN 1 AND (select max_row from max_row)
-    AND POOL_ID NOT IN (select pool_id from pools)
+        row_num,
+        CASE
+            WHEN row_num > 0
+            AND row_num < 3301 THEN '100'
+            WHEN row_num >= 3301
+            AND row_num < 6601 THEN '200'
+            WHEN row_num >= 6601
+            AND row_num < 9901 THEN '300'
+            WHEN row_num >= 9901
+            AND row_num < 13200 THEN '400'
+            WHEN row_num >= 13200 THEN '500'
+        END AS model
+    FROM
+        external_DEV.silver.defillama_yields
+    WHERE
+        row_num BETWEEN 1
+        AND (
+            SELECT
+                max_row
+            FROM
+                max_row
+        )
+        AND pool_id NOT IN (
+            SELECT
+                pool_id
+            FROM
+                pools
+        )
 ),
-groupings as (
+groupings AS (
     SELECT
-        '100' as model,
-        1 as min_row,
-        3300 as max_row
+        '100' AS model,
+        1 AS min_row,
+        3300 AS max_row
     UNION ALL
     SELECT
-        '200' as model,
-        3301 as min_row,
-        6600 as max_row
+        '200' AS model,
+        3301 AS min_row,
+        6600 AS max_row
     UNION ALL
     SELECT
-        '300' as model,
-        6601 as min_row,
-        9900 as max_row
+        '300' AS model,
+        6601 AS min_row,
+        9900 AS max_row
     UNION ALL
     SELECT
-        '400' as model,
-        9901 as min_row,
-        12000 as max_row
+        '400' AS model,
+        9901 AS min_row,
+        12000 AS max_row
     UNION ALL
     SELECT
-        '500' as model,
-        12001 as min_row,
-        15000 as max_row
+        '500' AS model,
+        12001 AS min_row,
+        (
+            SELECT
+                MAX(row_num)
+            FROM
+                {{ ref('silver__defillama_yields') }}
+        ) AS max_row
 )
 SELECT
-    model
-FROM    
-    groupings
-WHERE
-    max_row >(select min(row_num) from backfill_pools)
+    g.model,
+    COUNT(*) AS unfilled_row_count
+FROM
+    backfill_pools g
+    LEFT JOIN groupings p
+    ON g.model = p.model
+GROUP BY
+    1

models/defillama/bronze/bronze__defillama_historical_yields_401_500.sql

@@ -5,13 +5,6 @@
 ) }}
 
 WITH 
-{# upstream_complete_check as (
-    select
-        'check'
-    FROM
-        {{ ref('bronze__defillama_historical_yields_201_300') }}
-), #}
-
 historical_yield AS (
 
 {% for item in range(100) %}

models/defillama/silver/silver__defillama_historical_yields.sql

@@ -1,6 +1,6 @@
 {{ config(
-    materialized = 'incremental',
-    unique_key = 'defillama_historical_yield_id',
+    materialized = 'table',
+    unique_key = 'defillama_historical_yields_id',
     tags = ['defillama']
 ) }}