liquid balance bug fix (#105)

data/testing__daily_balances.csv

@@ -1,47 +1,30 @@
-date,balance_type,address,currency,balance
-2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/D189335C6E4A68B513C10AB227BF1C1D38C746766278BA3EEB4FB14124F1D858,133770065
-2022-07-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/606,4686338563878355485050
-2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/690,47627625428299477685
-2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/497,34442316185562221868
-2022-07-17,liquid,osmo1ka4zctlzr2uee5s29y9qn2rcer3hcaunxvcptc,uosmo,20304
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/498,12939530377800458661
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/1,570089175584047961768
-2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/7F1A862E98185A286F011DD093D8BD2FA1B7CD1A723EC5E6C59F76692F1728F7,371983
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/00B6E60AD3D65CBEF5579AC8AF609527C0B57535B6E32D96C80A735344FD9DCC,3911192009
-2022-07-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/605,5039040092969689838656
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/BE1BB42D4BE3C30D50B68D7C41DB4DFCE9678E8EF8C539F6E6A9345048894FCC,209050
-2022-07-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/498,12939530377800458661
-2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/601,5543512741386206132
-2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/AA1C80225BCA7B32ED1FC6ABF8B8E899BEB48ECDB4B417FD69873C6D715F97E7,6216
-2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/EA3E1640F9B1532AB129A571203A0B9F789A7F14BB66E350DCBFA18E1A1931F0,967659
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/9BBA9A1C257E971E38C1422780CE6F0B0686F0A3085E2D61118D904BFE0F5F5E,68884880
-2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/560,18665319896957969882
-2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/5,156901013595522054654364
-2022-07-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/1,101398840842185453211
-2022-07-17,staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,uosmo,102334774
-2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/1,511070108766580702658
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/6AE98883D4D5D5FF9E50D7130F1305DA2FFA0C652D1DD9C123657C6B4EB2DF8A,125010149067303700000
-2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/9989AD6CCA39D1131523DB0617B50F6442081162294B4795E26746292467B525,860527644
-2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/498,109158534505780395464
-2022-07-17,superfluid staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/678,275748711600873358
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/987C17B11ABC2B20019178ACE62929FE9840202CE79498E29FE8E5CB02B7C0A4,864436739
-2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/987C17B11ABC2B20019178ACE62929FE9840202CE79498E29FE8E5CB02B7C0A4,9447867
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/D189335C6E4A68B513C10AB227BF1C1D38C746766278BA3EEB4FB14124F1D858,500008565
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/27394FB092D2ECCD56123C74F36E4C1F926001CEADA9CA97EA622B25F41E5EB2,8365193
-2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/690,56084938505868894999
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,uosmo,15452376
-2022-07-17,superfluid staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/9,127540954286104745272018
+DATE,BALANCE_TYPE,ADDRESS,CURRENCY,BALANCE
 2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/0EF15DF2F02480ADE0BB6E85D9EBB5DAEA2836D3860E9F97F9AADE4F57A31AA0,825640692
-2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/CBA34207E969623D95D057D9B11B0C8B32B89A71F170577D982FDDE623813FFC,1149014
-2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/9,1277058429129285545890
-2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/600,68619721244831
+2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/D189335C6E4A68B513C10AB227BF1C1D38C746766278BA3EEB4FB14124F1D858,500008565
 2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/553,2988730689240198679174476547
-2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,uosmo,2275039
+2022-07-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/498,12939530377800458661
+2022-07-17,superfluid staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/9,127540954286104745272018
+2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/690,56084938505868894999
+2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/601,5543512741386206132
+2022-07-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/1,101398840842185453211
+2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/AA1C80225BCA7B32ED1FC6ABF8B8E899BEB48ECDB4B417FD69873C6D715F97E7,6216
+2022-07-17,staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,uosmo,102334774
+2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/600,68619721244831
+2022-07-17,superfluid staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/678,275748711600873358
+2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/5,156901013595522054654364
+2022-07-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/605,5039040092969689838656
 2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/611,14262075310226757145
-2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/584,5211764319661687113
-2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/604,12258739039587559
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/46B44899322F3CD854D2D46DEEF881958467CDD4B3B10086DA49296BBED94BED,10278225
+2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/7F1A862E98185A286F011DD093D8BD2FA1B7CD1A723EC5E6C59F76692F1728F7,371983
 2022-07-17,staked,osmo1ka4zctlzr2uee5s29y9qn2rcer3hcaunxvcptc,uosmo,14885383
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/B9E0A1A524E98BB407D3CED8720EFEFD186002F90C1B1B7964811DD0CCC12228,218286123
+2022-07-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/9989AD6CCA39D1131523DB0617B50F6442081162294B4795E26746292467B525,860527644
+2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/604,12258739039587559
+2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/584,5211764319661687113
+2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/9,1277058429129285545890
 2022-07-17,staked,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,uosmo,100298879
-2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/52B1AA623B34EB78FD767CEA69E8D7FA6C9CFE1FBF49C5406268FD325E2CC2AC,4977123627
+2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/498,109158534505780395464
+2022-07-17,liquid,osmo1ka4zctlzr2uee5s29y9qn2rcer3hcaunxvcptc,uosmo,20304
+2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/1,511070108766580702658
+2022-07-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/606,4686338563878355485050
+2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/560,18665319896957969882
+2022-07-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/497,34442316185562221868
+2022-07-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,uosmo,15452376

models/silver/silver__latest_liquid_balances.sql

@@ -1,6 +1,6 @@
 {{ config(
   materialized = 'incremental',
-  unique_key = "CONCAT_WS('-', address, balance_type, block_id, currency)",
+  unique_key = "address",
   incremental_strategy = 'delete+insert',
   cluster_by = ['block_timestamp::date']
 ) }}

models/silver/silver__liquid_balances.sql

@@ -2,7 +2,7 @@
     materialized = 'incremental',
     unique_key = "CONCAT_WS('-', block_id, address, currency)",
     incremental_strategy = 'delete+insert',
-    cluster_by = ['block_timestamp'],
+    cluster_by = ['block_timestamp::DATE'],
 ) }}
 
 WITH base AS (
@@ -10,7 +10,6 @@ WITH base AS (
     SELECT
         bal.block_id,
         bl.block_timestamp,
-        'liquid' AS balance_type,
         bal.address,
         b.value :denom :: STRING AS currency,
         b.value :amount :: INT AS balance,
@@ -31,14 +30,14 @@ WITH base AS (
 
 {% if is_incremental() %}
 WHERE
-    _inserted_timestamp >= (
+    block_timestamp::DATE >= (
         SELECT
-            MAX(
-                _inserted_timestamp
-            )
+            DATEADD('day', -1,   MAX(
+                block_timestamp::DATE
+            ))
         FROM
             {{ this }}
-    )
+    )    
 {% endif %}
 
 qualify(ROW_NUMBER() over(PARTITION BY bal.block_id, bal.address, currency
@@ -49,12 +48,12 @@ tbl AS (
 
 {% if is_incremental() %}
 SELECT
-    address, balance, balance_type, block_id, block_timestamp, currency, _inserted_timestamp
+    address, balance, block_id, block_timestamp, currency, _inserted_timestamp
 FROM
     base
 UNION
 SELECT
-    address, balance, balance_type, block_id, block_timestamp, currency, _inserted_timestamp
+    address, balance, block_id, block_timestamp, currency, _inserted_timestamp
 FROM
     silver.latest_liquid_balances
 WHERE
@@ -65,7 +64,7 @@ FROM
     base)
 {% else %}
 SELECT
-    address, balance, balance_type, block_id, block_timestamp, currency, _inserted_timestamp
+    address, balance, block_id, block_timestamp, currency, _inserted_timestamp
 FROM
     base
 {% endif %}),
@@ -76,36 +75,44 @@ all_currency AS (
     FROM
         tbl
 ),
+all_bts AS (
+    SELECT
+        DISTINCT address,
+        block_timestamp,
+        block_id,
+        _inserted_timestamp
+    FROM
+        tbl
+),
 tmp AS (
     SELECT
-        A.address,
+        ab.address,
         COALESCE(
-            A.balance,
+            bal.balance,
             0
         ) AS balance,
-        A.balance_type,
-        A.block_id,
-        A.block_timestamp,
-        COALESCE(
-            A.currency,
-            b.currency
-        ) AS currency,
-        A._inserted_timestamp
+        ab.block_id,
+        ab.block_timestamp,
+        ac.currency,
+        ab._inserted_timestamp
     FROM
-        tbl A
-        LEFT JOIN all_currency b
-        ON A.currency IS NULL
-        AND A.address = b.address qualify(ROW_NUMBER() over(PARTITION BY A.address, balance_type, block_id, COALESCE(A.currency, b.currency)
+        all_bts ab
+        JOIN all_currency ac
+        ON ab.address = ac.address
+        LEFT JOIN tbl bal
+        ON ab.address = bal.address
+        AND ab.block_timestamp = bal.block_timestamp
+        AND ac.currency = bal.currency
+         qualify(ROW_NUMBER() over(PARTITION BY ab.address, ab.block_id, ac.currency
     ORDER BY
-        _inserted_timestamp DESC)) = 1
+        ab._inserted_timestamp DESC)) = 1
 ),
 dense_tmp AS (
     SELECT
         *,
         DENSE_RANK() over(
             PARTITION BY address,
-            currency,
-            balance_type
+            currency
             ORDER BY
                 block_id ASC
         ) AS rn
@@ -116,14 +123,12 @@ joined AS (
     SELECT
         t.address AS t_address,
         t.balance AS t_balance,
-        t.balance_type AS t_balance_type,
         t.block_id AS t_block_id,
         t.block_timestamp AS t_block_timestamp,
         t.currency AS t_currency,
         t._inserted_timestamp AS t_inserted_timestamp,
         t2.address AS t2_address,
         t2.balance AS t2_balance,
-        t2.balance_type AS t2_balance_type,
         t2.block_id AS t2_block_id,
         t2.block_timestamp AS t2_block_timestamp,
         t2.currency AS t2_currency,
@@ -154,12 +159,11 @@ joined AS (
         t2.rn AS t2_rn,
         MAX(
             t2.rn
-        ) over(PARTITION BY COALESCE(t.address, t2.address), COALESCE(t.balance_type, t2.balance_type), COALESCE(t.currency, t2.currency)) AS max_t2_rn
+        ) over(PARTITION BY COALESCE(t.address, t2.address), COALESCE(t.currency, t2.currency)) AS max_t2_rn
     FROM
         dense_tmp t full
         OUTER JOIN dense_tmp t2
         ON t.address = t2.address
-        AND t.balance_type = t2.balance_type
         AND t.currency = t2.currency
         AND t.rn = t2.rn - 1
 )
@@ -172,10 +176,7 @@ SELECT
         t2_balance,
         0
     ) AS balance,
-    COALESCE(
-        t2_balance_type,
-        t_balance_type
-    ) AS balance_type,
+    'liquid' AS balance_type,
     COALESCE(
         t2_currency,
         t_currency