diff --git a/.DS_Store b/.DS_Store index 70bb8ca..d000231 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index 49f147c..9eac41f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,16 @@ - target/ +dbt_modules/ +# newer versions of dbt use this directory instead of dbt_modules for test dependencies dbt_packages/ logs/ + +.venv/ +.python-version + +# Visual Studio Code files +*/.vscode +*.code-workspace +.history/ +**/.DS_Store +.vscode/ +.env \ No newline at end of file diff --git a/data/testing__daily_balances.csv b/data/testing__daily_balances.csv new file mode 100644 index 0000000..370f7b7 --- /dev/null +++ b/data/testing__daily_balances.csv @@ -0,0 +1,98 @@ +date,balance_type,address,currency,balance +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/B9E0A1A524E98BB407D3CED8720EFEFD186002F90C1B1B7964811DD0CCC12228,218286123 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/611,14262075310226757145 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/816,474300148053517041397 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/EA3E1640F9B1532AB129A571203A0B9F789A7F14BB66E350DCBFA18E1A1931F0,989020 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/600,0 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/5,0 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/46B44899322F3CD854D2D46DEEF881958467CDD4B3B10086DA49296BBED94BED,10278225 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/9,56 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/8061A06D3BD4D52C4A28FFECF7150D370393AF0BA661C3776C54FF32836C3961,627533208992626770463 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/498,0 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/46B44899322F3CD854D2D46DEEF881958467CDD4B3B10086DA49296BBED94BED,0 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/0CD3A0285E1341859B5E86B6AB7682F023D03E97607CCC1DC95706411D866DF7,20296078667397774116 +2022-10-17,liquid,osmo1ka4zctlzr2uee5s29y9qn2rcer3hcaunxvcptc,uosmo,213378 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/604,12258739039587559 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/52B1AA623B34EB78FD767CEA69E8D7FA6C9CFE1FBF49C5406268FD325E2CC2AC,4977123627 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/690,145366506822530545318 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/498,0 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/1,511070108766580702658 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/584,5211764319661687113 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/9712DBB13B9631EDFA9BF61B55F1B2D290B2ADB67E3A4EB3A875F3B6081B3B84,2984970875 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/768,0 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/604,12258739039587559 +2022-10-17,superfluid staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/678,275748711600873358 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/987C17B11ABC2B20019178ACE62929FE9840202CE79498E29FE8E5CB02B7C0A4,9983000 +2022-10-17,liquid,osmo1ka4zctlzr2uee5s29y9qn2rcer3hcaunxvcptc,ibc/D189335C6E4A68B513C10AB227BF1C1D38C746766278BA3EEB4FB14124F1D858,0 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/584,66 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/B3FB7128CE957DE1ADB687A919AA0786C77C62FB1280C07CDD78AEA032D56853,1633836 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/27394FB092D2ECCD56123C74F36E4C1F926001CEADA9CA97EA622B25F41E5EB2,197819 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/7F1A862E98185A286F011DD093D8BD2FA1B7CD1A723EC5E6C59F76692F1728F7,5885023 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/9BBA9A1C257E971E38C1422780CE6F0B0686F0A3085E2D61118D904BFE0F5F5E,68884880 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/00B6E60AD3D65CBEF5579AC8AF609527C0B57535B6E32D96C80A735344FD9DCC,3911192009 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/BE1BB42D4BE3C30D50B68D7C41DB4DFCE9678E8EF8C539F6E6A9345048894FCC,209050 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/CBA34207E969623D95D057D9B11B0C8B32B89A71F170577D982FDDE623813FFC,102043604 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,uosmo,8955296 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/56C276FC136E239449DCE664292DBEEF5795C4EF4B5B35DB98BD1C0948274867,1788657 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/553,2988730689240198679174476547 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/497,0 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/766,0 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/8A34AF0C1943FD0DFCDE9ADBF0B2C9959C45E87E6088EA2FC6ADACD59261B8A2,2353385 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/7ABF696369EFB3387DF22B6A24204459FE5EFD010220E8E5618DC49DB877047B,3266057 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/717,1203579 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/497,0 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/767,0 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/6AE98883D4D5D5FF9E50D7130F1305DA2FFA0C652D1DD9C123657C6B4EB2DF8A,12707 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/751,0 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/A8CA5EE328FA10C9519DF6057DA1F69682D28F7D0F5CCC7ECB72E3DCA2D157A4,993184 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,uosmo,6250 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/577,112 +2022-10-17,superfluid staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/560,-298645118351327518112 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/765,0 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/49C2B2C444B7C5F0066657A4DBF19D676E0D185FF721CFD3E14FA253BCB9BC04,29056216 +2022-10-17,superfluid staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/9,148019469955289306575849 +2022-10-17,staked,osmo1ka4zctlzr2uee5s29y9qn2rcer3hcaunxvcptc,uosmo,15313591 +2022-10-17,staked,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,uosmo,103309510 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/BE1BB42D4BE3C30D50B68D7C41DB4DFCE9678E8EF8C539F6E6A9345048894FCC,0 +2022-10-17,superfluid staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/601,0 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/9,0 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/1,570089175584047961768 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/0EF15DF2F02480ADE0BB6E85D9EBB5DAEA2836D3860E9F97F9AADE4F57A31AA0,825640692 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/D189335C6E4A68B513C10AB227BF1C1D38C746766278BA3EEB4FB14124F1D858,133770065 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/613,524 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/600,0 +2022-10-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/803,0 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/987C17B11ABC2B20019178ACE62929FE9840202CE79498E29FE8E5CB02B7C0A4,864436739 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/27394FB092D2ECCD56123C74F36E4C1F926001CEADA9CA97EA622B25F41E5EB2,0 +2022-10-17,staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,uosmo,129716344 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/621,122 +2022-10-17,superfluid staked,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/604,12258739039587559 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/756,0 +2022-10-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/606,0 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/0954E1C28EB7AF5B72D24F3BC2B47BBB2FDF91BDDFD57B74B99E133AED40972A,0 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/FE2CD1E6828EC0FAB8AF39BAC45BC25B965BA67CCBC50C13A14BD610B0D1E2C4,2 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/E6931F78057F7CC5DA0FD6CEF82FF39373A6E0452BF1FD76910B93292CF356C1,88456387 +2022-10-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/605,0 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/771,12627929653 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/560,-722862402734530320250 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/D189335C6E4A68B513C10AB227BF1C1D38C746766278BA3EEB4FB14124F1D858,1598884234 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/9989AD6CCA39D1131523DB0617B50F6442081162294B4795E26746292467B525,2781427948 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/611,0 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/0EF15DF2F02480ADE0BB6E85D9EBB5DAEA2836D3860E9F97F9AADE4F57A31AA0,1561 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/423967B46B1A51D78619085105B04FCFA14F7CBC0BE7539A316B2DCDFC7D8C07,1743337 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/608,1544090 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/4E5444C35610CC76FC94E7F7886B93121175C28262DDFDDE6F84E82BF2425452,10000 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/601,5543512741386206132 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/C140AFD542AE77BD7DCC83F13FDD8C5E5BB8C4929785E6EC2F4C636F98F17901,1 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/239A507997222805E441956EBE8087D7E2D05D6535C6D4C75EF8DCF83B3DE1A1,1804817 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/1DE9A32D603EE05E9B8619DF24D90FD518C932AE0B5B3209A986B7262DBADE2A,11348094 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/AA1C80225BCA7B32ED1FC6ABF8B8E899BEB48ECDB4B417FD69873C6D715F97E7,23178 +2022-10-17,superfluid staked,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/1,0 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/498,12939530377800458661 +2022-10-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/1,101398840842185453211 +2022-10-17,locked liquidity,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,gamm/pool/498,0 +2022-10-17,locked liquidity,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/690,0 +2022-10-17,liquid,osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp,ibc/6AE98883D4D5D5FF9E50D7130F1305DA2FFA0C652D1DD9C123657C6B4EB2DF8A,10953 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,gamm/pool/637,13552520695 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/41999DF04D9441DAC0DF5D8291DF4333FBCBA810FFD63FDCE34FDF41EF37B6F7,11440 +2022-10-17,liquid,osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m,ibc/9BBC8BD41DC282F0DF4709ED7562F5B12F161D8DD391386B79EF179AA50621C3,234 \ No newline at end of file diff --git a/data/testing__locked_liquidity_balances.csv b/data/testing__locked_liquidity_balances.csv new file mode 100644 index 0000000..26dd392 --- /dev/null +++ b/data/testing__locked_liquidity_balances.csv @@ -0,0 +1,21 @@ +block_id,balance_type,address,lock_id,currency,balance +6532478,locked liquidity,osmo1d4yhw05y7mx82r5kvslcfftlkwtf9nvvyjxwcy,1482952,gamm/pool/722,134080549750170422358 +6532468,locked liquidity,osmo1pxu0splyl3skmw7xvwsq64760enaa279d5pc7m,1336898,gamm/pool/15,0 +6532489,locked liquidity,osmo10tv5zp58j90v926gpgn26g97arcmezar0dpynx,1519420,gamm/pool/803,9473404623772224284 +6532483,locked liquidity,osmo1k2q4lcv0ev8zr8j5jmt3907ycq74u63kefgy89,1515270,gamm/pool/678,90192458787151947823 +6532483,locked liquidity,osmo1sn93u7zp7v2guan0v69vyxqltl2rykt22t9cz7,1519419,gamm/pool/5,1680696246664686659241031 +6532463,locked liquidity,osmo1sn93u7zp7v2guan0v69vyxqltl2rykt22t9cz7,1519418,gamm/pool/3,8161297487936074614751 +6532440,locked liquidity,osmo1e937k54p7fhf0wprl5r54aty3akq7wxgrtceps,870214,gamm/pool/604,0 +6532445,locked liquidity,osmo1sn93u7zp7v2guan0v69vyxqltl2rykt22t9cz7,1519417,gamm/pool/678,10238753264947662642 +6532483,locked liquidity,osmo1eklq74xn54rsjc3rs5eynz3gfsrmmcrjpwyear,1506675,gamm/pool/806,137567224350931487 +6532461,locked liquidity,osmo1nsrrdnwlxxcqc506mscpxrfd8yhvfxzzhp23jn,1491892,gamm/pool/722,2700084557767665719310 +6532481,locked liquidity,osmo1a3r3uhfv870euw40fdsxelpmv2a3ua695kw0gn,1491913,gamm/pool/678,96384629445563903165 +6532483,locked liquidity,osmo128z54vzthsv83q3w5cwjn0qu0nrc6ww88szrqz,1440696,gamm/pool/722,159167371467601548396 +6532438,locked liquidity,osmo1d4hy8hph6sctn4fl2qkg3v9f4uk6zpjd08wsg6,1517935,gamm/pool/812,641123894029168475 +6532433,locked liquidity,osmo15spsaq3tqusu5xd3u6jmtmpjh3h5lcmmwrmuav,1479019,gamm/pool/778,689286359032672400161 +6532456,locked liquidity,osmo1v08k9f34uun46qeddmaxh5ssk8lz0jnvzuz0pe,1185665,gamm/pool/1,12966138136820230549757 +6532451,locked liquidity,osmo17qg3pf85kj56z8lye25kpwvjx6cgftq0c55r72,1513723,gamm/pool/806,8291207189609691 +6532454,locked liquidity,osmo1hmzk8ngj5zx4gxt80n8z72r50zxvlpk8ggsata,1335335,gamm/pool/629,0 +6532457,locked liquidity,osmo1h9pzv3plm7wcqwpkuyl6wx7vqy7rmnur6cz5p5,1485915,gamm/pool/712,0 +6532477,locked liquidity,osmo14w9sjmkuk9fcnz9ctttfswc7t3ne9fd5ek8sux,1487123,gamm/pool/678,1604693868143432599 +6532496,locked liquidity,osmo1sn93u7zp7v2guan0v69vyxqltl2rykt22t9cz7,1519421,gamm/pool/7,189219298449626813632949 \ No newline at end of file diff --git a/data/testing__superfluid_staked_balances.csv b/data/testing__superfluid_staked_balances.csv new file mode 100644 index 0000000..f9a649b --- /dev/null +++ b/data/testing__superfluid_staked_balances.csv @@ -0,0 +1,22 @@ +block_id,balance_type,address,lock_id,currency,balance +6503532,superfluid staked,osmo17zu5fd0u950ztnrv7yuku0cmum7rf8ehv3w7te,1518472,gamm/pool/481,1496747716695529656 +6503516,superfluid staked,osmo1sqgft4nkvj3ve2z8e37wn45jts2vp7xnqs8mcl,1491667,gamm/pool/722,602491847403950125045 +6503516,superfluid staked,osmo168aglajczs39lnk773c0gm89fz2jcxhkvswd6q,1410556,gamm/pool/712,1462418922784726421048 +6503500,superfluid staked,osmo186pn7jsrk7v83km044p0lfq0f5chtg4e0cp4v0,1514822,gamm/pool/678,29675489286704706282 +6503499,superfluid staked,osmo1ymvkf7cm8ls4pn0wcx4avcludmu6udrlmlyje4,1518470,gamm/pool/704,414050484824597584 +6503494,superfluid staked,osmo1nxrv5zazfhr9tjkrj9txg48rd7se22truqu6m7,1515857,gamm/pool/712,377706199513571281020 +6503478,superfluid staked,osmo1tzurd5r2mkn3uqfwekgs2gnw2dq0kuedtw20tr,1448737,gamm/pool/704,20279468919418634675 +6503466,superfluid staked,osmo16qh8dsxmrksycfnx9e987akjzkz2ql0wj2s9fn,1414340,gamm/pool/712,10434807775286508536 +6503466,superfluid staked,osmo1m8sxn5fnn74ftw2npw5d2fu4gxlha64ttur36x,1514423,gamm/pool/722,41306522651339372552 +6503459,superfluid staked,osmo1sekdhysy9e3ecjyzh69nwcrtch56r72wzxzv8v,1510678,gamm/pool/704,7848659752914243496 +6503453,superfluid staked,osmo10pe6tkpyxa8x6wtz9xsgze9u09fuwvasn5c3wt,1341428,gamm/pool/678,1485864211436078977 +6503420,superfluid staked,osmo134ea3yazv5rfr7ydtdq5czxw0sl4aa2ufrgwyf,904083,gamm/pool/1,639538360009237404643 +6503420,superfluid staked,osmo1glkwwjmd75j9a8rg5m6xxk7lzzzdakf0dj93ud,1355941,gamm/pool/678,3542477066453780805825 +6503395,superfluid staked,osmo1p0w387aud3wtgme22lcyctpz6wu3erzn3j7k7v,846543,gamm/pool/604,7719680153106079 +6503391,superfluid staked,osmo1xrfkdd6pamquamlqxvdlmh86kn9jxvk3saypjp,1451563,gamm/pool/722,6765059205335916395661 +6503389,superfluid staked,osmo1rgh58aew3sy0hqgfc2wkahk2s66cpud88q50fl,1518467,gamm/pool/678,307435023179693904 +6503385,superfluid staked,osmo1uejqm7ta3m82g7yw9t9wzz59z2rntpwkqtq3sq,1176116,gamm/pool/1,1241936752150148649017 +6503379,superfluid staked,osmo1838fvucr5g86udetp9vzc8cqmwr5sffq5edyka,1472310,gamm/pool/674,3900073004612675873 +6503375,superfluid staked,osmo1c284d7akq0l3sddxyrvw9j2nnn84u3phgsrkxz,1398513,gamm/pool/678,3341782944867417600683 +6503338,superfluid staked,osmo1fydcvenphk48r4axky70wqpewesp0spm6pg3xv,1410757,gamm/pool/722,105857639655819564025 +6503338,superfluid staked,osmo1smvk7pten7tkhlwefjhranertgnjea9gredupg,1518106,gamm/pool/678,6439585860615944576 \ No newline at end of file diff --git a/macros/tests/compare_model_subset.sql b/macros/tests/compare_model_subset.sql new file mode 100644 index 0000000..9b00eba --- /dev/null +++ b/macros/tests/compare_model_subset.sql @@ -0,0 +1,29 @@ +{% test compare_model_subset(model, compare_model, compare_columns, model_condition) %} + +{% set compare_cols_csv = compare_columns | join(', ') %} + +with a as ( + select {{compare_cols_csv}} from {{ model }} + {{ model_condition }} +), +b as ( + select {{compare_cols_csv}} from {{ compare_model }} +), +a_minus_b as ( + select * from a + {{ dbt_utils.except() }} + select * from b +), +b_minus_a as ( + select * from b + {{ dbt_utils.except() }} + select * from a +), +unioned as ( + select 'in_actual_not_in_expected' as which_diff, a_minus_b.* from a_minus_b + union all + select 'in_expected_not_in_actual' as which_diff, b_minus_a.* from b_minus_a +) +select * from unioned + +{% endtest %} \ No newline at end of file diff --git a/models/core/core__fact_locked_liquidity_actions.sql b/models/core/core__fact_locked_liquidity_actions.sql new file mode 100644 index 0000000..203ba1a --- /dev/null +++ b/models/core/core__fact_locked_liquidity_actions.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + A.block_id, + A.block_timestamp, + A.blockchain, + A.chain_id, + A.tx_id, + A.tx_status, + A.msg_group, + A.msg_type, + A.msg_action, + A.msg_action_description, + A.locker_address, + A.lock_id, + A.amount, + A.currency, + A.decimal, + A.pool_id, + A.lock_duration, + A.unlock_time, + A.is_superfluid, + A.unpool_new_lock_ids +FROM + {{ ref('silver__locked_liquidity_actions') }} A diff --git a/models/core/core__fact_locked_liquidity_actions.yml b/models/core/core__fact_locked_liquidity_actions.yml new file mode 100644 index 0000000..55f5b93 --- /dev/null +++ b/models/core/core__fact_locked_liquidity_actions.yml @@ -0,0 +1,81 @@ +version: 2 +models: + - name: core__fact_locked_liquidity_actions + description: Includes all actions related to locked LP tokens + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_STATUS + description: "{{ doc('tx_status') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_GROUP + description: "{{ doc('msg_group') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_ACTION + description: "{{ doc('msg_action') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_ACTION_DESCRIPTION + description: "{{ doc('msg_action_description') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LOCKER_ADDRESS + description: "{{ doc('locker_address') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LOCK_ID + description: "{{ doc('lock_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: POOL_ID + description: "{{ doc('pool_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: AMOUNT + description: "{{ doc('amount') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CURRENCY + description: "{{ doc('currency') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: DECIMAL + description: "{{ doc('decimal') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LOCK_DURATION + description: "{{ doc('lock_duration') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: UNLOCK_TIME + description: "{{ doc('unlock_time') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: IS_SUPERFLUID + description: "{{ doc('is_superfluid') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: UNPOOL_NEW_LOCK_IDS + description: "{{ doc('unpool_new_lock_ids') }}" + tests: + - dbt_expectations.expect_column_to_exist diff --git a/models/core/core__fact_superfluid_staking.sql b/models/core/core__fact_superfluid_staking.sql index ca7e5f9..3ab0da5 100644 --- a/models/core/core__fact_superfluid_staking.sql +++ b/models/core/core__fact_superfluid_staking.sql @@ -2,21 +2,20 @@ materialized = 'view' ) }} -SELECT - block_id, - block_timestamp, - blockchain, - chain_id, - tx_id, - tx_status, - tx_caller_address, - action, - delegator_address, - amount, - currency, - decimal, - validator_address, - lock_id, - original_superfluid_delegate_tx_ID -FROM - {{ ref('silver__superfluid_staking') }} \ No newline at end of file +SELECT + block_id, + block_timestamp, + blockchain, + chain_id, + tx_id, + tx_status, + {# tx_caller_address, #} + action, + delegator_address, + amount, + currency, + DECIMAL, + validator_address, + lock_id +FROM + {{ ref('silver__superfluid_staking') }} diff --git a/models/core/core__fact_superfluid_staking.yml b/models/core/core__fact_superfluid_staking.yml index 7c9f79a..728db17 100644 --- a/models/core/core__fact_superfluid_staking.yml +++ b/models/core/core__fact_superfluid_staking.yml @@ -27,10 +27,6 @@ models: description: "{{ doc('tx_status') }}" tests: - dbt_expectations.expect_column_to_exist - - name: TX_CALLER_ADDRESS - description: "{{ doc('tx_caller_address') }}" - tests: - - dbt_expectations.expect_column_to_exist - name: ACTION description: "{{ doc('action') }}" tests: @@ -58,8 +54,4 @@ models: - name: LOCK_ID description: An ID corresponding to the locking step of the transaction. tests: - - dbt_expectations.expect_column_to_exist - - name: ORIGINAL_SUPERFLUID_DELEGATE_TX_ID - description: The transaction id when the user initialized the superfluid stake. - tests: - - dbt_expectations.expect_column_to_exist + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/descriptions/is_superfluid.md b/models/descriptions/is_superfluid.md new file mode 100644 index 0000000..82e4942 --- /dev/null +++ b/models/descriptions/is_superfluid.md @@ -0,0 +1,5 @@ +{% docs is_superfluid %} + +The length of time the LP toke takes to unlock + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/lock_duration.md b/models/descriptions/lock_duration.md new file mode 100644 index 0000000..cca51c2 --- /dev/null +++ b/models/descriptions/lock_duration.md @@ -0,0 +1,5 @@ +{% docs lock_duration %} + +The length of time the LP toke takes to unlock + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/lock_id.md b/models/descriptions/lock_id.md new file mode 100644 index 0000000..64fabbe --- /dev/null +++ b/models/descriptions/lock_id.md @@ -0,0 +1,5 @@ +{% docs lock_id %} + +The unique id related to the specifc address, pool, and lock duration + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/locker_address.md b/models/descriptions/locker_address.md new file mode 100644 index 0000000..b1c6c0e --- /dev/null +++ b/models/descriptions/locker_address.md @@ -0,0 +1,5 @@ +{% docs locker_address %} + +The address of user that performed an action related to locked lp tokens + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/msg_action.md b/models/descriptions/msg_action.md new file mode 100644 index 0000000..4a310e1 --- /dev/null +++ b/models/descriptions/msg_action.md @@ -0,0 +1,5 @@ +{% docs msg_action %} + +The attribute value when the message type = 'mesage' and the attribute key = 'action' + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/msg_action_description.md b/models/descriptions/msg_action_description.md new file mode 100644 index 0000000..42f5269 --- /dev/null +++ b/models/descriptions/msg_action_description.md @@ -0,0 +1,5 @@ +{% docs msg_action_description %} + +A user friendly description of what action occured + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/unlock_time.md b/models/descriptions/unlock_time.md new file mode 100644 index 0000000..7919373 --- /dev/null +++ b/models/descriptions/unlock_time.md @@ -0,0 +1,5 @@ +{% docs unlock_time %} + +When the LP token is fully unlocked and liquid again + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/unpool_new_lock_ids.md b/models/descriptions/unpool_new_lock_ids.md new file mode 100644 index 0000000..251415c --- /dev/null +++ b/models/descriptions/unpool_new_lock_ids.md @@ -0,0 +1,5 @@ +{% docs unpool_new_lock_ids %} + +The new lock ids generated when a lp position is unpooled + +{% enddocs %} \ No newline at end of file diff --git a/models/silver/silver__daily_balances.sql b/models/silver/silver__daily_balances.sql index d151b16..663e72a 100644 --- a/models/silver/silver__daily_balances.sql +++ b/models/silver/silver__daily_balances.sql @@ -1,264 +1,303 @@ {{ config( - materialized = 'incremental', - unique_key = "CONCAT_WS('-', date, address, balance_type, currency)", - incremental_strategy = 'delete+insert', - cluster_by = ['date'], + materialized = 'incremental', + unique_key = "CONCAT_WS('-', date, address, balance_type, currency)", + incremental_strategy = 'delete+insert', + cluster_by = ['date'], ) }} -WITH +WITH {% if is_incremental() %} - recent AS ( - SELECT - date, - balance_type, - address, - balance, - currency, - decimal - FROM {{ this }} - - WHERE date = ( - SELECT - DATEADD('day', -1, MAX(date)) - FROM {{ this }} - ) -), -new AS ( - SELECT - block_timestamp :: date AS date, - balance_type, - address, - balance, - currency, - decimal, - 1 AS RANK - FROM - {{ ref('silver__liquid_balances') }} - - WHERE block_timestamp :: date >= ( - SELECT - DATEADD('day', -1, MAX(DATE)) - FROM - {{ this }} - ) - - qualify(ROW_NUMBER() over (PARTITION BY block_timestamp :: date, address, balance_type, currency - ORDER BY - block_timestamp DESC)) = 1 - - UNION ALL - - SELECT - block_timestamp :: date AS date, - balance_type, - address, - balance, - currency, - decimal, - 1 AS RANK - FROM - {{ ref('silver__staked_balances') }} - - WHERE block_timestamp :: date >= ( - SELECT - DATEADD('day', -1, MAX(DATE)) - FROM - {{ this }} - ) - - qualify(ROW_NUMBER() over (PARTITION BY block_timestamp :: date, address, balance_type, currency - ORDER BY - block_timestamp DESC)) = 1 - -), - -incremental AS ( SELECT - date, - balance_type, - address, - balance, - currency, - decimal - FROM - ( + DATE, + balance_type, + address, + balance, + currency, + DECIMAL + FROM + {{ this }} + WHERE + DATE = ( SELECT - date, - balance_type, - address, - balance, - currency, - decimal, - 2 AS RANK - FROM - recent - - UNION - + DATEADD('day', -1, MAX(DATE)) + FROM + {{ this }}) + ), + NEW AS ( SELECT - date, - balance_type, - address, - balance, - currency, - decimal, + block_timestamp :: DATE AS DATE, + balance_type, + address, + balance, + currency, + DECIMAL, 1 AS RANK - FROM - new - ) + FROM + {{ ref('silver__liquid_balances') }} + WHERE + block_timestamp :: DATE >= ( + SELECT + DATEADD('day', -1, MAX(DATE)) + FROM + {{ this }}) qualify(ROW_NUMBER() over (PARTITION BY block_timestamp :: DATE, address, balance_type, currency + ORDER BY + block_timestamp DESC)) = 1 + UNION ALL + SELECT + block_timestamp :: DATE AS DATE, + balance_type, + address, + balance, + currency, + DECIMAL, + 1 AS RANK + FROM + {{ ref('silver__staked_balances') }} + WHERE + block_timestamp :: DATE >= ( + SELECT + DATEADD('day', -1, MAX(DATE)) + FROM + {{ this }}) qualify(ROW_NUMBER() over (PARTITION BY block_timestamp :: DATE, address, balance_type, currency + ORDER BY + block_timestamp DESC)) = 1 + UNION ALL + SELECT + block_timestamp, + balance_type, + address, + SUM(balance) AS balance, + currency, + DECIMAL, + 1 AS RANK + FROM + ( + SELECT + block_timestamp, + balance_type, + address, + balance, + currency, + DECIMAL, + 1 AS RANK + FROM + {{ ref('silver__locked_liquidity_balances') }} + WHERE + block_timestamp :: DATE >= ( + SELECT + DATEADD('day', -1, MAX(DATE)) + FROM + {{ this }}) + AND lock_id || '---' || block_timestamp :: DATE :: STRING NOT IN ( + SELECT + lock_id || '---' || block_timestamp :: DATE :: STRING + FROM + {{ ref('silver__superfluid_staked_balances') }} + WHERE + block_timestamp :: DATE >= ( + SELECT + DATEADD('day', -1, MAX(DATE)) + FROM + {{ this }}) + ) qualify(ROW_NUMBER() over (PARTITION BY block_timestamp :: DATE, address, balance_type, currency, lock_id + ORDER BY + block_timestamp DESC)) = 1 + UNION ALL + SELECT + block_timestamp, + balance_type, + address, + balance, + currency, + DECIMAL, + 1 AS RANK + FROM + {{ ref('silver__superfluid_staked_balances') }} + WHERE + block_timestamp :: DATE >= ( + SELECT + DATEADD('day', -1, MAX(DATE)) + FROM + {{ this }}) qualify(ROW_NUMBER() over (PARTITION BY block_timestamp :: DATE, address, balance_type, currency, lock_id + ORDER BY + block_timestamp DESC)) = 1 + ) liq + GROUP BY + block_timestamp, + balance_type, + address, + currency, + DECIMAL + ), + incremental AS ( + SELECT + DATE, + balance_type, + address, + balance, + currency, + DECIMAL + FROM + ( + SELECT + DATE, + balance_type, + address, + balance, + currency, + DECIMAL, + 2 AS RANK + FROM + recent + UNION + SELECT + DATE, + balance_type, + address, + balance, + currency, + DECIMAL, + 1 AS RANK + FROM + NEW + ) qualify(ROW_NUMBER() over (PARTITION BY DATE, address, balance_type, currency + ORDER BY + RANK ASC)) = 1 + ), + {% endif %} - qualify(ROW_NUMBER() over (PARTITION BY date, address, balance_type, currency - ORDER BY - RANK ASC)) = 1 - -), -{% endif %} - - -base AS ( - - {% if is_incremental() %} - - SELECT - date AS block_timestamp, - balance_type, - address, - balance, - currency, - decimal - FROM - incremental - - {% else %} - - SELECT - block_timestamp, - balance_type, - address, - balance, - currency, - decimal - FROM - {{ ref('silver__liquid_balances') }} - - UNION ALL - - SELECT - block_timestamp, - balance_type, - address, - balance, - currency, - decimal - FROM - {{ ref('silver__staked_balances') }} - - {% endif %} -), + base AS ( +{% if is_incremental() %} +SELECT + DATE AS block_timestamp, balance_type, address, balance, currency, DECIMAL +FROM + incremental +{% else %} +SELECT + block_timestamp, balance_type, address, balance, currency, DECIMAL +FROM + {{ ref('silver__liquid_balances') }} +UNION ALL +SELECT + block_timestamp, balance_type, address, balance, currency, DECIMAL +FROM + {{ ref('silver__staked_balances') }} +UNION ALL +SELECT + block_timestamp, balance_type, address, SUM(balance) AS balance, currency, DECIMAL +FROM + ( +SELECT + block_timestamp, balance_type, address, balance, currency, DECIMAL +FROM + {{ ref('silver__locked_liquidity_balances') }} +WHERE + lock_id || '---' || block_timestamp :: DATE :: STRING NOT IN ( +SELECT + lock_id || '---' || block_timestamp :: DATE :: STRING +FROM + {{ ref('silver__superfluid_staked_balances') }}) +UNION ALL +SELECT + block_timestamp, balance_type, address, balance, currency, DECIMAL +FROM + {{ ref('silver__superfluid_staked_balances') }}) liq +GROUP BY + block_timestamp, balance_type, address, currency, DECIMAL +{% endif %}), address_ranges AS ( SELECT - address, - balance_type, - currency, - decimal, + address, + balance_type, + currency, + DECIMAL, MIN( - block_timestamp :: date - ) AS min_block_date, + block_timestamp :: DATE + ) AS min_block_date, MAX ( - CURRENT_TIMESTAMP :: date + CURRENT_TIMESTAMP :: DATE ) AS max_block_date - FROM - base - GROUP BY - address, - balance_type, - currency, - decimal -), - -ddate AS ( - SELECT - hour :: date AS date - FROM - {{ source( - 'shared2', - 'hours' - ) }} - GROUP BY date -), - -all_dates AS ( - SELECT - d.date, - a.balance_type, - a.address, - a.currency, - a.decimal - FROM - ddate d - - LEFT JOIN address_ranges a - ON d.date - BETWEEN a.min_block_date - AND a.max_block_date - - WHERE - a.address IS NOT NULL -), - -osmosis_balances AS ( - SELECT - block_timestamp, - balance_type, - address, - balance, - currency, - decimal FROM base - - qualify(ROW_NUMBER() over (PARTITION BY block_timestamp :: date, address, balance_type, currency - ORDER BY - block_timestamp DESC)) = 1 -), - + GROUP BY + address, + balance_type, + currency, + DECIMAL +), +ddate AS ( + SELECT + HOUR :: DATE AS DATE + FROM + {{ source( + 'shared2', + 'hours' + ) }} + GROUP BY + DATE +), +all_dates AS ( + SELECT + d.date, + A.balance_type, + A.address, + A.currency, + A.decimal + FROM + ddate d + LEFT JOIN address_ranges A + ON d.date BETWEEN A.min_block_date + AND A.max_block_date + WHERE + A.address IS NOT NULL +), +osmosis_balances AS ( + SELECT + block_timestamp, + balance_type, + address, + balance, + currency, + DECIMAL + FROM + base qualify(ROW_NUMBER() over (PARTITION BY block_timestamp :: DATE, address, balance_type, currency + ORDER BY + block_timestamp DESC)) = 1 +), balance_temp AS ( SELECT - d.date, - d.balance_type, - d.address, - b.balance, + d.date, + d.balance_type, + d.address, + b.balance, d.currency, d.decimal - - FROM - all_dates d - - LEFT JOIN osmosis_balances b - ON d.date = b.block_timestamp :: date - AND d.address = b.address - AND d.currency = b.currency - AND d.balance_type = b.balance_type + FROM + all_dates d + LEFT JOIN osmosis_balances b + ON d.date = b.block_timestamp :: DATE + AND d.address = b.address + AND d.currency = b.currency + AND d.balance_type = b.balance_type ) - SELECT - date, - balance_type, - address, - currency, - decimal, + DATE, + balance_type, + address, + currency, + DECIMAL, LAST_VALUE( balance ignore nulls ) over( - PARTITION BY address, - currency, - balance_type - ORDER BY - DATE ASC rows unbounded preceding - ) AS balance -FROM - balance_temp \ No newline at end of file + PARTITION BY address, + currency, + balance_type + ORDER BY + DATE ASC rows unbounded preceding + ) AS balance +FROM + balance_temp diff --git a/models/silver/silver__daily_balances.yml b/models/silver/silver__daily_balances.yml index 2f24bee..569f02e 100644 --- a/models/silver/silver__daily_balances.yml +++ b/models/silver/silver__daily_balances.yml @@ -9,6 +9,17 @@ models: - ADDRESS - BALANCE_TYPE - CURRENCY + - compare_model_subset: + name: silver__daily_balances_business_logic_test + compare_model: ref('testing__daily_balances') + compare_columns: + - date + - balance_type + - address + - currency + - balance + model_condition: "where date = '2022-10-17' + and ADDRESS IN ('osmo195khh6nr3zsvaxkm53wavkxv46uucdlatdux5m','osmo1ka4zctlzr2uee5s29y9qn2rcer3hcaunxvcptc','osmo1fjh798fyn9vqldc8ru09c99r89v5mt3czu4jfp')" columns: - name: DATE description: The day the balance was recorded on. diff --git a/models/silver/silver__locked_liquidity_actions.sql b/models/silver/silver__locked_liquidity_actions.sql new file mode 100644 index 0000000..c2fecd3 --- /dev/null +++ b/models/silver/silver__locked_liquidity_actions.sql @@ -0,0 +1,342 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'] +) }} + +WITH + +{% if is_incremental() %} +max_date AS ( + + SELECT + MAX( + _inserted_timestamp + ) _inserted_timestamp + FROM + {{ this }} +), +{% endif %} + +base_msg_atts AS ( + SELECT + A.block_id, + A.block_timestamp, + A.blockchain, + A.chain_id, + A.tx_id, + 'SUCCEEDED' AS tx_status, + A.msg_group, + A.msg_type, + A.attribute_key, + A.attribute_value, + COALESCE( + b.lock_id :: INT, + C.lock_id :: INT + ) AS lock_id, + A._inserted_timestamp + FROM + {{ ref('silver__msg_attributes') }} A + LEFT JOIN {{ ref('silver__locked_liquidity_transactions') }} + b + ON A.tx_id = b.tx_id + AND A.msg_group = b.msg_group + LEFT JOIN {{ ref('silver__superfluid_actions') }} C + ON A.tx_id = C.tx_id + AND A.msg_group = C.msg_group + WHERE + COALESCE( + b.tx_id, + C.tx_id + ) IS NOT NULL + AND ( + ( + A.msg_type = 'message' + AND A.attribute_key = 'action' + ) + OR A.msg_type IN ( + 'add_tokens_to_lock', + 'begin_unlock', + 'begin_unlock_all', + {# 'burn', #} --these are just the placeholder osmo we can ignore + 'lock_tokens', + 'superfluid_delegate', + 'superfluid_increase_delegation', + 'superfluid_unbond_lock', + 'superfluid_undelegate', + 'unbond', + 'unlock', + 'unlock_tokens', + 'unpool_pool_id' + ) + AND attribute_key IN ( + 'amount', + 'owner', + 'burner', + 'duration', + 'unlock_time', + 'sender', + 'denom', + 'new_lock_ids' + ) + ) --weird transactions that break the logic + AND A.tx_id NOT IN ( + '523CBB1403A90A2A45A90ADFFC17F72100B99C286BD66DEDF22DD7F8A825127D', + 'B26B72516A670B4FFD31F4F7853E65F7463F7A46BDE61800DC17A41F55AB87A3', + '34A6CEF2A87D6DB15DA1D7238D3A3BEABF8B4A1B460082B3C1F6C65DE5329CAC', + '504A0BD295DA63E28D55BC7C46575C6C49D9C2612D0AF118BA2A33A089A25A6D', + 'B312127A7914D26444DA2C1104122F9CB7D3B50940F079544775C7EA4EE4981D', + '413991DF25FF3A217BA42D84D811CABC4A580F12FA9A8BC204E45F22529185CB' + ) + +{% if is_incremental() %} +AND A._inserted_timestamp >= ( + SELECT + _inserted_timestamp + FROM + max_date +) +{% endif %} +), +tx_msg_flat AS ( + SELECT + block_id, + block_timestamp, + blockchain, + chain_id, + tx_id, + tx_status, + msg_group, + _inserted_timestamp, + LISTAGG( + DISTINCT msg_type, + '-' + ) within GROUP ( + ORDER BY + msg_type + ) AS msg_type, + lock_id, + OBJECT_AGG( + msg_type :: STRING || '--' || attribute_key :: STRING, + attribute_value :: variant + ) AS j + FROM + base_msg_atts + WHERE + msg_type <> 'message' + GROUP BY + block_id, + block_timestamp, + blockchain, + chain_id, + tx_id, + tx_status, + msg_group, + lock_id, + _inserted_timestamp +), +msg_based AS ( + SELECT + A.block_id, + A.block_timestamp, + A.blockchain, + A.chain_id, + A.tx_id, + A.tx_status, + A.msg_group, + A.msg_type, + A.lock_id, + b.attribute_value AS action, + CASE + WHEN b.attribute_value = '/osmosis.lockup.MsgExtendLockup' THEN 'extend lockup' + WHEN A.msg_type = 'unpool_pool_id' THEN 'unpool' + WHEN j :"lock_tokens--duration" IS NOT NULL THEN 'initial lock' + WHEN j :: STRING ILIKE '%unlock%' + OR j :: STRING ILIKE '%undelegate%' + OR j :: STRING ILIKE '%unbond%' + OR action ILIKE '%undelegate%' THEN 'unlock-undelegate' + WHEN j :: STRING LIKE '%add%' + OR j :: STRING LIKE '%increase%' THEN 'add to position' + WHEN j :: STRING ILIKE '%delegate%' THEN 'super upgrade' + END hybrid_action, + COALESCE( + j :"add_tokens_to_lock--amount", + j :"lock_tokens--amount", + j :"superfluid_increase_delegation--amount", + j :"burn--amount", + j :"unpool_pool_id--denom" + ) :: STRING AS amount, + COALESCE( + j :"add_tokens_to_lock--owner", + j :"lock_tokens--owner", + j :"begin_unlock--owner", + j :"unlock--owner", + j :"burn--burner", + j :"unpool_pool_id--sender" + ) :: STRING AS locker, + COALESCE( + j :"lock_tokens--duration", + j :"begin_unlock--duration", + j :"unlock--duration" + ) :: STRING AS DURATION, + NULLIF( + COALESCE( + j :"lock_tokens--unlock_time", + j :"begin_unlock--unlock_time", + j :"unlock--unlock_time" + ) :: STRING, + '0001-01-01 00:00:00 +0000 UTC' + ) AS unlock_time, + j :"unpool_pool_id--new_lock_ids" :: STRING AS new_lock_ids, + A._INSERTED_TIMESTAMP + FROM + tx_msg_flat A + LEFT JOIN base_msg_atts b + ON A.tx_id = b.tx_id + AND A.msg_group = b.msg_group + AND A.lock_id = b.lock_id + AND b.msg_type = 'message' +), +combo_with_super_undel AS ( + SELECT + * + FROM + msg_based + UNION ALL + SELECT + A.block_id, + A.block_timestamp, + A.blockchain, + A.chain_id, + A.tx_id, + A.tx_status, + A.msg_group, + A.msg_type, + A.lock_id, + 'unlock' AS action, + 'unlock' AS hybrid_action, + NULL :: STRING AS amount, + delegator_address AS locker, + NULL :: STRING AS DURATION, + NULL :: STRING AS unlock_time, + NULL :: STRING AS new_lock_ids, + A._INSERTED_TIMESTAMP + FROM + {{ ref('silver__superfluid_actions') }} A + WHERE + msg_type = '/osmosis.superfluid.MsgSuperfluidUndelegate' +), +tx_body AS ( + SELECT + tx_id, + tx_status, + msg_type, + msg_group, + delegator_address, + amount, + validator_address, + lock_id, + pool_id + FROM + {{ ref('silver__superfluid_actions') }} A + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + _inserted_timestamp + FROM + max_date + ) +{% endif %} +), +all_super_pools AS ( + SELECT + lock_id, + MIN(block_id) block_id + FROM + {{ ref('silver__superfluid_actions') }} + GROUP BY + lock_id +) +SELECT + A.block_id, + A.block_timestamp, + A.blockchain, + A.chain_id, + A.tx_id, + A.tx_status, + A.msg_group, + A.msg_type, + COALESCE( + b.msg_type, + A.action + ) AS msg_action, + A.hybrid_action AS msg_action_description, + A.locker AS locker_address, + A.lock_id, + CASE + WHEN A.amount LIKE 'gamm%' THEN NULL + WHEN A.amount LIKE '%uosmo' THEN REPLACE( + A.amount, + 'uosmo' + ) + ELSE SPLIT_PART( + TRIM( + REGEXP_REPLACE( + A.amount, + '[^[:digit:]]', + ' ' + ) + ), + ' ', + 0 + ) + END :: INT {# / pow( + 10, + 18 +) #} +AS amount, +CASE + WHEN A.amount LIKE 'gamm%' THEN A.amount + WHEN A.amount LIKE '%uosmo' THEN 'uosmo' + ELSE RIGHT(A.amount, LENGTH(A.amount) - LENGTH(SPLIT_PART(TRIM(REGEXP_REPLACE(A.amount, '[^[:digit:]]', ' ')), ' ', 0))) +END AS currency, +CASE + WHEN A.amount LIKE '%uosmo' THEN 6 + ELSE 18 +END AS DECIMAL, +COALESCE( + b.pool_id, + CASE + WHEN A.amount LIKE '%pool%' THEN RIGHT(A.amount, len(A.amount) - POSITION('pool', A.amount) -4) + END :: INT +) AS pool_id, +A.duration AS lock_duration, +A.unlock_time AS unlock_time, +CASE + WHEN b.tx_id IS NOT NULL + OR C.lock_id IS NOT NULL + AND A.block_id >= C.block_id THEN TRUE + ELSE FALSE +END is_superfluid, +A.new_lock_ids AS unpool_new_lock_ids, +concat_ws( + '-', + A.tx_id, + A.msg_group, + COALESCE( + A.lock_id, + -1 + ), + A.locker +) AS _unique_key, +A._INSERTED_TIMESTAMP +FROM + combo_with_super_undel A + LEFT JOIN tx_body b + ON A.tx_id = b.tx_id + AND A.msg_group = b.msg_group + LEFT JOIN all_super_pools C + ON A.lock_id = C.lock_id diff --git a/models/silver/silver__locked_liquidity_actions.yml b/models/silver/silver__locked_liquidity_actions.yml new file mode 100644 index 0000000..addd8bf --- /dev/null +++ b/models/silver/silver__locked_liquidity_actions.yml @@ -0,0 +1,88 @@ +version: 2 +models: + - name: silver__locked_liquidity_actions + description: Records of all LP token locking transactions that have occurred on Osmosis, dating back to the genesis block. These actions include lock, unlock, unpool, and all superfluid actions. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - MSG_GROUP + - LOCK_ID + - LOCKER_ADDRESS + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - not_null + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: TX_STATUS + description: "{{ doc('tx_status') }}" + tests: + - not_null + - name: MSG_GROUP + description: "{{ doc('msg_group') }}" + tests: + - not_null + - name: MSG_TYPE + description: "{{ doc('msg_type') }}" + tests: + - not_null + - name: MSG_ACTION + description: "{{ doc('action') }}" + tests: + - not_null + - name: MSG_ACTION_DESCRIPTION + description: "{{ doc('msg_action_description') }}" + tests: + - not_null + - name: LOCKER_ADDRESS + description: "{{ doc('locker_address') }}" + tests: + - not_null + - name: LOCK_ID + description: "{{ doc('lock_id') }}" + - name: AMOUNT + description: "{{ doc('amount') }}" + - name: CURRENCY + description: "{{ doc('currency') }}" + - name: POOL_ID + description: "{{ doc('pool_id') }}" + - name: LOCK_DURATION + description: "{{ doc('lock_duration') }}" + - name: UNLOCK_TIME + description: "{{ doc('unlock_time') }}" + - name: IS_SUPERFLUID + description: "{{ doc('is_superfluid') }}" + tests: + - not_null + - name: UNPOOL_NEW_LOCK_IDS + description: "{{ doc('unpool_new_lock_ids') }}" + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null diff --git a/models/silver/silver__locked_liquidity_balances.sql b/models/silver/silver__locked_liquidity_balances.sql new file mode 100644 index 0000000..f50092a --- /dev/null +++ b/models/silver/silver__locked_liquidity_balances.sql @@ -0,0 +1,179 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'], +) }} + +WITH all_staked AS ( + + SELECT + block_id, + block_timestamp, + locker_address AS address, + SUM(amount) amount, + currency, + DECIMAL, + lock_id, + _inserted_timestamp + FROM + {{ ref('silver__locked_liquidity_actions') }} + s + WHERE + msg_action_description IN ( + 'initial lock', + 'add to position' + ) + AND amount > 0 + +{% if is_incremental() %} +AND block_timestamp :: DATE >=( + SELECT + DATEADD('day', -2, MAX(block_timestamp)) + FROM + {{ this }}) + {% endif %} + GROUP BY + block_id, + block_timestamp, + locker_address, + currency, + DECIMAL, + lock_id, + _inserted_timestamp +), +undel_base AS ( + SELECT + DISTINCT block_id, + block_timestamp, + locker_address AS address, + currency, + DECIMAL, + lock_id, + msg_action_description, + tx_id, + _inserted_timestamp + FROM + {{ ref('silver__locked_liquidity_actions') }} + WHERE + msg_action_description IN ( + 'unlock', + 'unlock-undelegate', + 'unpool' + ) + +{% if is_incremental() %} +AND block_timestamp :: DATE >=( + SELECT + DATEADD('day', -2, MAX(block_timestamp)) + FROM + {{ this }}) + {% endif %} +), +unpool_lock_val AS ( + SELECT + A.lock_id, + A.currency, + A.locker_address, + b.block_id, + b.tx_id + FROM + {{ ref('silver__locked_liquidity_actions') }} A + JOIN undel_base b + ON b.msg_action_description = 'unpool' + AND A.locker_address = b.address + AND A.currency = b.currency + AND A.block_id < b.block_id + WHERE + A.lock_id IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY A.lock_id + ORDER BY + A.block_id DESC) = 1) +), +undel_bal AS ( + SELECT + A.lock_id, + A.currency, + A.decimal, + SUM(COALESCE(amount, 0)) amount + FROM + {{ ref('silver__locked_liquidity_actions') }} A + LEFT JOIN unpool_lock_val C + ON A.locker_address = C.locker_address + AND A.currency = C.currency + AND A.block_id <= C.block_id + LEFT JOIN undel_base b + ON A.lock_id = b.lock_id + LEFT JOIN undel_base bb + ON C.tx_id = bb.tx_id + WHERE + ( + bb.address IS NOT NULL + OR b.address IS NOT NULL + ) + AND A.amount > 0 + AND A.msg_action_description IN ( + 'initial lock', + 'add to position' + ) + GROUP BY + A.lock_id, + A.currency, + A.decimal +), +combine AS ( + SELECT + block_id, + block_timestamp, + address, + lock_id, + amount, + currency, + DECIMAL, + _inserted_timestamp + FROM + all_staked + UNION ALL + SELECT + A.block_id, + block_timestamp, + address, + b.lock_id, + -1 * b.amount, + b.currency, + b.decimal, + _inserted_timestamp + FROM + undel_base A + LEFT JOIN unpool_lock_val C + ON A.tx_id = C.tx_id + JOIN undel_bal b + ON COALESCE( + A.lock_id, + C.lock_id + ) = b.lock_id +) +SELECT + block_id, + block_timestamp, + 'locked liquidity' AS balance_type, + address, + lock_id, + currency, + DECIMAL, + SUM(amount) over( + PARTITION BY address, + currency, + lock_id + ORDER BY + block_timestamp ASC rows unbounded preceding + ) AS balance, + concat_ws( + '-', + block_id, + address, + lock_id, + currency + ) AS _unique_key, + _inserted_timestamp +FROM + combine diff --git a/models/silver/silver__locked_liquidity_balances.yml b/models/silver/silver__locked_liquidity_balances.yml new file mode 100644 index 0000000..9678e65 --- /dev/null +++ b/models/silver/silver__locked_liquidity_balances.yml @@ -0,0 +1,56 @@ +version: 2 +models: + - name: silver__locked_liquidity_balances + description: A table of each locked liquidity balance change for wallets on the Osmosis chain. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ADDRESS + - BLOCK_ID + - LOCK_ID + - CURRENCY + - compare_model_subset: + name: silver__locked_liquidity_balances_business_logic_test + compare_model: ref('testing__locked_liquidity_balances') + compare_columns: + - block_id + - balance_type + - address + - lock_id + - currency + - balance + model_condition: "where block_timestamp::date = '2022-10-20' + and block_ID IN (6532496,6532489,6532483,6532481,6532478,6532477,6532468,6532463,6532461, + 6532457,6532456,6532454,6532451,6532445,6532440,6532438,6532433)" + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - name: ADDRESS + description: "{{ doc('address') }}" + tests: + - not_null + - name: LOCK_ID + description: "{{ doc('lock_id') }}" + tests: + - not_null + - name: BALANCE + description: The amount of the currency the wallet held at the given time + tests: + - not_null + - name: CURRENCY + description: "{{ doc('currency') }}" + tests: + - not_null + - name: DECIMAL + description: "{{ doc('decimal') }}" + - name: _INSERTED_TIMESTAMP + description: "{{ doc('ingested_at') }}" + tests: + - not_null + \ No newline at end of file diff --git a/models/silver/silver__locked_liquidity_transactions.sql b/models/silver/silver__locked_liquidity_transactions.sql new file mode 100644 index 0000000..d00d015 --- /dev/null +++ b/models/silver/silver__locked_liquidity_transactions.sql @@ -0,0 +1,43 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['_inserted_timestamp::DATE'] +) }} + +SELECT + DISTINCT tx_id, + msg_group, + msg_sub_group, + attribute_value AS lock_id, + _inserted_timestamp, + concat_ws( + '-', + A.tx_id, + COALESCE( + A.msg_group, + -1 + ), + COALESCE( + A.msg_sub_group, + -1 + ) + ) AS _unique_key +FROM + {{ ref('silver__msg_attributes') }} A +WHERE + attribute_key IN ( + 'period_lock_id', + 'lock_id' + ) + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} +) +{% endif %} diff --git a/models/silver/silver__locked_liquidity_transactions.yml b/models/silver/silver__locked_liquidity_transactions.yml new file mode 100644 index 0000000..13b2188 --- /dev/null +++ b/models/silver/silver__locked_liquidity_transactions.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: silver__locked_liquidity_transactions + description: helper table for locked LPs + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - MSG_GROUP + - MSG_SUB_GROUP + - LOCK_ID + columns: + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: MSG_GROUP + description: "{{ doc('msg_group') }}" + tests: + - not_null + - name: MSG_SUB_GROUP + description: "{{ doc('msg_sub_group') }}" + tests: + - not_null + - name: LOCK_ID + description: "{{ doc('lock_id') }}" + tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__staked_balances.sql b/models/silver/silver__staked_balances.sql index f22e7d4..4818e20 100644 --- a/models/silver/silver__staked_balances.sql +++ b/models/silver/silver__staked_balances.sql @@ -1,85 +1,77 @@ {{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['block_timestamp'], + materialized = 'incremental', + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp'], ) }} WITH all_staked AS ( - SELECT - block_id, - block_timestamp, + + SELECT + block_id, + block_timestamp, delegator_address AS address, - amount, - currency, - CASE + amount, + currency, + CASE WHEN currency LIKE 'gamm/pool/%' THEN 18 - ELSE raw_metadata[1]:exponent - END AS decimal, + ELSE raw_metadata [1] :exponent + END AS DECIMAL, _inserted_timestamp - - FROM {{ ref('silver__staking') }} s - - LEFT OUTER JOIN {{ ref('silver__asset_metadata') }} a - ON s.currency = a.address - - WHERE action = 'delegate' + FROM + {{ ref('silver__staking') }} + s + LEFT OUTER JOIN {{ ref('silver__asset_metadata') }} A + ON s.currency = A.address + WHERE + action = 'delegate' {% if is_incremental() %} -AND - _inserted_timestamp >= ( +AND block_timestamp :: DATE >=( SELECT - MAX( - _inserted_timestamp - ) + DATEADD('day', -2, MAX(block_timestamp)) FROM - {{ this }} - ) -{% endif %} - - UNION ALL - - SELECT - block_id, - block_timestamp, - delegator_address AS address, - -amount, - currency, - CASE + {{ this }}) + {% endif %} + UNION ALL + SELECT + block_id, + block_timestamp, + delegator_address AS address,- amount, + currency, + CASE WHEN currency LIKE 'gamm/pool/%' THEN 18 - ELSE raw_metadata[1]:exponent - END AS decimal, + ELSE raw_metadata [1] :exponent + END AS DECIMAL, _inserted_timestamp - - FROM {{ ref('silver__staking') }} s - - LEFT OUTER JOIN {{ ref('silver__asset_metadata') }} a - ON s.currency = a.address - - WHERE action = 'undelegate' - - {% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) FROM - {{ this }} + {{ ref('silver__staking') }} + s + LEFT OUTER JOIN {{ ref('silver__asset_metadata') }} A + ON s.currency = A.address + WHERE + action = 'undelegate' + +{% if is_incremental() %} +AND block_timestamp :: DATE >=( + SELECT + DATEADD('day', -2, MAX(block_timestamp)) + FROM + {{ this }}) + {% endif %} ) -{% endif %} - -) - -SELECT - block_id, - block_timestamp, - 'staked' AS balance_type, - address, - currency, - decimal, - SUM(amount) OVER( PARTITION BY address, - currency - ORDER BY block_timestamp ASC ROWS UNBOUNDED PRECEDING - ) AS balance, +SELECT + block_id, + block_timestamp, + 'staked' AS balance_type, + address, + currency, + DECIMAL, + SUM(amount) over( + PARTITION BY address, + currency + ORDER BY + block_timestamp ASC rows unbounded preceding + ) AS balance, _inserted_timestamp -FROM all_staked \ No newline at end of file +FROM + all_staked diff --git a/models/silver/silver__superfluid_actions.sql b/models/silver/silver__superfluid_actions.sql new file mode 100644 index 0000000..324b341 --- /dev/null +++ b/models/silver/silver__superfluid_actions.sql @@ -0,0 +1,83 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['_inserted_timestamp::DATE'], +) }} + +SELECT + block_id, + block_timestamp, + blockchain, + chain_id, + A.tx_id, + A.tx_status, + A.msg_type, + A.msg_group, + msg :sender :: STRING AS delegator_address, + msg :coins [0] :amount :: INT AS amount, + msg :coins [0] :denom :: STRING AS currency, + msg :val_addr :: STRING AS validator_address, + COALESCE( + msg :lock_id :: INT, + b.lock_id :: INT, + msg: id :: INT + ) AS lock_id, + msg :pool_id :: INT AS pool_id, + concat_ws( + '-', + A.tx_id, + A.msg_group + ) AS _unique_key, + _inserted_timestamp +FROM + {{ ref('silver__tx_body_msgs') }} A + LEFT JOIN ( + SELECT + tx_id, + msg_group, + attribute_value AS lock_id + FROM + {{ ref('silver__msg_attributes') }} A + WHERE + msg_type IN ( + 'lock_tokens', + 'add_tokens_to_lock' + ) + AND attribute_key IN ( + 'period_lock_id', + 'lock_id' + ) + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} +) +{% endif %} +) b +ON A.tx_id = b.tx_id +AND A.msg_group = b.msg_group +WHERE + msg_type IN ( + '/osmosis.superfluid.MsgLockAndSuperfluidDelegate', + '/osmosis.superfluid.MsgSuperfluidUndelegate', + '/osmosis.superfluid.MsgSuperfluidDelegate', + '/osmosis.superfluid.MsgUnPoolWhitelistedPool' + ) + AND tx_status = 'SUCCEEDED' + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} +) +{% endif %} diff --git a/models/silver/silver__superfluid_actions.yml b/models/silver/silver__superfluid_actions.yml new file mode 100644 index 0000000..4a5b29d --- /dev/null +++ b/models/silver/silver__superfluid_actions.yml @@ -0,0 +1,80 @@ +version: 2 +models: + - name: silver__superfluid_actions + description: "Records of all superfluid related actions" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - MSG_GROUP + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - not_null + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: TX_STATUS + description: "{{ doc('tx_status') }}" + tests: + - not_null + - name: MSG_TYPE + description: "{{ doc('msg_type') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: MSG_GROUP + description: "{{ doc('msg_group') }}" + tests: + - not_null + - name: DELEGATOR_ADDRESS + description: "{{ doc('delegator_address') }}" + tests: + - not_null + - name: AMOUNT + description: "{{ doc('amount') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CURRENCY + description: "{{ doc('currency') }}" + - name: VALIDATOR_ADDRESS + description: "{{ doc('validator_address') }}" + - name: LOCK_ID + description: An ID corresponding to the locking step of the transaction. + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ diff --git a/models/silver/silver__superfluid_staked_balances.sql b/models/silver/silver__superfluid_staked_balances.sql new file mode 100644 index 0000000..2604af7 --- /dev/null +++ b/models/silver/silver__superfluid_staked_balances.sql @@ -0,0 +1,71 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'], +) }} + +WITH super AS ( + + SELECT + block_id, + delegator_address AS address, + lock_id + FROM + {{ ref('silver__superfluid_staking') }} + s + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >=( + SELECT + DATEADD('day', -2, MAX(block_timestamp)) + FROM + {{ this }}) + {% endif %} + ), + lp_balances AS ( + SELECT + A.block_id, + A.block_timestamp, + A.address, + A.balance, + A.currency, + A.decimal, + A.lock_id, + _inserted_timestamp + FROM + {{ ref('silver__locked_liquidity_balances') }} A + JOIN super b + ON A.block_id = b.block_id + AND A.address = b.address + AND A.lock_id = b.lock_id + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >=( + SELECT + DATEADD('day', -2, MAX(block_timestamp)) + FROM + {{ this }}) + {% endif %} + ) +SELECT + block_id, + block_timestamp, + 'superfluid staked' AS balance_type, + address, + lock_id, + currency, + DECIMAL, + balance, + concat_ws( + '-', + block_id, + address, + lock_id, + currency + ) AS _unique_key, + _inserted_timestamp +FROM + lp_balances diff --git a/models/silver/silver__superfluid_staked_balances.yml b/models/silver/silver__superfluid_staked_balances.yml new file mode 100644 index 0000000..8dc7cb6 --- /dev/null +++ b/models/silver/silver__superfluid_staked_balances.yml @@ -0,0 +1,46 @@ +version: 2 +models: + - name: silver__superfluid_staked_balances + description: A table of each superfluid staked balance change for wallets on the Osmosis chain. + tests: + - compare_model_subset: + name: silver__superfluid_staked_balances_business_logic_test + compare_model: ref('testing__superfluid_staked_balances') + compare_columns: + - block_id + - balance_type + - address + - lock_id + - currency + - balance + model_condition: "where block_timestamp::date = '2022-10-18' + and block_ID IN (6503532,6503516,6503500,6503499,6503494,6503478,6503466,6503466,6503459,6503453, + 6503420,6503420,6503395,6503391,6503389,6503385,6503379,6503375,6503338)" + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - name: ADDRESS + description: "{{ doc('address') }}" + tests: + - not_null + - name: BALANCE + description: The amount of the currency the wallet held at the given time + tests: + - not_null + - name: CURRENCY + description: "{{ doc('currency') }}" + tests: + - not_null + - name: DECIMAL + description: "{{ doc('decimal') }}" + - name: _INSERTED_TIMESTAMP + description: "{{ doc('ingested_at') }}" + tests: + - not_null + \ No newline at end of file diff --git a/models/silver/silver__superfluid_staking.sql b/models/silver/silver__superfluid_staking.sql index 79f6f4d..5f5641f 100644 --- a/models/silver/silver__superfluid_staking.sql +++ b/models/silver/silver__superfluid_staking.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE'], + cluster_by = ['block_timestamp::DATE'] ) }} WITH @@ -21,198 +21,136 @@ max_date AS ( base_txn AS ( SELECT - top 100 block_ID, - block_timestamp, - blockchain, - chain_id, - tx_id, - tx_status, - VALUE AS actio, - REPLACE(LEFT(path, CHARINDEX(path, ']')), '[') AS msg_group, - this :sender AS delegator_address, - this :coins [0] :amount AS amount, - this :coins [0] :denom AS currency, - this :val_addr AS validator_address, - this :duration, - _inserted_timestamp, - this :lock_id AS lock_id - FROM - {{ ref('silver__transactions') }} A, - LATERAL FLATTEN ( - input => tx_body :messages, - recursive => TRUE - ) b - WHERE - key = '@type' - AND VALUE :: STRING IN ( - '/osmosis.superfluid.MsgLockAndSuperfluidDelegate', - '/osmosis.superfluid.MsgSuperfluidUndelegate', - '/osmosis.superfluid.MsgSuperfluidDelegate', - '/osmosis.superfluid.MsgUnPoolWhitelistedPool' - ) - AND tx_status = 'SUCCEEDED' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) - FROM - max_date -) -{% endif %} -), -locks AS ( - SELECT - b.tx_ID ub_tx_id, - A.tx_ID, - attribute_value lock_id - FROM - {{ ref('silver__msg_attributes') }} A - JOIN ( - SELECT - DISTINCT lock_id, - tx_id - FROM - base_txn - ) b - ON b.lock_id = A.attribute_value - WHERE - msg_type IN ( - 'lock_tokens', - 'add_tokens_to_lock' - ) - AND attribute_key LIKE '%lock%' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) - FROM - max_date -) -{% endif %} -), -lock_body AS ( - SELECT - b.ub_tx_id, - b.lock_ID, + A.block_id, + A.block_timestamp, + A.blockchain, + A.chain_id, A.tx_id, - this :coins [0] :amount AS amount, - this :coins [0] :denom AS currecy, - this :val_addr AS validator_address, - this :duration, - this + A.tx_status, + A.msg_group, + A.msg_type, + A.msg_action_description, + A.locker_address, + A.lock_id, + A.amount, + A.currency, + A.decimal, + A.pool_id, + A.lock_duration, + A.unlock_time, + A.unpool_new_lock_ids, + A._unique_key, + A._inserted_timestamp FROM - {{ ref('silver__transactions') }} A, - locks b, - LATERAL FLATTEN ( - input => tx_body :messages, - recursive => TRUE - ) C + {{ ref('silver__locked_liquidity_actions') }} A WHERE - A.tx_id = b.tx_ID - AND key = '@type' - AND VALUE :: STRING IN ( - '/osmosis.superfluid.MsgLockAndSuperfluidDelegate', - '/osmosis.superfluid.MsgSuperfluidDelegate' - ) + is_superfluid = TRUE {% if is_incremental() %} AND _inserted_timestamp >= ( SELECT - MAX( - _inserted_timestamp - ) + _inserted_timestamp FROM max_date ) {% endif %} ), -tx_address AS ( +vals AS ( SELECT - A.tx_id, - OBJECT_AGG( - attribute_key :: STRING, - attribute_value :: variant - ) AS j, - SPLIT_PART( - j :acc_seq :: STRING, - '/', - 0 - ) AS tx_caller_address + lock_id, + validator_address FROM - {{ ref('silver__msg_attributes') }} A - JOIN ( - SELECT - DISTINCT tx_id - FROM - base_txn - ) b - ON A.tx_ID = b.tx_ID + {{ ref('silver__superfluid_actions') }} A WHERE - attribute_key = 'acc_seq' + validator_address IS NOT NULL {% if is_incremental() %} AND _inserted_timestamp >= ( SELECT - MAX( - _inserted_timestamp - ) + _inserted_timestamp FROM max_date ) {% endif %} -GROUP BY - A.tx_id, - msg_group + +qualify(ROW_NUMBER() over(PARTITION BY lock_id +ORDER BY + block_id DESC) = 1) +), +unpool_lock_val AS ( + SELECT + A.lock_id, + A.currency, + A.delegator_address, + A.validator_address + FROM + {{ ref('silver__superfluid_actions') }} A + JOIN {{ ref('silver__locked_liquidity_actions') }} + b + ON b.msg_action_description = 'unpool' + AND A.delegator_address = b.locker_address + AND A.currency = b.currency + AND A.block_id < b.block_id + WHERE + validator_address IS NOT NULL + +{% if is_incremental() %} +AND A._inserted_timestamp >= ( + SELECT + _inserted_timestamp + FROM + max_date +) +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY A.lock_id +ORDER BY + A.block_id DESC) = 1) ) SELECT A.block_id, A.block_timestamp, A.blockchain, - chain_ID, - A.tx_ID, + chain_id, + A.tx_id, + A.msg_group, A.tx_status, - tx.tx_caller_address, - REPLACE( - A.actio :: STRING, - '/osmosis.superfluid.Msg' - ) action, - A.delegator_address :: STRING AS delegator_address, - COALESCE( - A.amount :: INT, - C.amount :: INT - ) AS amount, - A.currency :: STRING AS currency, + {# msg_action_description, #} CASE - WHEN A.currency LIKE 'gamm/pool/%' THEN 18 - ELSE am.raw_metadata [1] :exponent - END AS DECIMAL, + msg_action_description + WHEN 'initial lock' THEN 'delegate' + WHEN 'add to position' THEN 'delegate' + WHEN 'unlock' THEN 'undelegate' + WHEN 'unpool' THEN 'undelegate' + END AS action, + A.locker_address AS delegator_address, + A.amount, + A.currency, + A.decimal, COALESCE( - A.validator_address :: STRING, - C.validator_address :: STRING + b.validator_address, + C.validator_address ) AS validator_address, COALESCE( A.lock_id, C.lock_id - ) AS lock_ID, - C.tx_ID AS original_superfluid_delegate_tx_ID, + ) AS lock_id, _inserted_timestamp, concat_ws( '-', A.tx_id, - action + A.msg_group, + COALESCE( + A.lock_id, + -1 + ), + msg_action_description ) AS _unique_key FROM base_txn A - LEFT JOIN lock_body C - ON A.tx_id = C.ub_tx_ID - LEFT JOIN tx_address tx - ON A.tx_id = tx.tx_id - LEFT JOIN {{ ref('silver__asset_metadata') }} - am - ON A.currency = am.address + LEFT JOIN vals b + ON A.lock_id = b.lock_id + LEFT JOIN unpool_lock_val C + ON A.locker_address = C.delegator_address + AND A.currency = C.currency + AND A.msg_action_description = 'unpool' diff --git a/models/silver/silver__superfluid_staking.yml b/models/silver/silver__superfluid_staking.yml index da79c07..7ccebb3 100644 --- a/models/silver/silver__superfluid_staking.yml +++ b/models/silver/silver__superfluid_staking.yml @@ -6,6 +6,8 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - TX_ID + - MSG_GROUP + - LOCK_ID - ACTION columns: - name: BLOCK_ID @@ -42,10 +44,6 @@ models: description: "{{ doc('tx_status') }}" tests: - not_null - - name: TX_CALLER_ADDRESS - description: "{{ doc('tx_caller_address') }}" - tests: - - not_null - name: ACTION description: "{{ doc('action') }}" tests: @@ -69,12 +67,8 @@ models: description: "{{ doc('currency') }}" - name: VALIDATOR_ADDRESS description: "{{ doc('validator_address') }}" - - name: DECIMAL - description: "{{ doc('decimal') }}" - name: LOCK_ID - description: An ID corresponding to the locking step of the transaction. - - name: ORIGINAL_SUPERFLUID_DELEGATE_TX_ID - description: The transaction id when the user initialized the superfluid stake. + description: "{{ doc('lock_id') }}" - name: _INSERTED_TIMESTAMP description: "{{ doc('inserted_timestamp') }}" tests: diff --git a/models/silver/silver__tx_body_msgs.sql b/models/silver/silver__tx_body_msgs.sql new file mode 100644 index 0000000..3b03f31 --- /dev/null +++ b/models/silver/silver__tx_body_msgs.sql @@ -0,0 +1,56 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['_inserted_timestamp::DATE'] +) }} + +WITH b AS ( + + SELECT + block_id, + block_timestamp, + blockchain, + chain_id, + tx_id, + tx_status, + INDEX AS msg_group, + VALUE :"@type" :: STRING AS msg_type, + VALUE AS msg, + _inserted_timestamp + FROM + {{ ref('silver__transactions') }} A, + LATERAL FLATTEN( + input => A.tx_body :messages + ) + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + block_id, + block_timestamp, + blockchain, + chain_id, + tx_id, + tx_status, + msg_group, + msg_type, + msg, + concat_ws( + '-', + tx_id, + msg_group + ) AS _unique_key, + _inserted_timestamp +FROM + b diff --git a/models/silver/silver__tx_body_msgs.yml b/models/silver/silver__tx_body_msgs.yml new file mode 100644 index 0000000..922a7bd --- /dev/null +++ b/models/silver/silver__tx_body_msgs.yml @@ -0,0 +1,57 @@ +version: 2 +models: + - name: silver__tx_body_msgs + description: Records of all messages associated to transactions that have occurred on Osmosis, dating back to the genesis block. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - MSG_GROUP + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - not_null + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: MSG_GROUP + description: "{{ doc('msg_group') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSG_TYPE + description: "{{ doc('msg_type') }}" + tests: + - not_null + - name: MSG + description: "The underlying json from the message or event within the transactions" + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ