From a73f9134c1ff659733a33547286769b934310ad2 Mon Sep 17 00:00:00 2001 From: tarikceric <46071768+tarikceric@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:26:22 -0800 Subject: [PATCH] An 4450/add swap protocols (#494) * add bonk, meteora, phoenix swaps * update desc * add meteora * add test case * remove solana reference * add dooar swaps * remove phoenix; remove solana prefix * add phoenix * phoenix fix * update backfill date * logic for multi swaps w incorrect join * add tests, update logic for multi swap txs * remove db prefix --- data/testing__swaps_intermediate_bonkswap.csv | 14 ++ data/testing__swaps_intermediate_dooar.csv | 7 + data/testing__swaps_intermediate_meteora.csv | 12 ++ data/testing__swaps_intermediate_phoenix.csv | 10 + models/gold/defi/defi__fact_swaps.sql | 93 +++++++++ models/gold/defi/defi__fact_swaps.yml | 2 +- .../silver__swaps_intermediate_bonkswap.sql | 185 ++++++++++++++++++ .../silver__swaps_intermediate_bonkswap.yml | 83 ++++++++ .../silver__swaps_intermediate_dooar.sql | 139 +++++++++++++ .../silver__swaps_intermediate_dooar.yml | 78 ++++++++ .../silver__swaps_intermediate_meteora.sql | 174 ++++++++++++++++ .../silver__swaps_intermediate_meteora.yml | 76 +++++++ .../silver__swaps_intermediate_phoenix.sql | 181 +++++++++++++++++ .../silver__swaps_intermediate_phoenix.yml | 85 ++++++++ 14 files changed, 1138 insertions(+), 1 deletion(-) create mode 100644 data/testing__swaps_intermediate_bonkswap.csv create mode 100644 data/testing__swaps_intermediate_dooar.csv create mode 100644 data/testing__swaps_intermediate_meteora.csv create mode 100644 data/testing__swaps_intermediate_phoenix.csv create mode 100644 models/silver/swaps/silver__swaps_intermediate_bonkswap.sql create mode 100644 models/silver/swaps/silver__swaps_intermediate_bonkswap.yml create mode 100644 models/silver/swaps/silver__swaps_intermediate_dooar.sql create mode 100644 models/silver/swaps/silver__swaps_intermediate_dooar.yml create mode 100644 models/silver/swaps/silver__swaps_intermediate_meteora.sql create mode 100644 models/silver/swaps/silver__swaps_intermediate_meteora.yml create mode 100644 models/silver/swaps/silver__swaps_intermediate_phoenix.sql create mode 100644 models/silver/swaps/silver__swaps_intermediate_phoenix.yml diff --git a/data/testing__swaps_intermediate_bonkswap.csv b/data/testing__swaps_intermediate_bonkswap.csv new file mode 100644 index 00000000..0c3a76f9 --- /dev/null +++ b/data/testing__swaps_intermediate_bonkswap.csv @@ -0,0 +1,14 @@ +tx_id,swapper,from_amt,from_mint,to_amt,to_mint,swap_index +6nfq85sETizzDw4uuZ7X1S36uNcWUKdvvmtSGJFf7rvM5r8d9XLqFyyE7LLS6uyCkg2ksmgsHcS3umHYMymyeXB,6DBRRXagTDX4yzGSpSZZLUv4LWzSehHWdFxPt4jk8KfT,19000000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,2.062776725,So11111111111111111111111111111111111111112,1 +mQxJQ3oPFRsmncvRWwU4ecR4TpsLBF4jdk6HwkHP69kwsPCvFxQxGUiMinGs4iEC5rz7p5ShTnqJ7SeFD6h9v8Z,YUPaMvS8JBJ1qphcXpyxUvfsysGKwFTznsJ6bjcurSV,72176000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,7.903077732,So11111111111111111111111111111111111111112,1 +4n9zk2i8qsQVjzgYGPuqJbrUjX92JToVEcTiAnhfExp5VDGs7Lz56ZGySgaxTDMVNQMhXYtMj9E7YcbzJjqBcdyv,4kH5sRStQiJKw5VaKuxaEYMSDnh8pSKwf5W33vz6JbHP,20095000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,2.193551995,So11111111111111111111111111111111111111112,1 +3ATCz7Vf2sMNKsqnMDs7hsQdTPzJA2ediwP3RTdcLCGPV1tgGNsaHn2MrEtHvo57zuQhsA1XFz1NBjU8iyPW8C2Z,4kH5sRStQiJKw5VaKuxaEYMSDnh8pSKwf5W33vz6JbHP,3260999.92356,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,34.306508,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1 +5KzijZM4SmdFBjRx5qQSusr8f17M9mDwyDm1vdhdoa6VF2TmFGNJfLCHCzJRp3aHkheJaDojNF28qx2qWTg1FAKR,CapuXNQoDviLvU1PxFiizLgPNQCxrsag1uMeyk6zLVps,8145100.20245,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,1.143266579,So11111111111111111111111111111111111111112,1 +5KzijZM4SmdFBjRx5qQSusr8f17M9mDwyDm1vdhdoa6VF2TmFGNJfLCHCzJRp3aHkheJaDojNF28qx2qWTg1FAKR,CapuXNQoDviLvU1PxFiizLgPNQCxrsag1uMeyk6zLVps,1279944.31754,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,17.825619,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,2 +2HemzKoadNauTGqGmjsxnTN1mYm7vfRTjyeM38DYGM7qdGvmMfkFL5VQkHnTJ8giuUD1EV9w9NPQRvRSjSAFZ12A,GGztQqQ6pCPaJQnNpXBgELr5cs3WwDakRbh1iEMzjgSJ,0.052981954,mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So,1673066.26297,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,1 +2HemzKoadNauTGqGmjsxnTN1mYm7vfRTjyeM38DYGM7qdGvmMfkFL5VQkHnTJ8giuUD1EV9w9NPQRvRSjSAFZ12A,GGztQqQ6pCPaJQnNpXBgELr5cs3WwDakRbh1iEMzjgSJ,1673066.26297,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,0.060935689,So11111111111111111111111111111111111111112,2 +5ghJVhQyZuQoXLTjznjhmJ3EiGSDtdy1PT6xk7iw5GhukX7F4CmsYtPmh6C5Si8cY31Wuwo1dkX7iCzWgXuvNVgc,2MFoS3MPtvyQ4Wh4M9pdfPjz6UhVoNbFbGJAskCPCj3h,5,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,2700400.56329,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,1 +5ghJVhQyZuQoXLTjznjhmJ3EiGSDtdy1PT6xk7iw5GhukX7F4CmsYtPmh6C5Si8cY31Wuwo1dkX7iCzWgXuvNVgc,2MFoS3MPtvyQ4Wh4M9pdfPjz6UhVoNbFbGJAskCPCj3h,1890280.3943,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,0.075264275,So11111111111111111111111111111111111111112,2 +5ghJVhQyZuQoXLTjznjhmJ3EiGSDtdy1PT6xk7iw5GhukX7F4CmsYtPmh6C5Si8cY31Wuwo1dkX7iCzWgXuvNVgc,2MFoS3MPtvyQ4Wh4M9pdfPjz6UhVoNbFbGJAskCPCj3h,810120.16899,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,0.029617528,bSo13r4TkiE4KumL71LsHTPpL2euBYLFx6h9HP3piy1,3 +6HuXhDg9cc5nkbWmTQYjj7hpAxrJwMW3ZQT21jNgTouabAm3CxBtAdAf9Q21Ffov9kQnADjEmxYLaSZrDFjcbDx,7376pEqjeLxMCJFcUukakevhCxBz18YxWUAqHfRo35v3,5036865.56417,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,0.166144844,So11111111111111111111111111111111111111112,1 +fgmeRM6JRo9jZk53h4ECQpe5pt2joQK9PpBetFt9pSHpGNBrNjqDhW7Hihivb4n4d6VX2Gkpy3Dzk6wkCTvRjt1,629M6EYnSc3cZ8pTuSFg7bbVy8R1bCW2yoRUcSwR6j73,334187000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,13.191226035,So11111111111111111111111111111111111111112,1 \ No newline at end of file diff --git a/data/testing__swaps_intermediate_dooar.csv b/data/testing__swaps_intermediate_dooar.csv new file mode 100644 index 00000000..098d0bb9 --- /dev/null +++ b/data/testing__swaps_intermediate_dooar.csv @@ -0,0 +1,7 @@ +tx_id,swapper,from_amt,from_mint,to_amt,to_mint,swap_index +31at6mYffEH2eFEFQxFt5vqsJpNpZx8Uofn1R53ABLaLyqAsgrUDQDqxiWSJ5k7JEZk8NF9V4kzT87PTctBHy28G,7GAJstE7XxkopkD2kxZ4Fr2Dq8tLnfRWdYpeEGc159Wr,630.79998877,AFbX8oGjGpmVFywbVouvhQSRmiW2aR1mohfahi4Y2AdB,7.297702,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1 +31at6mYffEH2eFEFQxFt5vqsJpNpZx8Uofn1R53ABLaLyqAsgrUDQDqxiWSJ5k7JEZk8NF9V4kzT87PTctBHy28G,7GAJstE7XxkopkD2kxZ4Fr2Dq8tLnfRWdYpeEGc159Wr,7.261195,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,0.068950166,So11111111111111111111111111111111111111112,2 +4n6VDefX6cPhJ3pdTpUSqo7FxSDeqeymz7abto5V7KJppBZfhH7KDvZ4m8FCEdkeckkajbjW1KbMq6zqpzYptQk5,BPiunxZKqnVjfyGwxEGDJ8fo2Anc9KZW2w81gpbPRJV3,336.10383,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1253.5036093,7i5KKsX2weiTkry7jA4ZwSuXGhs5eJBEjY8vVxR4pfRx,1 +3dKk8sbjZ2yocabr8qUaLHsa7bkd4qxnW8gjsqwhTprVfMrPGeCFqmwu2vYFytaE1NViqtiY4ALZjR2we3xnsmP,BgGwn7CJdaiYaxg3vzMrWU2BKd8bzfk1bH1kNXtm75k5,269.253167,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,3.662990159,So11111111111111111111111111111111111111112,1 +4KvNEeQpzCx9QmsoGt8QNjtBDomYBnGMSNn1GeyWn6qQn2tsavPGEH26jbfoHHHUcesVbogJfXr9ghD6hv9rz6hD,bwwogaBkcPXDAsQ8jiK5L32NqLtwLEfAKjnKf1LKTL2,0.501813453,So11111111111111111111111111111111111111112,36.383661,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1 +4KvNEeQpzCx9QmsoGt8QNjtBDomYBnGMSNn1GeyWn6qQn2tsavPGEH26jbfoHHHUcesVbogJfXr9ghD6hv9rz6hD,bwwogaBkcPXDAsQ8jiK5L32NqLtwLEfAKjnKf1LKTL2,36.201742,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,117.29260577,7i5KKsX2weiTkry7jA4ZwSuXGhs5eJBEjY8vVxR4pfRx,2 \ No newline at end of file diff --git a/data/testing__swaps_intermediate_meteora.csv b/data/testing__swaps_intermediate_meteora.csv new file mode 100644 index 00000000..5a224b63 --- /dev/null +++ b/data/testing__swaps_intermediate_meteora.csv @@ -0,0 +1,12 @@ +tx_id,swapper,from_amt,from_mint,to_amt,to_mint,swap_index +3DjUBQE4GwTMSFLGMi69TGMiBoNE8FfnryR8BCEmovGVmMz9Jfh9dG3ikLAPuQPWpv12ypS4nYx1Esc4TYUw3s44,BQ72nSv9f3PRyRKCBnHLVrerrv37CYTHm5h3s9VSGQDV,12095.46106,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,0.016580859,DK6PWMyuZ4NMjsm9AWNCTMKrajQYrtfMjMJ3QauX2UH5,1 +3DjUBQE4GwTMSFLGMi69TGMiBoNE8FfnryR8BCEmovGVmMz9Jfh9dG3ikLAPuQPWpv12ypS4nYx1Esc4TYUw3s44,BQ72nSv9f3PRyRKCBnHLVrerrv37CYTHm5h3s9VSGQDV,0.016580859,DK6PWMyuZ4NMjsm9AWNCTMKrajQYrtfMjMJ3QauX2UH5,1744.06551,WENWENvqqNya429ubCdR81ZmD69brwQaaBYY6p3LCpk,2 +3DjUBQE4GwTMSFLGMi69TGMiBoNE8FfnryR8BCEmovGVmMz9Jfh9dG3ikLAPuQPWpv12ypS4nYx1Esc4TYUw3s44,BQ72nSv9f3PRyRKCBnHLVrerrv37CYTHm5h3s9VSGQDV,1744.06551,WENWENvqqNya429ubCdR81ZmD69brwQaaBYY6p3LCpk,0.001507191,So11111111111111111111111111111111111111112,3 +3RCs1zgEKNemq1syYM3ZZKHkckZgBbkgtKVq41W7j2yh5Ce4pVgHLU6QSiZX3nrijZ4S2yhYDb3K61seLR4c9FYu,E8HBNczbRicd9dhtqFAXScVh2WD8N6uLJFAf82KEtm4L,1,So11111111111111111111111111111111111111112,0.918152768,J1toso1uCk3RLmjorhTtrVwY9HJ7X8V9yYac6Y7kGCPn,1 +iBuwG5mU4LKMrN6iNVzGZSC7vXqvAfRQ98YJeKXPoDbM2oi7dvV8ZMnwgv55Db6JT4iokQ6JbVpN7jHnKkFtJqC,5nM1CTQwKXFZo5yJYC8J1pgj32JW6Fx8DpQAtPZ8aiLw,6,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,0.052992004,So11111111111111111111111111111111111111112,1 +NfvsjHzJx5s5JWDK7pJaWUv7hXJGdWUYwqaoRVLxQ2d5q3Jyrs2uft9FUyeNFVyxVSVZFPdsTv45dpgdog9RyzQ,9nnLbotNTcUhvbrsA6Mdkx45Sm82G35zo28AqUvjExn8,0.01658499,rndrizKT3MK1iimdxRdWabcF7Zg7AR5T4nud4EkHBof,0.099306,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1 +2eV5PUm4WL7X9T1X4nSbBmAGDLTx9pUiygK34ngMSyR9QwQUS8ZpnzGKTCbuJHeVGYNHjHBGtPh4fLYxkBLGVJeB,BQ72nSv9f3PRyRKCBnHLVrerrv37CYTHm5h3s9VSGQDV,40,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,40.176596896,9TVjnzpF3X8DHsfVqYWoCGphJxtGYh1PDCFN5QmsHW5t,1 +2eV5PUm4WL7X9T1X4nSbBmAGDLTx9pUiygK34ngMSyR9QwQUS8ZpnzGKTCbuJHeVGYNHjHBGtPh4fLYxkBLGVJeB,BQ72nSv9f3PRyRKCBnHLVrerrv37CYTHm5h3s9VSGQDV,40.176596896,9TVjnzpF3X8DHsfVqYWoCGphJxtGYh1PDCFN5QmsHW5t,0.838353755,So11111111111111111111111111111111111111112,2 +2eV5PUm4WL7X9T1X4nSbBmAGDLTx9pUiygK34ngMSyR9QwQUS8ZpnzGKTCbuJHeVGYNHjHBGtPh4fLYxkBLGVJeB,BQ72nSv9f3PRyRKCBnHLVrerrv37CYTHm5h3s9VSGQDV,10,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,0.00026839,3NZ9JMVBmGAqocybic2c7LQCJScmgsAZ6vQqTDzcqmJh,3 +35QkSCVWRNFaqFM8DZrDWKWsBdDTrHVMmskd3H9D3pcQstdYsWqZiXVhMpkaCFrBQTi8XZaWgNZaWnQZzwPpZ1dv,6U91aKa8pmMxkJwBCfPTmUEfZi6dHe7DcFq2ALvB2tbB,5.226666,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,2993956.3575,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,1 +35QkSCVWRNFaqFM8DZrDWKWsBdDTrHVMmskd3H9D3pcQstdYsWqZiXVhMpkaCFrBQTi8XZaWgNZaWnQZzwPpZ1dv,6U91aKa8pmMxkJwBCfPTmUEfZi6dHe7DcFq2ALvB2tbB,0.027582317,So11111111111111111111111111111111111111112,747886.05184,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,2 \ No newline at end of file diff --git a/data/testing__swaps_intermediate_phoenix.csv b/data/testing__swaps_intermediate_phoenix.csv new file mode 100644 index 00000000..7e45ff0a --- /dev/null +++ b/data/testing__swaps_intermediate_phoenix.csv @@ -0,0 +1,10 @@ +tx_id,swapper,from_amt,from_mint,to_amt,to_mint,swap_index +3fp9QzP6EMn9cUDgCRmHACSxmDhh3zrMebYKSm8Auq3xyBck7X6nRSrrVDThDhM1wRBaVidwh4BCDLn86Bve3thL,BPiunxZKqnVjfyGwxEGDJ8fo2Anc9KZW2w81gpbPRJV3,0.715816712,So11111111111111111111111111111111111111112,5954000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,1 +671GWKC1AGyvrAThDzM9tq9UqVW8W2WdULAikkLaaTSZVYEmzLEhXutdDjZTenLp5oAtc4G7f4bztZ978rN1Zz1d,bobCPc5nqVoX7r8gKzCMPLrKjFidjnSCrAdcYGCH2Ye,399.966167,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,3.421,So11111111111111111111111111111111111111112,1 +3FgxCNCndXDSx95mrsBwcfsHSM9fDzWSq2bJqN74ym5MTgF2VrLJRn4w5UfdKpeKNA3mzCmm7KJfy7wuDAo4y7Ko,FreyaXYaCwVy86BdNECd7BXnqEvFUbt3p6d2B5eS5zDv,1.494221215,So11111111111111111111111111111111111111112,13137000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,2 +3FgxCNCndXDSx95mrsBwcfsHSM9fDzWSq2bJqN74ym5MTgF2VrLJRn4w5UfdKpeKNA3mzCmm7KJfy7wuDAo4y7Ko,FreyaXYaCwVy86BdNECd7BXnqEvFUbt3p6d2B5eS5zDv,169.118343,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1.494,So11111111111111111111111111111111111111112,1 +21qdi1d8vYyMp3AcJmPdKBi1VsEoaBn4qhB1WJhzRR9N3qvtXRkENh5caaBGDaeUVD68CUw72bD2neCQA7dmgUuS,ao31QJtoqLEynyqrZbwkUTGC8mh8fU55RtiUk2pzK8R,142.767,So11111111111111111111111111111111111111112,2775.994087,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1 +21qdi1d8vYyMp3AcJmPdKBi1VsEoaBn4qhB1WJhzRR9N3qvtXRkENh5caaBGDaeUVD68CUw72bD2neCQA7dmgUuS,ao31QJtoqLEynyqrZbwkUTGC8mh8fU55RtiUk2pzK8R,135.1432,J1toso1uCk3RLmjorhTtrVwY9HJ7X8V9yYac6Y7kGCPn,142.766958204,So11111111111111111111111111111111111111112,2 +21qdi1d8vYyMp3AcJmPdKBi1VsEoaBn4qhB1WJhzRR9N3qvtXRkENh5caaBGDaeUVD68CUw72bD2neCQA7dmgUuS,ao31QJtoqLEynyqrZbwkUTGC8mh8fU55RtiUk2pzK8R,142.727107646,So11111111111111111111111111111111111111112,135.1432,J1toso1uCk3RLmjorhTtrVwY9HJ7X8V9yYac6Y7kGCPn,3 +3EweEGRQkdQ4e5sRRcU5V9Jz2E9sMHzDrbmSD13C2wLQJkANJEZkk9EHJqpUSKUSWCAnY5nFnLgXUqJF5ham6ARk,2SLowUE6xmrddRd4VeJ2i39ybekDg22BRcwsBR3vpwAG,10.188376343,So11111111111111111111111111111111111111112,271971000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,1 +3EweEGRQkdQ4e5sRRcU5V9Jz2E9sMHzDrbmSD13C2wLQJkANJEZkk9EHJqpUSKUSWCAnY5nFnLgXUqJF5ham6ARk,2SLowUE6xmrddRd4VeJ2i39ybekDg22BRcwsBR3vpwAG,1.271997083,So11111111111111111111111111111111111111112,33955000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,2 \ No newline at end of file diff --git a/models/gold/defi/defi__fact_swaps.sql b/models/gold/defi/defi__fact_swaps.sql index 071a3842..9278d263 100644 --- a/models/gold/defi/defi__fact_swaps.sql +++ b/models/gold/defi/defi__fact_swaps.sql @@ -106,3 +106,96 @@ FROM LEFT OUTER JOIN {{ ref('core__dim_labels') }} l ON s.program_id = l.address +UNION ALL +SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + swapper, + from_amt AS swap_from_amount, + from_mint AS swap_from_mint, + to_amt AS swap_to_amount, + to_mint AS swap_to_mint, + program_id, + l.address_name AS swap_program, + concat_ws('-',tx_id,swap_index) as _log_id, + swaps_intermediate_bonkswap_id as fact_swaps_id, + s.inserted_timestamp, + s.modified_timestamp +FROM + {{ ref('silver__swaps_intermediate_bonkswap') }} + s + LEFT OUTER JOIN {{ ref('core__dim_labels') }} + l + ON s.program_id = l.address +UNION ALL +SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + swapper, + from_amt AS swap_from_amount, + from_mint AS swap_from_mint, + to_amt AS swap_to_amount, + to_mint AS swap_to_mint, + program_id, + l.address_name AS swap_program, + concat_ws('-',tx_id,swap_index) as _log_id, + swaps_intermediate_meteora_id as fact_swaps_id, + s.inserted_timestamp, + s.modified_timestamp +FROM + {{ ref('silver__swaps_intermediate_meteora') }} + s + LEFT OUTER JOIN {{ ref('core__dim_labels') }} + l + ON s.program_id = l.address +UNION ALL +SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + swapper, + from_amt AS swap_from_amount, + from_mint AS swap_from_mint, + to_amt AS swap_to_amount, + to_mint AS swap_to_mint, + program_id, + l.address_name AS swap_program, + concat_ws('-',tx_id,swap_index) as _log_id, + swaps_intermediate_dooar_id as fact_swaps_id, + s.inserted_timestamp, + s.modified_timestamp +FROM + {{ ref('silver__swaps_intermediate_dooar') }} + s + LEFT OUTER JOIN {{ ref('core__dim_labels') }} + l + ON s.program_id = l.address +UNION ALL +SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + swapper, + from_amt AS swap_from_amount, + from_mint AS swap_from_mint, + to_amt AS swap_to_amount, + to_mint AS swap_to_mint, + program_id, + l.address_name AS swap_program, + concat_ws('-',tx_id,swap_index) as _log_id, + swaps_intermediate_phoenix_id as fact_swaps_id, + s.inserted_timestamp, + s.modified_timestamp +FROM + {{ ref('silver__swaps_intermediate_phoenix') }} + s + LEFT OUTER JOIN {{ ref('core__dim_labels') }} + l + ON s.program_id = l.address + diff --git a/models/gold/defi/defi__fact_swaps.yml b/models/gold/defi/defi__fact_swaps.yml index e144e187..3278310d 100644 --- a/models/gold/defi/defi__fact_swaps.yml +++ b/models/gold/defi/defi__fact_swaps.yml @@ -1,7 +1,7 @@ version: 2 models: - name: defi__fact_swaps - description: This table contains swaps performed on Jupiter, Orca, Raydium, and Saber dexes. Intermediate swaps are aggregated over the DEX programs, so the values showcase the final mint/amount swap values. Ie. a swap on Jupiter that swaps SOL->USDC->mSOL->ETH would show the initial amount in and the final amount out of SOL->ETH. + description: This table contains swaps performed on Jupiter, Orca, Raydium, Saber, Bonkswap, Dooar, Phoenix and Meteora swap programs. Intermediate swaps are aggregated over the DEX programs, so the values showcase the final mint/amount swap values. Ie. a swap on Jupiter that swaps SOL->USDC->mSOL->ETH would show the initial amount in and the final amount out of SOL->ETH. columns: - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp') }}" diff --git a/models/silver/swaps/silver__swaps_intermediate_bonkswap.sql b/models/silver/swaps/silver__swaps_intermediate_bonkswap.sql new file mode 100644 index 00000000..6574c549 --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_bonkswap.sql @@ -0,0 +1,185 @@ +{{ config( + materialized = 'incremental', + unique_key = ['swaps_intermediate_bonkswap_id'], + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['scheduled_non_core'], +) }} + +WITH base AS ( + + SELECT + * + FROM + {{ ref('silver__decoded_instructions_combined') }} + WHERE + program_id = 'BSwp6bEBihVLdqJRKGgzjcGLHkcTuzmSo1TQkHepzH8p' + AND event_type = 'swap' + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 hour' + FROM + {{ this }} +) +{% else %} + AND block_timestamp :: DATE >= '2023-04-12' +{% endif %} +), +decoded AS ( + SELECT + block_timestamp, + block_id, + tx_id, + INDEX, + inner_index, + decoded_instruction :args :xToY :: BOOLEAN AS xToY, + COALESCE(LEAD(inner_index) over (PARTITION BY tx_id, INDEX + ORDER BY + inner_index) -1, 999999) AS inner_index_end, + program_id, + silver.udf_get_account_pubkey_by_name( + 'swapper', + decoded_instruction :accounts + ) AS swapper, + CASE + WHEN xToY THEN silver.udf_get_account_pubkey_by_name( + 'swapperXAccount', + decoded_instruction :accounts + ) + ELSE silver.udf_get_account_pubkey_by_name( + 'swapperYAccount', + decoded_instruction :accounts + ) + END AS source_token_account, + silver.udf_get_account_pubkey_by_name( + 'tokenX', + decoded_instruction :accounts + ) AS source_mint, + silver.udf_get_account_pubkey_by_name( + 'tokenY', + decoded_instruction :accounts + ) AS destination_mint, + CASE + WHEN xToY THEN silver.udf_get_account_pubkey_by_name( + 'swapperYAccount', + decoded_instruction :accounts + ) + ELSE silver.udf_get_account_pubkey_by_name( + 'swapperXAccount', + decoded_instruction :accounts + ) + END AS destination_token_account, + CASE + WHEN xToY THEN silver.udf_get_account_pubkey_by_name( + 'poolYAccount', + decoded_instruction :accounts + ) + ELSE silver.udf_get_account_pubkey_by_name( + 'poolXAccount', + decoded_instruction :accounts + ) + END AS program_destination_token_account, + CASE + WHEN xToY THEN silver.udf_get_account_pubkey_by_name( + 'poolXAccount', + decoded_instruction :accounts + ) + ELSE silver.udf_get_account_pubkey_by_name( + 'poolYAccount', + decoded_instruction :accounts + ) + END AS program_source_token_account, + _inserted_timestamp + FROM + base +), +transfers AS ( + SELECT + A.*, + COALESCE(SPLIT_PART(INDEX :: text, '.', 1) :: INT, INDEX :: INT) AS index_1, + NULLIF(SPLIT_PART(INDEX :: text, '.', 2), '') :: INT AS inner_index_1 + FROM + {{ ref('silver__transfers') }} A + INNER JOIN ( + SELECT + DISTINCT tx_id + FROM + decoded + ) d + ON d.tx_id = A.tx_id + WHERE + A.succeeded + +{% if is_incremental() %} +AND A._inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 day' + FROM + {{ this }} +) +{% else %} + AND A.block_timestamp :: DATE >= '2023-04-12' +{% endif %} +), +pre_final AS ( + SELECT + A.block_id, + A.block_timestamp, + A.program_id, + A.tx_id, + A.index, + A.inner_index, + A.inner_index_end, + C.succeeded, + A.swapper, + b.amount AS from_amt, + b.mint AS from_mint, + C.amount AS to_amt, + C.mint AS to_mint, + A._inserted_timestamp + FROM + decoded A + INNER JOIN transfers b + ON A.tx_id = b.tx_id + AND A.source_token_account = b.source_token_account + AND A.program_source_token_account = b.dest_token_account + AND A.index = b.index_1 + AND ((b.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) or a.inner_index is null) + INNER JOIN transfers C + ON A.tx_id = C.tx_id + AND A.destination_token_account = C.dest_token_account + AND A.program_destination_token_account = C.source_token_account + AND A.index = C.index_1 + AND ((C.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) or a.inner_index is null) + AND C.amount <> 0 + qualify(ROW_NUMBER() over (PARTITION BY A.tx_id, A.index, A.inner_INDEX + ORDER BY + inner_index)) = 1 +) +SELECT + block_id, + block_timestamp, + program_id, + tx_id, + ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + INDEX, + inner_index + ) AS swap_index, + succeeded, + swapper, + from_amt, + from_mint, + to_amt, + to_mint, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['tx_id','swap_index','program_id']) }} AS swaps_intermediate_bonkswap_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id +FROM + pre_final diff --git a/models/silver/swaps/silver__swaps_intermediate_bonkswap.yml b/models/silver/swaps/silver__swaps_intermediate_bonkswap.yml new file mode 100644 index 00000000..f298808f --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_bonkswap.yml @@ -0,0 +1,83 @@ +version: 2 +models: + - name: silver__swaps_intermediate_bonkswap + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - SWAP_INDEX + - PROGRAM_ID + where: block_timestamp::date > current_date - 30 + - compare_model_subset: + name: silver__swaps_intermediate_bonkswap_business_logic_test + compare_model: ref('testing__swaps_intermediate_bonkswap') + compare_columns: + - tx_id + - swapper + - round(from_amt,8) + - from_mint + - round(to_amt,8) + - to_mint + - swap_index + model_condition: "where tx_id in ('6nfq85sETizzDw4uuZ7X1S36uNcWUKdvvmtSGJFf7rvM5r8d9XLqFyyE7LLS6uyCkg2ksmgsHcS3umHYMymyeXB', + 'mQxJQ3oPFRsmncvRWwU4ecR4TpsLBF4jdk6HwkHP69kwsPCvFxQxGUiMinGs4iEC5rz7p5ShTnqJ7SeFD6h9v8Z', + '4n9zk2i8qsQVjzgYGPuqJbrUjX92JToVEcTiAnhfExp5VDGs7Lz56ZGySgaxTDMVNQMhXYtMj9E7YcbzJjqBcdyv', + '3ATCz7Vf2sMNKsqnMDs7hsQdTPzJA2ediwP3RTdcLCGPV1tgGNsaHn2MrEtHvo57zuQhsA1XFz1NBjU8iyPW8C2Z', + '5KzijZM4SmdFBjRx5qQSusr8f17M9mDwyDm1vdhdoa6VF2TmFGNJfLCHCzJRp3aHkheJaDojNF28qx2qWTg1FAKR', + '2HemzKoadNauTGqGmjsxnTN1mYm7vfRTjyeM38DYGM7qdGvmMfkFL5VQkHnTJ8giuUD1EV9w9NPQRvRSjSAFZ12A', + '5ghJVhQyZuQoXLTjznjhmJ3EiGSDtdy1PT6xk7iw5GhukX7F4CmsYtPmh6C5Si8cY31Wuwo1dkX7iCzWgXuvNVgc', + '6HuXhDg9cc5nkbWmTQYjj7hpAxrJwMW3ZQT21jNgTouabAm3CxBtAdAf9Q21Ffov9kQnADjEmxYLaSZrDFjcbDx', + 'fgmeRM6JRo9jZk53h4ECQpe5pt2joQK9PpBetFt9pSHpGNBrNjqDhW7Hihivb4n4d6VX2Gkpy3Dzk6wkCTvRjt1')" + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - name: PROGRAM_ID + description: "{{ doc('program_id') }}" + tests: + - not_null + - name: SWAPPER + description: "{{ doc('swaps_swapper') }}" + tests: + - not_null: + where: succeeded = TRUE + - name: FROM_AMT + description: "{{ doc('swaps_from_amt') }}" + tests: + - not_null + - name: FROM_MINT + description: "{{ doc('swaps_from_mint') }}" + tests: + - not_null + - name: TO_AMT + description: "{{ doc('swaps_to_amt') }}" + tests: + - not_null + - name: TO_MINT + description: "{{ doc('swaps_to_mint') }}" + tests: + - not_null + - name: SWAP_INDEX + description: "{{ doc('swaps_swap_index') }}" + tests: + - not_null + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null \ No newline at end of file diff --git a/models/silver/swaps/silver__swaps_intermediate_dooar.sql b/models/silver/swaps/silver__swaps_intermediate_dooar.sql new file mode 100644 index 00000000..de676398 --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_dooar.sql @@ -0,0 +1,139 @@ +{{ config( + materialized = 'incremental', + unique_key = ['swaps_intermediate_dooar_id'], + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['scheduled_non_core'], +) }} + +WITH base AS ( + + SELECT + * + FROM + {{ ref('silver__decoded_instructions_combined') }} + WHERE + program_id = 'Dooar9JkhdZ7J3LHN3A7YCuoGRUggXhQaG4kijfLGU2j' + AND + event_type = 'swap' +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 hour' + FROM + {{ this }} +) +{% else %} + AND block_timestamp :: DATE >= '2022-06-09' +{% endif %} +), +decoded AS ( + SELECT + block_timestamp, + block_id, + tx_id, + index, + inner_index, + COALESCE(LEAD(inner_index) over (PARTITION BY tx_id, INDEX + ORDER BY + inner_index) -1, 999999) AS inner_index_end, + program_id, + signers[0]::string as swapper, + silver.udf_get_account_pubkey_by_name('userSource', decoded_instruction:accounts) as source_token_account, + null as source_mint, + null as destination_mint, + silver.udf_get_account_pubkey_by_name('userDestination', decoded_instruction:accounts) as destination_token_account, + silver.udf_get_account_pubkey_by_name('poolDestination', decoded_instruction:accounts) as program_destination_token_account, + silver.udf_get_account_pubkey_by_name('poolSource', decoded_instruction:accounts) as program_source_token_account, + _inserted_timestamp + FROM + base +), +transfers AS ( + SELECT + A.*, + COALESCE(SPLIT_PART(INDEX :: text, '.', 1) :: INT, INDEX :: INT) AS index_1, + NULLIF(SPLIT_PART(INDEX :: text, '.', 2), '') :: INT AS inner_index_1 + FROM + {{ ref('silver__transfers') }} A + INNER JOIN ( + SELECT + DISTINCT tx_id + FROM + decoded + ) d + ON d.tx_id = A.tx_id + WHERE + A.succeeded + +{% if is_incremental() %} +AND A._inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 day' + FROM + {{ this }} +) +{% else %} + AND A.block_timestamp :: DATE >= '2022-06-09' +{% endif %} +), + +pre_final as ( + SELECT + A.block_id, + A.block_timestamp, + A.program_id, + A.tx_id, + A.index, + A.inner_index, + A.inner_index_end, + C.succeeded, + A.swapper, + b.amount AS from_amt, + b.mint AS from_mint, + C.amount AS to_amt, + C.mint AS to_mint, + A._inserted_timestamp + FROM + decoded A + INNER JOIN transfers b + ON A.tx_id = b.tx_id + AND A.source_token_account = b.source_token_account + AND A.program_source_token_account = b.dest_token_account + AND A.index = b.index_1 + AND ((b.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) or a.inner_index is null) + INNER JOIN transfers C + ON A.tx_id = C.tx_id + AND A.destination_token_account = C.dest_token_account + AND A.program_destination_token_account = C.source_token_account + AND A.index = C.index_1 + AND ((c.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) or a.inner_index is null) + qualify(ROW_NUMBER() over (PARTITION BY A.tx_id, A.index, A.inner_INDEX ORDER BY inner_index)) = 1 +) + +SELECT + block_id, + block_timestamp, + program_id, + tx_id, + ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + INDEX, + inner_index + ) AS swap_index, + succeeded, + swapper, + from_amt, + from_mint, + to_amt, + to_mint, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['tx_id','swap_index','program_id']) }} AS swaps_intermediate_dooar_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id +FROM + pre_final + diff --git a/models/silver/swaps/silver__swaps_intermediate_dooar.yml b/models/silver/swaps/silver__swaps_intermediate_dooar.yml new file mode 100644 index 00000000..b2f3dacc --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_dooar.yml @@ -0,0 +1,78 @@ +version: 2 +models: + - name: silver__swaps_intermediate_dooar + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - SWAP_INDEX + - PROGRAM_ID + where: block_timestamp::date > current_date - 30 + - compare_model_subset: + name: silver__swaps_intermediate_dooar_business_logic_test + compare_model: ref('testing__swaps_intermediate_dooar') + compare_columns: + - tx_id + - swapper + - round(from_amt,8) + - from_mint + - round(to_amt,8) + - to_mint + - swap_index + model_condition: "where tx_id in ('31at6mYffEH2eFEFQxFt5vqsJpNpZx8Uofn1R53ABLaLyqAsgrUDQDqxiWSJ5k7JEZk8NF9V4kzT87PTctBHy28G', + '4n6VDefX6cPhJ3pdTpUSqo7FxSDeqeymz7abto5V7KJppBZfhH7KDvZ4m8FCEdkeckkajbjW1KbMq6zqpzYptQk5', + '4KvNEeQpzCx9QmsoGt8QNjtBDomYBnGMSNn1GeyWn6qQn2tsavPGEH26jbfoHHHUcesVbogJfXr9ghD6hv9rz6hD', + '3dKk8sbjZ2yocabr8qUaLHsa7bkd4qxnW8gjsqwhTprVfMrPGeCFqmwu2vYFytaE1NViqtiY4ALZjR2we3xnsmP')" + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - name: PROGRAM_ID + description: "{{ doc('program_id') }}" + tests: + - not_null + - name: SWAPPER + description: "{{ doc('swaps_swapper') }}" + tests: + - not_null: + where: succeeded = TRUE + - name: FROM_AMT + description: "{{ doc('swaps_from_amt') }}" + tests: + - not_null + - name: FROM_MINT + description: "{{ doc('swaps_from_mint') }}" + tests: + - not_null + - name: TO_AMT + description: "{{ doc('swaps_to_amt') }}" + tests: + - not_null + - name: TO_MINT + description: "{{ doc('swaps_to_mint') }}" + tests: + - not_null + - name: SWAP_INDEX + description: "{{ doc('swaps_swap_index') }}" + tests: + - not_null + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null \ No newline at end of file diff --git a/models/silver/swaps/silver__swaps_intermediate_meteora.sql b/models/silver/swaps/silver__swaps_intermediate_meteora.sql new file mode 100644 index 00000000..96d91612 --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_meteora.sql @@ -0,0 +1,174 @@ +{{ config( + materialized = 'incremental', + unique_key = ['swaps_intermediate_meteora_id'], + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['scheduled_non_core'], +) }} + +WITH base AS ( + + SELECT + * + FROM + {{ ref('silver__decoded_instructions_combined') }} + WHERE + program_id in ('LBUZKhRxPF3XUpBCjp4YzTKgLccjZhTSDM9YuVaPwxo', -- DLMM program + 'Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB' -- AMM program + ) + AND + event_type = 'swap' +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 hour' + FROM + {{ this }} +) +{% else %} + AND block_timestamp :: DATE >= '2022-07-14' +{% endif %} +), +decoded AS ( + SELECT + block_timestamp, + block_id, + tx_id, + index, + inner_index, + COALESCE(LEAD(inner_index) over (PARTITION BY tx_id, INDEX + ORDER BY + inner_index) -1, 999999) AS inner_index_end, + program_id, + silver.udf_get_account_pubkey_by_name('user', decoded_instruction:accounts) as swapper, + silver.udf_get_account_pubkey_by_name('userTokenIn', decoded_instruction:accounts) as source_token_account, + silver.udf_get_account_pubkey_by_name('tokenXMint', decoded_instruction:accounts) as source_mint, + silver.udf_get_account_pubkey_by_name('tokenYMint', decoded_instruction:accounts) as destination_mint, + silver.udf_get_account_pubkey_by_name('userTokenOut', decoded_instruction:accounts) as destination_token_account, + silver.udf_get_account_pubkey_by_name('reserveY', decoded_instruction:accounts) as program_destination_token_account, + silver.udf_get_account_pubkey_by_name('reserveX', decoded_instruction:accounts) as program_source_token_account, + _inserted_timestamp + FROM + base + where program_id = 'LBUZKhRxPF3XUpBCjp4YzTKgLccjZhTSDM9YuVaPwxo' + union all + SELECT + block_timestamp, + block_id, + tx_id, + index, + inner_index, + COALESCE(LEAD(inner_index) over (PARTITION BY tx_id, INDEX + ORDER BY + inner_index) -1, 999999) AS inner_index_end, + program_id, + silver.udf_get_account_pubkey_by_name('user', decoded_instruction:accounts) as swapper, + silver.udf_get_account_pubkey_by_name('userSourceToken', decoded_instruction:accounts) as source_token_account, + null as source_mint, + null as destination_mint, + silver.udf_get_account_pubkey_by_name('userDestinationToken', decoded_instruction:accounts) as destination_token_account, + silver.udf_get_account_pubkey_by_name('aTokenVault', decoded_instruction:accounts) as program_destination_token_account, + silver.udf_get_account_pubkey_by_name('bTokenVault', decoded_instruction:accounts) as program_source_token_account, + _inserted_timestamp + FROM + base + where program_id = 'Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB' + and (decoded_instruction:args:amountIn::int <> 0 or decoded_instruction:args:inAmount::int <> 0) +), +transfers AS ( + SELECT + A.*, + COALESCE(SPLIT_PART(INDEX :: text, '.', 1) :: INT, INDEX :: INT) AS index_1, + NULLIF(SPLIT_PART(INDEX :: text, '.', 2), '') :: INT AS inner_index_1 + FROM + {{ ref('silver__transfers') }} A + INNER JOIN ( + SELECT + DISTINCT tx_id + FROM + decoded + ) d + ON d.tx_id = A.tx_id + WHERE + A.succeeded + +{% if is_incremental() %} +AND A._inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 day' + FROM + {{ this }} +) +{% else %} + AND A.block_timestamp :: DATE >= '2022-07-14' +{% endif %} +), +pre_final as ( +SELECT + A.block_id, + A.block_timestamp, + A.program_id, + A.tx_id, + A.index, + A.inner_index, + A.inner_index_end, + coalesce (b.succeeded,d.succeeded) AS succeeded, + A.swapper, + coalesce (b.amount,d.amount) AS from_amt, + coalesce(b.mint,d.mint) AS from_mint, + coalesce(C.amount,e.amount) AS to_amt, + coalesce(c.mint,e.mint) AS to_mint, + A._inserted_timestamp +FROM + decoded A + left JOIN transfers b + ON A.tx_id = b.tx_id + AND A.source_token_account = b.source_token_account + AND A.program_source_token_account = b.dest_token_account + AND A.index = b.index_1 + AND ((b.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) or a.inner_index is null) + left JOIN transfers C + ON A.tx_id = C.tx_id + AND A.destination_token_account = C.dest_token_account + AND A.program_destination_token_account = C.source_token_account + AND A.index = C.index_1 + AND ((C.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) or a.inner_index is null) + left JOIN transfers d + ON A.tx_id = d.tx_id + AND A.source_token_account = d.source_token_account + AND A.program_destination_token_account = d.dest_token_account + AND A.index = d.index_1 + AND ((d.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) or a.inner_index is null) + left JOIN transfers e + ON A.tx_id = e.tx_id + AND A.destination_token_account = e.dest_token_account + AND A.program_source_token_account = e.source_token_account + AND A.index = e.index_1 + AND ((e.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) or a.inner_index is null) + qualify(ROW_NUMBER() over (PARTITION BY A.tx_id, A.index, A.inner_INDEX ORDER BY inner_index)) = 1) + +SELECT + block_id, + block_timestamp, + program_id, + tx_id, + ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + INDEX, + inner_index + ) AS swap_index, + succeeded, + swapper, + from_amt, + from_mint, + to_amt, + to_mint, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['tx_id','swap_index','program_id']) }} AS swaps_intermediate_meteora_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id +FROM + pre_final diff --git a/models/silver/swaps/silver__swaps_intermediate_meteora.yml b/models/silver/swaps/silver__swaps_intermediate_meteora.yml new file mode 100644 index 00000000..18a99eee --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_meteora.yml @@ -0,0 +1,76 @@ +version: 2 +models: + - name: silver__swaps_intermediate_meteora + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - SWAP_INDEX + - PROGRAM_ID + where: block_timestamp::date > current_date - 30 + - compare_model_subset: + name: silver__swaps_intermediate_meteora_business_logic_test + compare_model: ref('testing__swaps_intermediate_meteora') + compare_columns: + - tx_id + - swapper + - round(from_amt,8) + - from_mint + - round(to_amt,8) + - to_mint + - swap_index + model_condition: "where tx_id in ('3DjUBQE4GwTMSFLGMi69TGMiBoNE8FfnryR8BCEmovGVmMz9Jfh9dG3ikLAPuQPWpv12ypS4nYx1Esc4TYUw3s44', + '3RCs1zgEKNemq1syYM3ZZKHkckZgBbkgtKVq41W7j2yh5Ce4pVgHLU6QSiZX3nrijZ4S2yhYDb3K61seLR4c9FYu', + 'iBuwG5mU4LKMrN6iNVzGZSC7vXqvAfRQ98YJeKXPoDbM2oi7dvV8ZMnwgv55Db6JT4iokQ6JbVpN7jHnKkFtJqC', + 'NfvsjHzJx5s5JWDK7pJaWUv7hXJGdWUYwqaoRVLxQ2d5q3Jyrs2uft9FUyeNFVyxVSVZFPdsTv45dpgdog9RyzQ', + '2eV5PUm4WL7X9T1X4nSbBmAGDLTx9pUiygK34ngMSyR9QwQUS8ZpnzGKTCbuJHeVGYNHjHBGtPh4fLYxkBLGVJeB', + '35QkSCVWRNFaqFM8DZrDWKWsBdDTrHVMmskd3H9D3pcQstdYsWqZiXVhMpkaCFrBQTi8XZaWgNZaWnQZzwPpZ1dv')" + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - name: PROGRAM_ID + description: "{{ doc('program_id') }}" + tests: + - not_null + - name: SWAPPER + description: "{{ doc('swaps_swapper') }}" + tests: + - not_null: + where: succeeded = TRUE + - name: FROM_AMT + description: "{{ doc('swaps_from_amt') }}" + tests: + - not_null + - name: FROM_MINT + description: "{{ doc('swaps_from_mint') }}" + tests: + - not_null + - name: TO_AMT + description: "{{ doc('swaps_to_amt') }}" + - name: TO_MINT + description: "{{ doc('swaps_to_mint') }}" + - name: SWAP_INDEX + description: "{{ doc('swaps_swap_index') }}" + tests: + - not_null + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null \ No newline at end of file diff --git a/models/silver/swaps/silver__swaps_intermediate_phoenix.sql b/models/silver/swaps/silver__swaps_intermediate_phoenix.sql new file mode 100644 index 00000000..0861d346 --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_phoenix.sql @@ -0,0 +1,181 @@ +{{ config( + materialized = 'incremental', + unique_key = ['swaps_intermediate_phoenix_id'], + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['scheduled_non_core'], +) }} + +WITH base AS ( + + SELECT + * + FROM + {{ ref('silver__decoded_instructions_combined') }} + WHERE + program_id = 'PhoeNiXZ8ByJGLkxNfZRnkUfjvmuYqLR89jjFHGqdXY' + AND event_type = 'Swap' + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 hour' + FROM + {{ this }} +) +{% else %} + AND block_timestamp :: DATE >= '2023-02-15' +{% endif %} +), +decoded AS ( + SELECT + block_timestamp, + block_id, + tx_id, + INDEX, + inner_index, + COALESCE(LEAD(inner_index) over (PARTITION BY tx_id, INDEX + ORDER BY + inner_index) -1, 999999) AS inner_index_end, + program_id, + CASE + WHEN decoded_instruction :args :orderPacket :immediateOrCancel :side :bid IS NOT NULL THEN 'bid' + ELSE 'ask' + END AS side, + silver.udf_get_account_pubkey_by_name( + 'trader', + decoded_instruction :accounts + ) AS swapper, + CASE + WHEN side = 'ask' THEN silver.udf_get_account_pubkey_by_name( + 'quoteVault', + decoded_instruction :accounts + ) + ELSE silver.udf_get_account_pubkey_by_name( + 'baseVault', + decoded_instruction :accounts + ) + END AS source_token_account, + NULL AS source_mint, + NULL AS destination_mint, + CASE + WHEN side = 'ask' THEN silver.udf_get_account_pubkey_by_name( + 'baseVault', + decoded_instruction :accounts + ) + ELSE silver.udf_get_account_pubkey_by_name( + 'quoteVault', + decoded_instruction :accounts + ) + END AS destination_token_account, + CASE + WHEN side = 'ask' THEN silver.udf_get_account_pubkey_by_name( + 'baseAccount', + decoded_instruction :accounts + ) + ELSE silver.udf_get_account_pubkey_by_name( + 'quoteAccount', + decoded_instruction :accounts + ) + END AS program_destination_token_account, + CASE + WHEN side = 'ask' THEN silver.udf_get_account_pubkey_by_name( + 'quoteAccount', + decoded_instruction :accounts + ) + ELSE silver.udf_get_account_pubkey_by_name( + 'baseAccount', + decoded_instruction :accounts + ) + END AS program_source_token_account, + _inserted_timestamp + FROM + base +), +transfers AS ( + SELECT + A.*, + COALESCE(SPLIT_PART(INDEX :: text, '.', 1) :: INT, INDEX :: INT) AS index_1, + NULLIF(SPLIT_PART(INDEX :: text, '.', 2), '') :: INT AS inner_index_1 + FROM + {{ ref('silver__transfers') }} A + INNER JOIN ( + SELECT + DISTINCT tx_id + FROM + decoded + ) d + ON d.tx_id = A.tx_id + WHERE + A.succeeded + +{% if is_incremental() %} +AND A._inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 day' + FROM + {{ this }} +) +{% else %} + AND A.block_timestamp :: DATE >= '2023-02-15' +{% endif %} +), +pre_final AS ( + SELECT + A.block_id, + A.block_timestamp, + A.program_id, + A.tx_id, + A.index, + A.inner_index, + A.inner_index_end, + C.succeeded, + A.swapper, + b.amount AS from_amt, + b.mint AS from_mint, + C.amount AS to_amt, + C.mint AS to_mint, + A._inserted_timestamp + FROM + decoded A + INNER JOIN transfers b + ON A.tx_id = b.tx_id + AND A.source_token_account = b.source_token_account + AND A.program_source_token_account = b.dest_token_account + AND A.index = b.index_1 + AND b.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end + INNER JOIN transfers C + ON A.tx_id = C.tx_id + AND A.destination_token_account = C.dest_token_account + AND A.program_destination_token_account = C.source_token_account + AND A.index = C.index_1 + AND C.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end + qualify(ROW_NUMBER() over (PARTITION BY A.tx_id, A.index, A.inner_INDEX + ORDER BY + inner_index)) = 1 +) +SELECT + block_id, + block_timestamp, + program_id, + tx_id, + ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + INDEX, + inner_index + ) AS swap_index, + succeeded, + swapper, + from_amt, + from_mint, + to_amt, + to_mint, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['tx_id','swap_index','program_id']) }} AS swaps_intermediate_phoenix_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id +FROM + pre_final diff --git a/models/silver/swaps/silver__swaps_intermediate_phoenix.yml b/models/silver/swaps/silver__swaps_intermediate_phoenix.yml new file mode 100644 index 00000000..e65593df --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_phoenix.yml @@ -0,0 +1,85 @@ +version: 2 +models: + - name: silver__swaps_intermediate_phoenix + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - SWAP_INDEX + - PROGRAM_ID + where: block_timestamp::date > current_date - 30 + - compare_model_subset: + name: silver__swaps_intermediate_phoenix_business_logic_test + compare_model: ref('testing__swaps_intermediate_phoenix') + compare_columns: + - tx_id + - swapper + - round(from_amt,8) + - from_mint + - round(to_amt,8) + - to_mint + - swap_index + model_condition: "where tx_id in ('3fp9QzP6EMn9cUDgCRmHACSxmDhh3zrMebYKSm8Auq3xyBck7X6nRSrrVDThDhM1wRBaVidwh4BCDLn86Bve3thL', + '671GWKC1AGyvrAThDzM9tq9UqVW8W2WdULAikkLaaTSZVYEmzLEhXutdDjZTenLp5oAtc4G7f4bztZ978rN1Zz1d', + '3FgxCNCndXDSx95mrsBwcfsHSM9fDzWSq2bJqN74ym5MTgF2VrLJRn4w5UfdKpeKNA3mzCmm7KJfy7wuDAo4y7Ko', + '21qdi1d8vYyMp3AcJmPdKBi1VsEoaBn4qhB1WJhzRR9N3qvtXRkENh5caaBGDaeUVD68CUw72bD2neCQA7dmgUuS', + '3EweEGRQkdQ4e5sRRcU5V9Jz2E9sMHzDrbmSD13C2wLQJkANJEZkk9EHJqpUSKUSWCAnY5nFnLgXUqJF5ham6ARk')" + - tx_excluded: + name: silver__swaps_intermediate_phoenix_tx_excluded_test + excluded_tx_ids: + - 44b7tkzuKHqQsn7Eoh3RnTy12sLijudPFz3ZX2ewJ7vxKqs5LveeNKxBVLUstggr5Jk3LFzoNs9SSZxcMSSgVE7j + - 1M48Smd6EQ4phktTsGfdk3HjXf2sEW6J8TpUpPjEaGjLy5viWPC9D4rS9SoutVWTJkLkjbVLQpEG9eboscoybQ6 + - fLwJbyNV8kF3qZcmUrPrQB7B1keipAB5XNiEsJ8oFtvMroZ4SttabxgP2nhUsTT1jKvN4NfrSf8DMYVoadY57Da + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - name: PROGRAM_ID + description: "{{ doc('program_id') }}" + tests: + - not_null + - name: SWAPPER + description: "{{ doc('swaps_swapper') }}" + tests: + - not_null: + where: succeeded = TRUE + - name: FROM_AMT + description: "{{ doc('swaps_from_amt') }}" + tests: + - not_null + - name: FROM_MINT + description: "{{ doc('swaps_from_mint') }}" + tests: + - not_null + - name: TO_AMT + description: "{{ doc('swaps_to_amt') }}" + tests: + - not_null + - name: TO_MINT + description: "{{ doc('swaps_to_mint') }}" + tests: + - not_null + - name: SWAP_INDEX + description: "{{ doc('swaps_swap_index') }}" + tests: + - not_null + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null \ No newline at end of file