From 68c1eec35c858c67da40c16a0ec57af38753588b Mon Sep 17 00:00:00 2001 From: tarikceric <46071768+tarikceric@users.noreply.github.com> Date: Tue, 11 Feb 2025 13:22:16 -0800 Subject: [PATCH] optimize cte distinct joins (#797) * optimize cte distinct joins * formatting * formatting --- ...silver__bridge_mayan_transfers_decoded.sql | 6 +-- .../silver__liquidity_pool_actions_orcav1.sql | 15 ++++--- .../silver__liquidity_pool_actions_orcav2.sql | 15 ++++--- ..._liquidity_pool_actions_orca_whirlpool.sql | 14 ++++--- ...r__liquidity_pool_actions_raydium_clmm.sql | 14 ++++--- ...r__liquidity_pool_actions_raydium_cpmm.sql | 14 ++++--- ...lver__liquidity_pool_actions_raydiumv4.sql | 14 ++++--- .../silver__nft_mint_price_candy_machine.sql | 12 ++++-- .../silver__nft_sales_hadeswap_decoded.sql | 6 +-- ...ilver__nft_sales_magic_eden_v2_decoded.sql | 8 ++-- .../nfts/silver__nft_sales_solsniper.sql | 9 ++-- .../nfts/silver__nft_sales_tensorswap.sql | 16 ++++--- ...ilver__marinade_liquid_staking_actions.sql | 42 +++++++------------ .../silver__swaps_intermediate_phoenix.sql | 11 ++--- 14 files changed, 105 insertions(+), 91 deletions(-) diff --git a/models/silver/bridges/silver__bridge_mayan_transfers_decoded.sql b/models/silver/bridges/silver__bridge_mayan_transfers_decoded.sql index f2606ae1..4c2a6212 100644 --- a/models/silver/bridges/silver__bridge_mayan_transfers_decoded.sql +++ b/models/silver/bridges/silver__bridge_mayan_transfers_decoded.sql @@ -131,9 +131,9 @@ transfers AS ( FROM {{ ref('silver__transfers') }} a INNER JOIN ( - SELECT DISTINCT tx_id - FROM bridge_to - ) d ON d.tx_id = a.tx_id + SELECT DISTINCT tx_id, block_timestamp::date as bt FROM bridge_to) d + ON d.tx_id = a.tx_id + AND d.bt = a.block_timestamp::date WHERE a.succeeded AND {{ between_stmts }} diff --git a/models/silver/liquidity_pool/orca/v1/silver__liquidity_pool_actions_orcav1.sql b/models/silver/liquidity_pool/orca/v1/silver__liquidity_pool_actions_orcav1.sql index edcf61b8..1a559f9f 100644 --- a/models/silver/liquidity_pool/orca/v1/silver__liquidity_pool_actions_orcav1.sql +++ b/models/silver/liquidity_pool/orca/v1/silver__liquidity_pool_actions_orcav1.sql @@ -106,13 +106,18 @@ WITH base AS ( transfers AS ( SELECT - * exclude(index), - split_part(index,'.',1)::int AS index, - nullif(split_part(index,'.',2),'')::int AS inner_index + t.* exclude(index), + split_part(t.index,'.',1)::int AS index, + nullif(split_part(t.index,'.',2),'')::int AS inner_index FROM - {{ ref('silver__transfers') }} + {{ ref('silver__transfers') }} AS t + INNER JOIN + (SELECT DISTINCT block_timestamp::date AS bt, tx_id FROM base) AS b + ON b.bt = t.block_timestamp::date + AND b.tx_id = t.tx_id WHERE - {{ between_stmts }} + t.succeeded + AND {{ between_stmts }} ), deposit_transfers AS ( diff --git a/models/silver/liquidity_pool/orca/v2/silver__liquidity_pool_actions_orcav2.sql b/models/silver/liquidity_pool/orca/v2/silver__liquidity_pool_actions_orcav2.sql index c3fad6e3..4c63bbd3 100644 --- a/models/silver/liquidity_pool/orca/v2/silver__liquidity_pool_actions_orcav2.sql +++ b/models/silver/liquidity_pool/orca/v2/silver__liquidity_pool_actions_orcav2.sql @@ -92,13 +92,18 @@ WITH base AS ( transfers AS ( SELECT - * exclude(index), - split_part(index,'.',1)::int AS index, - nullif(split_part(index,'.',2),'')::int AS inner_index + t.* exclude(index), + split_part(t.index,'.',1)::int AS index, + nullif(split_part(t.index,'.',2),'')::int AS inner_index FROM - {{ ref('silver__transfers') }} + {{ ref('silver__transfers') }} AS t + INNER JOIN + (SELECT DISTINCT block_timestamp::date AS bt, tx_id FROM base) AS b + ON b.bt = t.block_timestamp::date + AND b.tx_id = t.tx_id WHERE - {{ between_stmts }} + t.succeeded + AND {{ between_stmts }} ), deposit_transfers AS ( diff --git a/models/silver/liquidity_pool/orca/whirlpool/silver__liquidity_pool_actions_orca_whirlpool.sql b/models/silver/liquidity_pool/orca/whirlpool/silver__liquidity_pool_actions_orca_whirlpool.sql index 3c49534a..dfca49a3 100644 --- a/models/silver/liquidity_pool/orca/whirlpool/silver__liquidity_pool_actions_orca_whirlpool.sql +++ b/models/silver/liquidity_pool/orca/whirlpool/silver__liquidity_pool_actions_orca_whirlpool.sql @@ -84,13 +84,17 @@ WITH base AS ( transfers AS ( SELECT - * exclude(index), - split_part(index,'.',1)::int AS index, - nullif(split_part(index,'.',2),'')::int AS inner_index + t.* exclude(index), + split_part(t.index,'.',1)::int AS index, + nullif(split_part(t.index,'.',2),'')::int AS inner_index FROM - {{ ref('silver__transfers') }} + {{ ref('silver__transfers') }} AS t + INNER JOIN + (SELECT DISTINCT block_timestamp::date AS bt, tx_id FROM base) AS b + ON b.bt = t.block_timestamp::date + AND b.tx_id = t.tx_id WHERE - succeeded + t.succeeded AND {{ between_stmts }} ), diff --git a/models/silver/liquidity_pool/raydium/clmm/silver__liquidity_pool_actions_raydium_clmm.sql b/models/silver/liquidity_pool/raydium/clmm/silver__liquidity_pool_actions_raydium_clmm.sql index 47326eb4..8a91fc61 100644 --- a/models/silver/liquidity_pool/raydium/clmm/silver__liquidity_pool_actions_raydium_clmm.sql +++ b/models/silver/liquidity_pool/raydium/clmm/silver__liquidity_pool_actions_raydium_clmm.sql @@ -120,13 +120,17 @@ WITH base AS ( transfers AS ( SELECT - * exclude(index), - split_part(index,'.',1)::int AS index, - nullif(split_part(index,'.',2),'')::int AS inner_index + t.* exclude(index), + split_part(t.index,'.',1)::int AS index, + nullif(split_part(t.index,'.',2),'')::int AS inner_index FROM - {{ ref('silver__transfers') }} + {{ ref('silver__transfers') }} AS t + INNER JOIN + (SELECT DISTINCT block_timestamp::date AS bt, tx_id FROM base) AS b + ON b.bt = t.block_timestamp::date + AND b.tx_id = t.tx_id WHERE - succeeded + t.succeeded AND {{ between_stmts }} ), diff --git a/models/silver/liquidity_pool/raydium/cpmm/silver__liquidity_pool_actions_raydium_cpmm.sql b/models/silver/liquidity_pool/raydium/cpmm/silver__liquidity_pool_actions_raydium_cpmm.sql index a0cdea9e..2db9bd4e 100644 --- a/models/silver/liquidity_pool/raydium/cpmm/silver__liquidity_pool_actions_raydium_cpmm.sql +++ b/models/silver/liquidity_pool/raydium/cpmm/silver__liquidity_pool_actions_raydium_cpmm.sql @@ -78,13 +78,17 @@ WITH base AS ( transfers AS ( SELECT - * exclude(index), - split_part(index,'.',1)::int AS index, - nullif(split_part(index,'.',2),'')::int AS inner_index + t.* exclude(index), + split_part(t.index,'.',1)::int AS index, + nullif(split_part(t.index,'.',2),'')::int AS inner_index FROM - {{ ref('silver__transfers') }} + {{ ref('silver__transfers') }} AS t + INNER JOIN + (SELECT DISTINCT block_timestamp::date AS bt, tx_id FROM base) AS b + ON b.bt = t.block_timestamp::date + AND b.tx_id = t.tx_id WHERE - succeeded + t.succeeded AND {{ between_stmts }} ), diff --git a/models/silver/liquidity_pool/raydium/v4/silver__liquidity_pool_actions_raydiumv4.sql b/models/silver/liquidity_pool/raydium/v4/silver__liquidity_pool_actions_raydiumv4.sql index 6bc79ed8..d4602482 100644 --- a/models/silver/liquidity_pool/raydium/v4/silver__liquidity_pool_actions_raydiumv4.sql +++ b/models/silver/liquidity_pool/raydium/v4/silver__liquidity_pool_actions_raydiumv4.sql @@ -124,13 +124,17 @@ WITH base AS ( transfers AS ( SELECT - * exclude(index), - split_part(index,'.',1)::int AS index, - nullif(split_part(index,'.',2),'')::int AS inner_index + t.* exclude(index), + split_part(t.index,'.',1)::int AS index, + nullif(split_part(t.index,'.',2),'')::int AS inner_index FROM - {{ ref('silver__transfers') }} + {{ ref('silver__transfers') }} AS t + INNER JOIN + (SELECT DISTINCT block_timestamp::date AS bt, tx_id FROM base) AS b + ON b.bt = t.block_timestamp::date + AND b.tx_id = t.tx_id WHERE - succeeded + t.succeeded AND {{ between_stmts }} ), diff --git a/models/silver/nfts/silver__nft_mint_price_candy_machine.sql b/models/silver/nfts/silver__nft_mint_price_candy_machine.sql index a6f07354..54c77aa5 100644 --- a/models/silver/nfts/silver__nft_mint_price_candy_machine.sql +++ b/models/silver/nfts/silver__nft_mint_price_candy_machine.sql @@ -40,11 +40,15 @@ AND _inserted_timestamp >= ( ), base_ptb AS ( SELECT - DISTINCT mint AS mint_paid, - account, - DECIMAL + DISTINCT p.mint AS mint_paid, + p.account, + p.DECIMAL FROM - {{ ref('silver___post_token_balances') }} + {{ ref('silver___post_token_balances') }} AS p + INNER JOIN + (SELECT DISTINCT block_timestamp::date AS bt, tx_id FROM base_candy_machine_events) AS b + ON b.bt = p.block_timestamp::date + AND b.tx_id = p.tx_id {% if is_incremental() %} WHERE diff --git a/models/silver/nfts/silver__nft_sales_hadeswap_decoded.sql b/models/silver/nfts/silver__nft_sales_hadeswap_decoded.sql index ed28b88a..d84f195a 100644 --- a/models/silver/nfts/silver__nft_sales_hadeswap_decoded.sql +++ b/models/silver/nfts/silver__nft_sales_hadeswap_decoded.sql @@ -91,9 +91,9 @@ transfers AS ( FROM {{ ref('silver__transfers') }} A INNER JOIN ( - SELECT DISTINCT tx_id - FROM decoded - ) d ON d.tx_id = A.tx_id + SELECT DISTINCT tx_id, block_timestamp::date as bt FROM decoded) d + ON d.tx_id = A.tx_id + AND d.bt = A.block_timestamp::date WHERE A.succeeded AND {{ between_stmts }} diff --git a/models/silver/nfts/silver__nft_sales_magic_eden_v2_decoded.sql b/models/silver/nfts/silver__nft_sales_magic_eden_v2_decoded.sql index 43f5d739..c5e78e47 100644 --- a/models/silver/nfts/silver__nft_sales_magic_eden_v2_decoded.sql +++ b/models/silver/nfts/silver__nft_sales_magic_eden_v2_decoded.sql @@ -95,10 +95,10 @@ transfers AS ( a._inserted_timestamp FROM {{ ref('silver__transfers') }} a - INNER JOIN ( - SELECT DISTINCT tx_id - FROM decoded - ) d ON d.tx_id = a.tx_id + INNER JOIN + (SELECT DISTINCT tx_id, block_timestamp::date as bt FROM decoded) d + ON d.tx_id = a.tx_id + AND d.bt = a.block_timestamp::date WHERE a.succeeded AND {{ between_stmts }} diff --git a/models/silver/nfts/silver__nft_sales_solsniper.sql b/models/silver/nfts/silver__nft_sales_solsniper.sql index 4605cc36..f5604416 100644 --- a/models/silver/nfts/silver__nft_sales_solsniper.sql +++ b/models/silver/nfts/silver__nft_sales_solsniper.sql @@ -78,13 +78,10 @@ transfers AS ( NULLIF(SPLIT_PART(INDEX :: text, '.', 2), '') :: INT AS inner_index_1 FROM {{ ref('silver__transfers') }} A - INNER JOIN ( - SELECT - DISTINCT tx_id - FROM - decoded - ) d + INNER JOIN + (SELECT DISTINCT tx_id, block_timestamp::date as bt FROM decoded) d ON d.tx_id = A.tx_id + AND d.bt = A.block_timestamp::date WHERE A.succeeded AND {{ between_stmts }} diff --git a/models/silver/nfts/silver__nft_sales_tensorswap.sql b/models/silver/nfts/silver__nft_sales_tensorswap.sql index a8736ddd..0aa21b32 100644 --- a/models/silver/nfts/silver__nft_sales_tensorswap.sql +++ b/models/silver/nfts/silver__nft_sales_tensorswap.sql @@ -151,14 +151,18 @@ WITH decoded AS ( ), base_transfers AS ( SELECT - block_timestamp, - tx_id, - index, - tx_from, - mint, - amount + t.block_timestamp, + t.tx_id, + t.index, + t.tx_from, + t.mint, + t.amount FROM {{ ref('silver__transfers') }} t + INNER JOIN + (SELECT DISTINCT block_timestamp::date AS bt, tx_id FROM decoded) AS d + ON d.bt = t.block_timestamp::date + AND d.tx_id = t.tx_id WHERE {{ between_stmts }} ), diff --git a/models/silver/protocols/marinade/silver__marinade_liquid_staking_actions.sql b/models/silver/protocols/marinade/silver__marinade_liquid_staking_actions.sql index 9ebd046c..ac8f1df2 100644 --- a/models/silver/protocols/marinade/silver__marinade_liquid_staking_actions.sql +++ b/models/silver/protocols/marinade/silver__marinade_liquid_staking_actions.sql @@ -55,16 +55,12 @@ WITH base AS ( mints AS ( SELECT a.* - FROM {{ ref('silver__token_mint_actions') }} a - INNER JOIN ( - SELECT - DISTINCT tx_id - FROM - base - WHERE - event_type IN ('deposit', 'depositStakeAccount') - ) b - ON b.tx_id = a.tx_id + FROM + {{ ref('silver__token_mint_actions') }} a + INNER JOIN + (SELECT DISTINCT tx_id, block_timestamp::date as bt FROM base WHERE event_type IN ('deposit', 'depositStakeAccount')) b + ON b.tx_id = a.tx_id + AND b.bt = a.block_timestamp::date WHERE a.succeeded AND {{ between_stmts }} @@ -76,15 +72,10 @@ transfers AS ( nullif(split_part(a.index,'.',2),'')::int AS inner_index FROM {{ ref('silver__transfers') }} a - INNER JOIN ( - SELECT - DISTINCT tx_id - FROM - base - WHERE - event_type = 'claim' - ) b - ON b.tx_id = a.tx_id + INNER JOIN + (SELECT DISTINCT tx_id, block_timestamp::date as bt FROM base WHERE event_type = 'claim') b + ON b.tx_id = a.tx_id + AND b.bt = a.block_timestamp::date WHERE a.succeeded AND {{ between_stmts }} @@ -94,15 +85,10 @@ sol_balances as ( a.* FROM {{ ref('silver__sol_balances') }} a - INNER JOIN ( - SELECT - DISTINCT tx_id - FROM - base - WHERE - event_type = 'depositStakeAccount' - ) b - ON b.tx_id = a.tx_id + INNER JOIN + (SELECT DISTINCT tx_id, block_timestamp::date as bt FROM base WHERE event_type = 'depositStakeAccount') b + ON b.tx_id = a.tx_id + AND b.bt = a.block_timestamp::date WHERE a.succeeded AND {{ between_stmts }} diff --git a/models/silver/swaps/phoenix/silver__swaps_intermediate_phoenix.sql b/models/silver/swaps/phoenix/silver__swaps_intermediate_phoenix.sql index d2ab9856..b3426c5f 100644 --- a/models/silver/swaps/phoenix/silver__swaps_intermediate_phoenix.sql +++ b/models/silver/swaps/phoenix/silver__swaps_intermediate_phoenix.sql @@ -116,14 +116,11 @@ transfers AS ( NULLIF(SPLIT_PART(INDEX :: text, '.', 2), '') :: INT AS inner_index_1 FROM {{ ref('silver__transfers') }} A - INNER JOIN ( - SELECT - DISTINCT tx_id - FROM - decoded - ) d + INNER JOIN + (SELECT DISTINCT tx_id, block_timestamp::date as bt FROM decoded) d ON d.tx_id = A.tx_id - WHERE + AND d.bt = A.block_timestamp::date + WHERE A.succeeded and {{ between_stmts }} ),