fix incremental, logic, fix from/to cols (#539)

* fix incremental, logic, fix from/to cols

* update desc

* update silver desc

data/testing__swaps_intermediate_phoenix.csv

@@ -1,10 +1,11 @@
 tx_id,swapper,from_amt,from_mint,to_amt,to_mint,swap_index
-3fp9QzP6EMn9cUDgCRmHACSxmDhh3zrMebYKSm8Auq3xyBck7X6nRSrrVDThDhM1wRBaVidwh4BCDLn86Bve3thL,BPiunxZKqnVjfyGwxEGDJ8fo2Anc9KZW2w81gpbPRJV3,0.715816712,So11111111111111111111111111111111111111112,5954000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,1
-671GWKC1AGyvrAThDzM9tq9UqVW8W2WdULAikkLaaTSZVYEmzLEhXutdDjZTenLp5oAtc4G7f4bztZ978rN1Zz1d,bobCPc5nqVoX7r8gKzCMPLrKjFidjnSCrAdcYGCH2Ye,399.966167,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,3.421,So11111111111111111111111111111111111111112,1
-3FgxCNCndXDSx95mrsBwcfsHSM9fDzWSq2bJqN74ym5MTgF2VrLJRn4w5UfdKpeKNA3mzCmm7KJfy7wuDAo4y7Ko,FreyaXYaCwVy86BdNECd7BXnqEvFUbt3p6d2B5eS5zDv,1.494221215,So11111111111111111111111111111111111111112,13137000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,2
-3FgxCNCndXDSx95mrsBwcfsHSM9fDzWSq2bJqN74ym5MTgF2VrLJRn4w5UfdKpeKNA3mzCmm7KJfy7wuDAo4y7Ko,FreyaXYaCwVy86BdNECd7BXnqEvFUbt3p6d2B5eS5zDv,169.118343,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1.494,So11111111111111111111111111111111111111112,1
-21qdi1d8vYyMp3AcJmPdKBi1VsEoaBn4qhB1WJhzRR9N3qvtXRkENh5caaBGDaeUVD68CUw72bD2neCQA7dmgUuS,ao31QJtoqLEynyqrZbwkUTGC8mh8fU55RtiUk2pzK8R,142.767,So11111111111111111111111111111111111111112,2775.994087,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1
-21qdi1d8vYyMp3AcJmPdKBi1VsEoaBn4qhB1WJhzRR9N3qvtXRkENh5caaBGDaeUVD68CUw72bD2neCQA7dmgUuS,ao31QJtoqLEynyqrZbwkUTGC8mh8fU55RtiUk2pzK8R,135.1432,J1toso1uCk3RLmjorhTtrVwY9HJ7X8V9yYac6Y7kGCPn,142.766958204,So11111111111111111111111111111111111111112,2
-21qdi1d8vYyMp3AcJmPdKBi1VsEoaBn4qhB1WJhzRR9N3qvtXRkENh5caaBGDaeUVD68CUw72bD2neCQA7dmgUuS,ao31QJtoqLEynyqrZbwkUTGC8mh8fU55RtiUk2pzK8R,142.727107646,So11111111111111111111111111111111111111112,135.1432,J1toso1uCk3RLmjorhTtrVwY9HJ7X8V9yYac6Y7kGCPn,3
-3EweEGRQkdQ4e5sRRcU5V9Jz2E9sMHzDrbmSD13C2wLQJkANJEZkk9EHJqpUSKUSWCAnY5nFnLgXUqJF5ham6ARk,2SLowUE6xmrddRd4VeJ2i39ybekDg22BRcwsBR3vpwAG,10.188376343,So11111111111111111111111111111111111111112,271971000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,1
-3EweEGRQkdQ4e5sRRcU5V9Jz2E9sMHzDrbmSD13C2wLQJkANJEZkk9EHJqpUSKUSWCAnY5nFnLgXUqJF5ham6ARk,2SLowUE6xmrddRd4VeJ2i39ybekDg22BRcwsBR3vpwAG,1.271997083,So11111111111111111111111111111111111111112,33955000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,2
+3fp9QzP6EMn9cUDgCRmHACSxmDhh3zrMebYKSm8Auq3xyBck7X6nRSrrVDThDhM1wRBaVidwh4BCDLn86Bve3thL,BPiunxZKqnVjfyGwxEGDJ8fo2Anc9KZW2w81gpbPRJV3,5954000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,0.715816712,So11111111111111111111111111111111111111112,1
+671GWKC1AGyvrAThDzM9tq9UqVW8W2WdULAikkLaaTSZVYEmzLEhXutdDjZTenLp5oAtc4G7f4bztZ978rN1Zz1d,bobCPc5nqVoX7r8gKzCMPLrKjFidjnSCrAdcYGCH2Ye,3.421,So11111111111111111111111111111111111111112,399.966167,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1
+3FgxCNCndXDSx95mrsBwcfsHSM9fDzWSq2bJqN74ym5MTgF2VrLJRn4w5UfdKpeKNA3mzCmm7KJfy7wuDAo4y7Ko,FreyaXYaCwVy86BdNECd7BXnqEvFUbt3p6d2B5eS5zDv,1.494,So11111111111111111111111111111111111111112,169.118343,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1
+3FgxCNCndXDSx95mrsBwcfsHSM9fDzWSq2bJqN74ym5MTgF2VrLJRn4w5UfdKpeKNA3mzCmm7KJfy7wuDAo4y7Ko,FreyaXYaCwVy86BdNECd7BXnqEvFUbt3p6d2B5eS5zDv,13137000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,1.494221215,So11111111111111111111111111111111111111112,2
+21qdi1d8vYyMp3AcJmPdKBi1VsEoaBn4qhB1WJhzRR9N3qvtXRkENh5caaBGDaeUVD68CUw72bD2neCQA7dmgUuS,ao31QJtoqLEynyqrZbwkUTGC8mh8fU55RtiUk2pzK8R,2775.994087,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,142.767,So11111111111111111111111111111111111111112,1
+21qdi1d8vYyMp3AcJmPdKBi1VsEoaBn4qhB1WJhzRR9N3qvtXRkENh5caaBGDaeUVD68CUw72bD2neCQA7dmgUuS,ao31QJtoqLEynyqrZbwkUTGC8mh8fU55RtiUk2pzK8R,142.766958204,So11111111111111111111111111111111111111112,135.1432,J1toso1uCk3RLmjorhTtrVwY9HJ7X8V9yYac6Y7kGCPn,2
+21qdi1d8vYyMp3AcJmPdKBi1VsEoaBn4qhB1WJhzRR9N3qvtXRkENh5caaBGDaeUVD68CUw72bD2neCQA7dmgUuS,ao31QJtoqLEynyqrZbwkUTGC8mh8fU55RtiUk2pzK8R,135.1432,J1toso1uCk3RLmjorhTtrVwY9HJ7X8V9yYac6Y7kGCPn,142.727107646,So11111111111111111111111111111111111111112,3
+3EweEGRQkdQ4e5sRRcU5V9Jz2E9sMHzDrbmSD13C2wLQJkANJEZkk9EHJqpUSKUSWCAnY5nFnLgXUqJF5ham6ARk,2SLowUE6xmrddRd4VeJ2i39ybekDg22BRcwsBR3vpwAG,271971000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,10.188376343,So11111111111111111111111111111111111111112,1
+3EweEGRQkdQ4e5sRRcU5V9Jz2E9sMHzDrbmSD13C2wLQJkANJEZkk9EHJqpUSKUSWCAnY5nFnLgXUqJF5ham6ARk,2SLowUE6xmrddRd4VeJ2i39ybekDg22BRcwsBR3vpwAG,33955000,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,1.271997083,So11111111111111111111111111111111111111112,2
+5jP7nmkNNGnjKeqP82HRr9WNv4fBdU1dHCpNGp5bSMVNeHfs7wf1uCtifPRvvugt6gX4ESUdZHQXnhDbx9fy5sfA,MfDuWeqSHEqTFVYZ7LoexgAK9dxk7cy4DFJWjWMGVWa,380.46,EKpQGSJtjMFqKZ9KQanSqYXRcF8fBopzLHYxdM65zcjm,1618.922491,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1

models/gold/defi/defi__fact_swaps.yml

@@ -1,7 +1,7 @@
 version: 2
 models:
   - name: defi__fact_swaps
-    description: This table contains swaps performed on Jupiter, Orca, Raydium, Saber, Bonkswap, Dooar, Phoenix and Meteora swap programs. Intermediate swaps are aggregated over the DEX programs, so the values showcase the final mint/amount swap values. Ie. a swap on Jupiter that swaps SOL->USDC->mSOL->ETH would show the initial amount in and the final amount out of SOL->ETH.
+    description: This table contains swaps performed on Jupiter, Orca, Raydium, Saber, Bonkswap, Dooar, Phoenix and Meteora swap programs. Intermediate swaps are aggregated over the DEX programs, so the values showcase the final mint/amount swap values. Ie. a swap on Jupiter that swaps SOL->USDC->mSOL->ETH would show the initial amount in and the final amount out of SOL->ETH. For Phoenix, we are not capturing swaps where there are separate transactions for placing the order and filling the order. 
     columns:
       - name: BLOCK_TIMESTAMP
         description: "{{ doc('block_timestamp') }}"

models/silver/swaps/silver__swaps_intermediate_phoenix.sql

@@ -1,14 +1,17 @@
+ -- depends_on: {{ ref('silver__decoded_instructions_combined') }}
+
 {{ config(
     materialized = 'incremental',
     unique_key = ['swaps_intermediate_phoenix_id'],
-    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
     tags = ['scheduled_non_core'],
 ) }}
 
-WITH base AS (
-
+{% if execute %}
+    {% set base_query %}
+        CREATE OR REPLACE TEMPORARY TABLE silver.swaps_intermediate_phoenix__intermediate_tmp AS 
         SELECT 
             *
         FROM 
@@ -16,17 +19,29 @@ WITH base AS (
         WHERE
             program_id = 'PhoeNiXZ8ByJGLkxNfZRnkUfjvmuYqLR89jjFHGqdXY'
         AND event_type = 'Swap'
-
-{% if is_incremental() %}
-AND _inserted_timestamp >= (
+        AND succeeded
+        {% if is_incremental() %}
+        AND _inserted_timestamp >= (
             SELECT
                 MAX(_inserted_timestamp) - INTERVAL '1 hour'
             FROM
                 {{ this }}
-)
-{% else %}
-    AND block_timestamp :: DATE >= '2023-02-15'
+        )
+        {% else %} 
+            AND _inserted_timestamp :: DATE >= '2024-02-14'
+            AND _inserted_timestamp :: DATE < '2024-02-17'
+        {% endif %}
+    {% endset %}
+    {% do run_query(base_query) %}
+    {% set between_stmts = fsc_utils.dynamic_range_predicate("silver.swaps_intermediate_phoenix__intermediate_tmp","block_timestamp::date") %}
 {% endif %}
+
+WITH base AS (
+
+    SELECT
+        *
+    FROM
+        silver.swaps_intermediate_phoenix__intermediate_tmp
 ),
 decoded AS (
     SELECT
@@ -35,6 +50,7 @@ decoded AS (
         tx_id,
         INDEX,
         inner_index,
+        COALESCE(inner_index,-1) AS inner_index_start,
         COALESCE(LEAD(inner_index) over (PARTITION BY tx_id, INDEX
     ORDER BY
         inner_index) -1, 999999) AS inner_index_end,
@@ -109,17 +125,7 @@ transfers AS (
         ON d.tx_id = A.tx_id
     WHERE
         A.succeeded
-
-{% if is_incremental() %}
-AND A._inserted_timestamp >= (
-    SELECT
-        MAX(_inserted_timestamp) - INTERVAL '1 day'
-    FROM
-        {{ this }}
-)
-{% else %}
-    AND A.block_timestamp :: DATE >= '2023-02-15'
-{% endif %}
+    and {{ between_stmts }}
 ),
 pre_final AS (
     SELECT
@@ -132,10 +138,10 @@ pre_final AS (
         A.inner_index_end,
         C.succeeded,
         A.swapper,
-        b.amount AS from_amt,
-        b.mint AS from_mint,
-        C.amount AS to_amt,
-        C.mint AS to_mint,
+        c.amount AS from_amt,
+        c.mint AS from_mint,
+        b.amount AS to_amt,
+        b.mint AS to_mint,
         A._inserted_timestamp
     FROM
         decoded A
@@ -144,13 +150,13 @@ pre_final AS (
         AND A.source_token_account = b.source_token_account
         AND A.program_source_token_account = b.dest_token_account
         AND A.index = b.index_1
-        AND b.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end
+        AND b.inner_index_1 BETWEEN A.inner_index_start AND A.inner_index_end
         INNER JOIN transfers C
         ON A.tx_id = C.tx_id
         AND A.destination_token_account = C.dest_token_account
         AND A.program_destination_token_account = C.source_token_account
         AND A.index = C.index_1
-        AND C.inner_index_1 BETWEEN A.inner_index  AND A.inner_index_end
+        AND C.inner_index_1 BETWEEN A.inner_index_start  AND A.inner_index_end
         qualify(ROW_NUMBER() over (PARTITION BY A.tx_id, A.index, A.inner_INDEX
     ORDER BY
         inner_index)) = 1

models/silver/swaps/silver__swaps_intermediate_phoenix.yml

@@ -1,6 +1,7 @@
 version: 2
 models:
   - name: silver__swaps_intermediate_phoenix
+    description: This table contains swaps performed on Phoenix Trade. We are currently not capturing swaps where there are separate transactions for placing the order and filling the order. 
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
@@ -23,7 +24,8 @@ models:
           '671GWKC1AGyvrAThDzM9tq9UqVW8W2WdULAikkLaaTSZVYEmzLEhXutdDjZTenLp5oAtc4G7f4bztZ978rN1Zz1d',
           '3FgxCNCndXDSx95mrsBwcfsHSM9fDzWSq2bJqN74ym5MTgF2VrLJRn4w5UfdKpeKNA3mzCmm7KJfy7wuDAo4y7Ko',
           '21qdi1d8vYyMp3AcJmPdKBi1VsEoaBn4qhB1WJhzRR9N3qvtXRkENh5caaBGDaeUVD68CUw72bD2neCQA7dmgUuS',
-          '3EweEGRQkdQ4e5sRRcU5V9Jz2E9sMHzDrbmSD13C2wLQJkANJEZkk9EHJqpUSKUSWCAnY5nFnLgXUqJF5ham6ARk')"  
+          '3EweEGRQkdQ4e5sRRcU5V9Jz2E9sMHzDrbmSD13C2wLQJkANJEZkk9EHJqpUSKUSWCAnY5nFnLgXUqJF5ham6ARk',
+          '5jP7nmkNNGnjKeqP82HRr9WNv4fBdU1dHCpNGp5bSMVNeHfs7wf1uCtifPRvvugt6gX4ESUdZHQXnhDbx9fy5sfA')"  
       - tx_excluded:
           name: silver__swaps_intermediate_phoenix_tx_excluded_test
           excluded_tx_ids: