AN-4096/DEX Swaps Bug (#228)

* add event counts col to easily ignore infra events

* excl txs where to/from is null

* add tests

models/descriptions/event_count.md

@@ -0,0 +1,5 @@
+{% docs event_count %}
+
+Number of events in a transaction.
+
+{% enddocs %}

models/gold/defi/defi__ez_swaps.sql

@@ -68,3 +68,5 @@ SELECT
     *
 FROM
     FINAL
+WHERE
+    token_in_destination IS NOT NULL

models/gold/defi/defi__ez_swaps.yml

@@ -8,36 +8,60 @@ models:
     columns:
       - name: TX_ID
         description: "{{ doc('tx_id') }}"
+        tests:
+          - not_null
 
       - name: BLOCK_TIMESTAMP
         description: "{{ doc('block_timestamp') }}"
+        tests:
+          - not_null
 
       - name: BLOCK_HEIGHT
         description: "{{ doc('block_height') }}"
+        tests:
+          - not_null
 
       - name: SWAP_CONTRACT
         description: "{{ doc('swap_contract') }}"
+        tests:
+          - not_null
 
       - name: SWAP_INDEX
         description: "{{ doc('swap_index') }}"
+        tests:
+          - not_null
 
       - name: TRADER
         description: "{{ doc('trader') }}"
+        tests:
+          - not_null
 
       - name: TOKEN_OUT_SOURCE
         description: "{{ doc('token_out_source') }}"
+        tests:
+          - not_null
 
       - name: TOKEN_OUT_AMOUNT
         description: "{{ doc('token_out_amount') }}"
+        tests:
+          - not_null
 
       - name: TOKEN_OUT_CONTRACT
         description: "{{ doc('token_out_contract') }}"
+        tests:
+          - not_null
 
       - name: TOKEN_IN_DESTINATION
         description: "{{ doc('token_in_destination') }}"
+        tests:
+          - not_null
 
       - name: TOKEN_IN_AMOUNT
         description: "{{ doc('token_in_amount') }}"
+        tests:
+          - not_null
 
       - name: TOKEN_IN_CONTRACT
         description: "{{ doc('token_in_contract') }}"
+        tests:
+          - not_null

models/silver/core/silver__streamline_events.sql

@@ -29,6 +29,7 @@ flatten_events AS (
         block_timestamp,
         tx_id,
         tx_succeeded,
+        events_count,
         VALUE :: variant AS event_data_full,
         VALUE :event_index :: INT AS event_index,
         concat_ws(
@@ -105,6 +106,7 @@ FINAL AS (
         e.block_timestamp,
         e.event_id,
         e.event_index,
+        e.events_count,
         e.payload,
         e.event_contract,
         e.event_type,

models/silver/core/silver__streamline_events.yml

@@ -45,6 +45,9 @@ models:
           - dbt_expectations.expect_column_values_to_be_of_type:
               column_type: NUMBER
 
+      - name: EVENT_COUNT
+        description: "{{ doc('event_count') }}"
+
       - name: payload
         description: "{{ doc('payload') }}"
         tests:

models/silver/core/silver__streamline_transactions_final.sql

@@ -129,6 +129,7 @@ FINAL AS (
         t.script,
         tr.error_message,
         tr.events,
+        ARRAY_SIZE(tr.events) AS events_count,
         tr.status,
         tr.status_code,
         GREATEST(
@@ -159,6 +160,7 @@ SELECT
     proposer,
     script,
     events,
+    events_count,
     status,
     status_code,
     error_message,

models/silver/core/silver__streamline_transactions_final.yml

@@ -89,6 +89,9 @@ models:
           - dbt_expectations.expect_column_values_to_be_of_type:
               column_type: ARRAY
 
+      - name: EVENT_COUNT
+        description: "{{ doc('event_count') }}"
+
       - name: status
         description: "{{ doc('status') }}"
         tests:

models/silver/swaps/silver__swaps_events_s.sql

@@ -6,27 +6,14 @@
   tags = ['scheduled', 'streamline_scheduled', 'scheduled_non_core']
 ) }}
 
-WITH swap_contracts AS (
+WITH swaps_txs AS (
 
-  SELECT
-    *
-  FROM
-    {{ ref('silver__contract_labels') }}
-  WHERE
-    contract_name LIKE '%SwapPair%'
-),
-swaps_txs AS (
   SELECT
     *
   FROM
     {{ ref('silver__streamline_events') }}
   WHERE
-    event_contract IN (
-      SELECT
-        event_contract
-      FROM
-        swap_contracts
-    )
+    event_contract LIKE '%SwapPair%'
 
 {% if is_incremental() %}
 AND _inserted_timestamp >= (
@@ -48,7 +35,9 @@ swap_events AS (
         tx_id
       FROM
         swaps_txs
-    )
+    ) 
+    -- exclude infra events, always final 3
+    AND event_index < events_count - 3
 
 {% if is_incremental() %}
 AND _inserted_timestamp >= (

models/silver/swaps/silver__swaps_s.sql

@@ -123,6 +123,7 @@ token_withdraws AS (
             FROM
                 pool_info
         )
+        AND event_data :from :: STRING != 'null'
 ),
 token_deposits AS (
     SELECT
@@ -157,6 +158,7 @@ token_deposits AS (
             FROM
                 pool_info
         )
+        AND event_data :to :: STRING != 'null'
 ),
 link_token_movement AS (
     SELECT