add tx_index for data sourced from streamline 2.0 (#738)

* add tx_index for data sourced from streamline 2.0

* update tests and descriptions

models/descriptions/tx_index.md

@@ -0,0 +1,5 @@
+{% docs tx_index %}
+
+The index of the transaction in the block. Index of 0 is the first transaction executed in the block.
+
+{% enddocs %}

models/silver/core/silver__transactions.sql

@@ -39,6 +39,7 @@ WITH pre_final AS (
         t.data:transaction:message:addressTableLookups::array as address_table_lookups,
         t.data :meta :computeUnitsConsumed :: NUMBER as compute_units_consumed,
         t.data :version :: STRING as version,
+        NULL AS tx_index,
         t._partition_id,
         t._inserted_timestamp
     FROM
@@ -87,6 +88,7 @@ WITH pre_final AS (
         t.data:transaction:message:addressTableLookups::array as address_table_lookups,
         t.data :meta :computeUnitsConsumed :: NUMBER as compute_units_consumed,
         t.data :version :: STRING as version,
+        t.value:array_index::int AS tx_index,
         t._partition_id,
         t._inserted_timestamp
     FROM
@@ -133,6 +135,7 @@ prev_null_block_timestamp_txs AS (
         t.units_limit,
         t.tx_size,
         t.version,
+        t.tx_index,
         t._partition_id,
         GREATEST(
             t._inserted_timestamp,
@@ -200,6 +203,7 @@ SELECT
     silver.udf_get_compute_units_total(log_messages, instructions) as units_limit,
     silver.udf_get_tx_size(account_keys,instructions,version,address_table_lookups,signers) as tx_size,
     version,
+    tx_index,
     _partition_id,
     _inserted_timestamp,
     {{ dbt_utils.generate_surrogate_key(

models/silver/core/silver__transactions.yml

@@ -1,7 +1,10 @@
 version: 2
 models:
   - name: silver__transactions
-    tests:
+    recent_date_filter: &recent_date_filter
+      config:
+        where: _inserted_timestamp >= current_date - 7
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - TX_ID
@@ -9,38 +12,36 @@ models:
     columns:
       - name: BLOCK_TIMESTAMP
         description: "{{ doc('block_timestamp') }}"
-        tests:
-          - not_null:
-              where: block_id > 39824213 and _inserted_timestamp::date < current_date
+        data_tests:
+          - not_null: *recent_date_filter
       - name: BLOCK_ID
         description: "{{ doc('block_id') }}"
-        tests:
-          - not_null
+        data_tests:
+          - not_null: *recent_date_filter
       - name: RECENT_BLOCK_HASH
         description: Previous block's hash value
-        tests:
-          - not_null
+        data_tests:
+          - not_null: *recent_date_filter
       - name: TX_ID
         description: "{{ doc('tx_id') }}"
-        tests:
-          - not_null
+        data_tests:
+          - not_null: *recent_date_filter
       - name: FEE
         description: Transaction fee (in lamports)
-        tests: 
+        data_tests: 
           - null_threshold:
               threshold_percent: 0.99 # some older transactions have valid null fees
               where: block_timestamp::date > current_date - 30
       - name: SUCCEEDED
         description: "{{ doc('tx_succeeded') }}"
-        tests: 
-          - not_null
+        data_tests: 
+          - not_null: *recent_date_filter
       - name: SIGNERS
         description: List of accounts that signed the transaction
       - name: ACCOUNT_KEYS
         description: List of accounts that are referenced by pre/post sol/token balances objects
-        tests: 
-          - not_null:
-              where: block_timestamp::date > current_date - 30
+        data_tests: 
+          - not_null: *recent_date_filter
       - name: PRE_BALANCES
         description: List of pre-transaction balances for different accounts
       - name: POST_BALANCES
@@ -51,14 +52,12 @@ models:
         description: List of post-transaction token balances for different token accounts
       - name: INSTRUCTIONS
         description: "{{ doc('instruction') }}"
-        tests: 
-          - not_null:
-              where: block_timestamp::date > current_date - 30
+        data_tests: 
+          - not_null: *recent_date_filter
       - name: INNER_INSTRUCTIONS
         description: "{{ doc('inner_instruction') }}"
-        tests: 
-          - not_null:
-              where: block_timestamp::date > current_date - 30
+        data_tests: 
+          - not_null: *recent_date_filter
       - name: LOG_MESSAGES
         description: Array of log messages written by the program for this transaction
       - name: ADDRESS_TABLE_LOOKUPS
@@ -71,7 +70,16 @@ models:
         description: The size of the transaction in bytes. 
       - name: VERSION
         description: Transaction version, legacy version is listed as NULL or 'legacy'
+      - name: TX_INDEX
+        description: "{{ doc('tx_index') }}"
+        # TODO: update once we know when the first partition is that has tx_index values
+        # data_tests:
+        #   - not_null:
+        #       config:
+        #         where: >
+        #           _inserted_timestamp >= current_date - 7
+        #           AND _partition_id >= 150300
       - name: _INSERTED_TIMESTAMP
         description: "{{ doc('_inserted_timestamp') }}"
-        tests: 
+        data_tests: 
           - not_null