From b4a953cbf93cbb5c7a3bc66b3bde4f7bd40dc108 Mon Sep 17 00:00:00 2001 From: desmond-hui <97470747+desmond-hui@users.noreply.github.com> Date: Mon, 16 Dec 2024 07:54:50 -0800 Subject: [PATCH] add tx_index for data sourced from streamline 2.0 (#738) * add tx_index for data sourced from streamline 2.0 * update tests and descriptions --- models/descriptions/tx_index.md | 5 ++ models/silver/core/silver__transactions.sql | 4 ++ models/silver/core/silver__transactions.yml | 54 ++++++++++++--------- 3 files changed, 40 insertions(+), 23 deletions(-) create mode 100644 models/descriptions/tx_index.md diff --git a/models/descriptions/tx_index.md b/models/descriptions/tx_index.md new file mode 100644 index 00000000..b7622f0f --- /dev/null +++ b/models/descriptions/tx_index.md @@ -0,0 +1,5 @@ +{% docs tx_index %} + +The index of the transaction in the block. Index of 0 is the first transaction executed in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index a2853c53..b5db92ec 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -39,6 +39,7 @@ WITH pre_final AS ( t.data:transaction:message:addressTableLookups::array as address_table_lookups, t.data :meta :computeUnitsConsumed :: NUMBER as compute_units_consumed, t.data :version :: STRING as version, + NULL AS tx_index, t._partition_id, t._inserted_timestamp FROM @@ -87,6 +88,7 @@ WITH pre_final AS ( t.data:transaction:message:addressTableLookups::array as address_table_lookups, t.data :meta :computeUnitsConsumed :: NUMBER as compute_units_consumed, t.data :version :: STRING as version, + t.value:array_index::int AS tx_index, t._partition_id, t._inserted_timestamp FROM @@ -133,6 +135,7 @@ prev_null_block_timestamp_txs AS ( t.units_limit, t.tx_size, t.version, + t.tx_index, t._partition_id, GREATEST( t._inserted_timestamp, @@ -200,6 +203,7 @@ SELECT silver.udf_get_compute_units_total(log_messages, instructions) as units_limit, silver.udf_get_tx_size(account_keys,instructions,version,address_table_lookups,signers) as tx_size, version, + tx_index, _partition_id, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( diff --git a/models/silver/core/silver__transactions.yml b/models/silver/core/silver__transactions.yml index 5ee55aa4..4c7f4c6f 100644 --- a/models/silver/core/silver__transactions.yml +++ b/models/silver/core/silver__transactions.yml @@ -1,7 +1,10 @@ version: 2 models: - name: silver__transactions - tests: + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - TX_ID @@ -9,38 +12,36 @@ models: columns: - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp') }}" - tests: - - not_null: - where: block_id > 39824213 and _inserted_timestamp::date < current_date + data_tests: + - not_null: *recent_date_filter - name: BLOCK_ID description: "{{ doc('block_id') }}" - tests: - - not_null + data_tests: + - not_null: *recent_date_filter - name: RECENT_BLOCK_HASH description: Previous block's hash value - tests: - - not_null + data_tests: + - not_null: *recent_date_filter - name: TX_ID description: "{{ doc('tx_id') }}" - tests: - - not_null + data_tests: + - not_null: *recent_date_filter - name: FEE description: Transaction fee (in lamports) - tests: + data_tests: - null_threshold: threshold_percent: 0.99 # some older transactions have valid null fees where: block_timestamp::date > current_date - 30 - name: SUCCEEDED description: "{{ doc('tx_succeeded') }}" - tests: - - not_null + data_tests: + - not_null: *recent_date_filter - name: SIGNERS description: List of accounts that signed the transaction - name: ACCOUNT_KEYS description: List of accounts that are referenced by pre/post sol/token balances objects - tests: - - not_null: - where: block_timestamp::date > current_date - 30 + data_tests: + - not_null: *recent_date_filter - name: PRE_BALANCES description: List of pre-transaction balances for different accounts - name: POST_BALANCES @@ -51,14 +52,12 @@ models: description: List of post-transaction token balances for different token accounts - name: INSTRUCTIONS description: "{{ doc('instruction') }}" - tests: - - not_null: - where: block_timestamp::date > current_date - 30 + data_tests: + - not_null: *recent_date_filter - name: INNER_INSTRUCTIONS description: "{{ doc('inner_instruction') }}" - tests: - - not_null: - where: block_timestamp::date > current_date - 30 + data_tests: + - not_null: *recent_date_filter - name: LOG_MESSAGES description: Array of log messages written by the program for this transaction - name: ADDRESS_TABLE_LOOKUPS @@ -71,7 +70,16 @@ models: description: The size of the transaction in bytes. - name: VERSION description: Transaction version, legacy version is listed as NULL or 'legacy' + - name: TX_INDEX + description: "{{ doc('tx_index') }}" + # TODO: update once we know when the first partition is that has tx_index values + # data_tests: + # - not_null: + # config: + # where: > + # _inserted_timestamp >= current_date - 7 + # AND _partition_id >= 150300 - name: _INSERTED_TIMESTAMP description: "{{ doc('_inserted_timestamp') }}" - tests: + data_tests: - not_null