From fbf2d0502e346bf9d99acd7ea4c3ee5b844e1306 Mon Sep 17 00:00:00 2001 From: tarikceric Date: Thu, 18 Sep 2025 09:07:09 -0700 Subject: [PATCH] update per v1 deletion --- ...t_run_decode_instructions_orchestrator.yml | 2 +- models/silver/core/silver__blocks.sql | 74 +++++++------- models/silver/core/silver__transactions.sql | 98 +++++++++---------- 3 files changed, 87 insertions(+), 87 deletions(-) diff --git a/.github/workflows/dbt_run_decode_instructions_orchestrator.yml b/.github/workflows/dbt_run_decode_instructions_orchestrator.yml index a83ba248..e656329d 100644 --- a/.github/workflows/dbt_run_decode_instructions_orchestrator.yml +++ b/.github/workflows/dbt_run_decode_instructions_orchestrator.yml @@ -53,7 +53,7 @@ jobs: - name: Run DBT Jobs run: | - dbt run -s streamline__complete_decoded_instructions_2 streamline__complete_decoded_instructions_3 + dbt run -s streamline__complete_decoded_instructions_3 - name: Run Real Time Core on minutes 0 and 30 every hour run: | diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index 0c566be9..e4c6cc95 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -21,19 +21,19 @@ {% if is_incremental() %} {% set get_dates_to_load_query %} WITH base_blocks AS ( - SELECT - block_id, - value, - data, - error, - _inserted_date, - _inserted_timestamp - FROM - {{ ref('bronze__blocks2') }} - WHERE - _inserted_date >= '{{max_inserted_timestamp}}'::DATE - INTERVAL '1 DAY' - AND block_id < {{cutover_block_id}} - UNION ALL + -- SELECT + -- block_id, + -- value, + -- data, + -- error, + -- _inserted_date, + -- _inserted_timestamp + -- FROM + -- {{ ref('bronze__blocks2') }} + -- WHERE + -- _inserted_date >= '{{max_inserted_timestamp}}'::DATE - INTERVAL '1 DAY' + -- AND block_id < {{cutover_block_id}} + -- UNION ALL SELECT block_id, value, @@ -69,30 +69,30 @@ This CTE (pre_final) combines data from two sources: 2. New data using streamline 2.0 raw data from 'bronze__streamline_blocks_2' for blocks after and including the cutover_block_id */ WITH pre_final AS ( - SELECT - value:block_id::INTEGER AS block_id, - to_timestamp_ntz(data:blockTime) AS block_timestamp, - 'mainnet' AS network, - 'solana' AS chain_id, - data:blockHeight AS block_height, - data:blockhash::STRING AS block_hash, - data:parentSlot AS previous_block_id, - data:previousBlockhash::STRING AS previous_block_hash, - _inserted_date, - _inserted_timestamp - FROM - {{ ref('bronze__blocks2') }} - WHERE - block_id < {{cutover_block_id}} - AND block_id IS NOT NULL - AND error IS NULL - {% if is_incremental() %} - AND _inserted_date = '{{ load_date }}' - AND _inserted_timestamp >= '{{ load_timestamp }}' - {% else %} - AND _inserted_date = '2022-08-12' - {% endif %} - UNION ALL +-- SELECT +-- value:block_id::INTEGER AS block_id, +-- to_timestamp_ntz(data:blockTime) AS block_timestamp, +-- 'mainnet' AS network, +-- 'solana' AS chain_id, +-- data:blockHeight AS block_height, +-- data:blockhash::STRING AS block_hash, +-- data:parentSlot AS previous_block_id, +-- data:previousBlockhash::STRING AS previous_block_hash, +-- _inserted_date, +-- _inserted_timestamp +-- FROM +-- {{ ref('bronze__blocks2') }} +-- WHERE +-- block_id < {{cutover_block_id}} +-- AND block_id IS NOT NULL +-- AND error IS NULL +-- {% if is_incremental() %} +-- AND _inserted_date = '{{ load_date }}' +-- AND _inserted_timestamp >= '{{ load_timestamp }}' +-- {% else %} +-- AND _inserted_date = '2022-08-12' +-- {% endif %} +-- UNION ALL SELECT block_id, to_timestamp_ntz(data:result:blockTime) AS block_timestamp, diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index 99ac7a5d..bd8b207a 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -41,55 +41,55 @@ WITH base AS ( AND t._partition_id >= {{ cutover_partition_id }} ), pre_final AS ( - SELECT - COALESCE(TO_TIMESTAMP_NTZ(t.value :block_time), b.block_timestamp) AS block_timestamp, - t.block_id, - t.tx_id, - t.data :transaction :message :recentBlockhash :: STRING AS recent_block_hash, - t.data :meta :fee :: NUMBER AS fee, - CASE - WHEN IS_NULL_VALUE( - t.data :meta :err - ) THEN TRUE - ELSE FALSE - END AS succeeded, - t.data :transaction :message :accountKeys :: ARRAY AS account_keys, - t.data :meta :preBalances :: ARRAY AS pre_balances, - t.data :meta :postBalances :: ARRAY AS post_balances, - t.data :meta :preTokenBalances :: ARRAY AS pre_token_balances, - t.data :meta :postTokenBalances :: ARRAY AS post_token_balances, - t.data :transaction :message :instructions :: ARRAY AS instructions, - t.data :meta :innerInstructions :: ARRAY AS inner_instructions, - t.data :meta :logMessages :: ARRAY AS log_messages, - t.data:transaction:message:addressTableLookups::array as address_table_lookups, - t.data :meta :computeUnitsConsumed :: NUMBER as compute_units_consumed, - t.data :version :: STRING as version, - NULL AS tx_index, - t._partition_id, - t._inserted_timestamp - FROM - {{ ref('bronze__transactions2') }} AS t - LEFT OUTER JOIN {{ ref('silver__blocks') }} AS b - ON b.block_id = t.block_id - WHERE - t.block_id < {{cutover_block_id}} - AND tx_id IS NOT NULL - AND ( - COALESCE(t.data :transaction :message :instructions [0] :programId :: STRING,'') <> 'Vote111111111111111111111111111111111111111' - OR - ( - array_size(t.data :transaction :message :instructions) > 1 - ) - ) - {% if is_incremental() %} - AND _partition_id >= (SELECT max(_partition_id)-1 FROM {{ this }}) - AND _partition_id <= (SELECT max(_partition_id) FROM {{ source('solana_streamline','complete_block_txs') }}) - AND t._inserted_timestamp > (SELECT max(_inserted_timestamp) FROM {{ this }}) - {% else %} - AND _partition_id IN (1,2) - {% endif %} - AND _partition_id < {{cutover_partition_id}} - UNION ALL + -- SELECT + -- COALESCE(TO_TIMESTAMP_NTZ(t.value :block_time), b.block_timestamp) AS block_timestamp, + -- t.block_id, + -- t.tx_id, + -- t.data :transaction :message :recentBlockhash :: STRING AS recent_block_hash, + -- t.data :meta :fee :: NUMBER AS fee, + -- CASE + -- WHEN IS_NULL_VALUE( + -- t.data :meta :err + -- ) THEN TRUE + -- ELSE FALSE + -- END AS succeeded, + -- t.data :transaction :message :accountKeys :: ARRAY AS account_keys, + -- t.data :meta :preBalances :: ARRAY AS pre_balances, + -- t.data :meta :postBalances :: ARRAY AS post_balances, + -- t.data :meta :preTokenBalances :: ARRAY AS pre_token_balances, + -- t.data :meta :postTokenBalances :: ARRAY AS post_token_balances, + -- t.data :transaction :message :instructions :: ARRAY AS instructions, + -- t.data :meta :innerInstructions :: ARRAY AS inner_instructions, + -- t.data :meta :logMessages :: ARRAY AS log_messages, + -- t.data:transaction:message:addressTableLookups::array as address_table_lookups, + -- t.data :meta :computeUnitsConsumed :: NUMBER as compute_units_consumed, + -- t.data :version :: STRING as version, + -- NULL AS tx_index, + -- t._partition_id, + -- t._inserted_timestamp + -- FROM + -- {{ ref('bronze__transactions2') }} AS t + -- LEFT OUTER JOIN {{ ref('silver__blocks') }} AS b + -- ON b.block_id = t.block_id + -- WHERE + -- t.block_id < {{cutover_block_id}} + -- AND tx_id IS NOT NULL + -- AND ( + -- COALESCE(t.data :transaction :message :instructions [0] :programId :: STRING,'') <> 'Vote111111111111111111111111111111111111111' + -- OR + -- ( + -- array_size(t.data :transaction :message :instructions) > 1 + -- ) + -- ) + -- {% if is_incremental() %} + -- AND _partition_id >= (SELECT max(_partition_id)-1 FROM {{ this }}) + -- AND _partition_id <= (SELECT max(_partition_id) FROM {{ source('solana_streamline','complete_block_txs') }}) + -- AND t._inserted_timestamp > (SELECT max(_inserted_timestamp) FROM {{ this }}) + -- {% else %} + -- AND _partition_id IN (1,2) + -- {% endif %} + -- AND _partition_id < {{cutover_partition_id}} + -- UNION ALL SELECT t.block_timestamp, t.block_id,