An 5767/deprecate old decoder and optimizations (#821)

* deprecate old decoded_instructions_data

* use events_inner and set max ts

* wip

* use events_inner

dbt_project.yml

@@ -32,7 +32,6 @@ tests:
 on-run-start:
   - '{{ create_sps() }}'
   - '{{ create_udfs() }}'
-  - '{{ sp_create_bulk_get_decoded_instructions_data() }}'
   - '{{ sp_create_snapshot_get_stake_accounts() }}'
 
 on-run-end:

macros/create_udfs.sql

@@ -2,7 +2,6 @@
     {% if var("UPDATE_UDFS_AND_SPS") %}
         {% set sql %}
         {% if target.database != "SOLANA_COMMUNITY_DEV" %}
-            {{ udf_bulk_get_decoded_instructions_data() }};
             {{ udf_snapshot_get_stake_accounts() }};
             {{ udf_bulk_program_parser() }};
             {{ udf_decode_instructions() }};

models/silver/core/silver__decoded_instructions_data.sql

@@ -3,7 +3,8 @@
     unique_key = "CONCAT_WS('-', tx_id, event_index)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['program_id'],
-    tags = ['scheduled_core']
+    full_refresh = false,
+    enabled = false,
 ) }}
 
 SELECT

models/silver/core/silver__transfers.sql

@@ -9,6 +9,18 @@
     tags = ['scheduled_core']
 ) }}
 
+{% if execute %}
+
+    {% if is_incremental() %}
+        {% set query %}
+            SELECT MAX(_inserted_timestamp) AS max_inserted_timestamp
+            FROM {{ this }}
+        {% endset %}
+
+        {% set max_inserted_timestamp = run_query(query).columns[0].values()[0] %}
+    {% endif %}
+{% endif %}
+
 WITH base_transfers_i AS (
     SELECT
         block_id,
@@ -49,12 +61,7 @@ AND
             {{ this }}
         ) 
 {% elif is_incremental() %}
-AND _inserted_timestamp >= (
-    SELECT
-        MAX(_inserted_timestamp)
-    FROM
-        {{ this }}
-    )
+AND _inserted_timestamp >= '{{ max_inserted_timestamp }}'
 {% else %}
 AND
     block_id between 4260184 and 5260184
@@ -62,26 +69,24 @@ AND
 
     UNION
     SELECT
-        e.block_id,
-        e.block_timestamp,
-        e.tx_id,
+        block_id,
+        block_timestamp,
+        tx_id,
         CONCAT(
-            e.inner_instruction :index :: NUMBER,
+            instruction_index,
             '.',
-            ii.index
+            inner_index
         ) AS INDEX,
-        ii.value :parsed :type :: STRING AS event_type,
-        ii.value :programId :: STRING AS program_id,
-        ii.value as instruction,
+        event_type,
+        program_id,
+        instruction,
         NULL AS inner_instruction,
-        e.succeeded,
+        succeeded,
         _inserted_timestamp
     FROM
-        {{ ref('silver__events') }}
-        e,
-        TABLE(FLATTEN(e.inner_instruction :instructions)) ii
+        {{ ref('silver__events_inner') }}
     WHERE
-        ii.value :parsed :type :: STRING IN (
+        event_type IN (
         'transfer',
         'transferChecked',
         'transferWithSeed',
@@ -106,12 +111,7 @@ AND
             {{ this }}
         ) 
 {% elif is_incremental() %}
-AND _inserted_timestamp >= (
-    SELECT
-        MAX(_inserted_timestamp)
-    FROM
-        {{ this }}
-    )
+AND _inserted_timestamp >= '{{ max_inserted_timestamp }}'
 {% else %}
 AND
     block_id between 4260184 and 5260184
@@ -146,13 +146,7 @@ WHERE
             {{ this }}
         ) 
 {% elif is_incremental() %}
-WHERE
-    _inserted_timestamp >= (
-        SELECT
-            MAX(_inserted_timestamp)
-        FROM
-            {{ this }}
-    )
+WHERE _inserted_timestamp >= '{{ max_inserted_timestamp }}'
 {% else %}
 WHERE
     block_id between 4260184 and 5260184
@@ -186,13 +180,7 @@ WHERE
             {{ this }}
         ) 
 {% elif is_incremental() %}
-WHERE
-    _inserted_timestamp >= (
-        SELECT
-            MAX(_inserted_timestamp)
-        FROM
-            {{ this }}
-    )
+WHERE _inserted_timestamp >= '{{ max_inserted_timestamp }}'
 {% else %}
 WHERE
     block_id between 4260184 and 5260184
@@ -222,13 +210,7 @@ base_sol_account_keys AS (
                 {{ this }}
         ) 
     {% elif is_incremental() %}
-    WHERE
-        _inserted_timestamp >= (
-            SELECT
-                MAX(_inserted_timestamp)
-            FROM
-                {{ this }}
-        )
+    WHERE _inserted_timestamp >= '{{ max_inserted_timestamp }}'
     {% else %}
     WHERE
         block_id between 4260184 and 5260184

models/silver/non_core/silver__mint_actions.sql

@@ -7,91 +7,88 @@
     tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
-WITH base_events AS (
-
+WITH prefinal as (
     SELECT
-        *
+        block_id,
+        block_timestamp,
+        tx_id,
+        succeeded,
+        index,
+        null as inner_index,
+        event_type,
+        instruction :parsed :info :mint :: STRING AS mint,
+        instruction :parsed :info :account :: STRING as token_account, 
+        instruction :parsed :info :decimals :: INTEGER AS DECIMAL,
+        COALESCE(
+            instruction :parsed :info :amount :: INTEGER,
+            instruction :parsed :info :tokenAmount: amount :: INTEGER
+        ) AS mint_amount,
+        COALESCE(
+            instruction :parsed :info :mintAuthority :: string,
+            instruction :parsed :info :multisigMintAuthority :: string,
+            instruction :parsed :info :authority :: string
+        ) AS mint_authority,
+        instruction :parsed :info :signers :: string AS signers,
+        _inserted_timestamp
     FROM
         {{ ref('silver__events') }}
     WHERE
-        1 = 1
-    {% if is_incremental() %}
-        {% if execute %}
-        {{ get_batch_load_logic(this,30,'2023-02-14') }}
+        event_type IN (
+            'mintTo',
+            'initializeMint',
+            'mintToChecked',
+            'initializeMint2',
+            'initializeConfidentialTransferMint',
+            'initializeNonTransferableMint'
+        )
+        {% if is_incremental() %}
+            {% if execute %}
+            {{ get_batch_load_logic(this,30,'2023-02-14') }}
+            {% endif %}
+        {% else %}
+            AND _inserted_timestamp::date between '2022-08-12' and '2022-09-05'
+        {% endif %}
+    UNION
+    SELECT
+        block_id,
+        block_timestamp,
+        tx_id,
+        succeeded,
+        instruction_index as index,
+        inner_index,
+        event_type,
+        instruction :parsed :info :mint :: STRING AS mint,
+        instruction :parsed :info :account :: STRING as token_account, 
+        instruction :parsed :info :decimals :: INTEGER AS DECIMAL,
+        COALESCE(
+            instruction :parsed :info :amount :: INTEGER,
+            instruction :parsed :info :tokenAmount: amount :: INTEGER
+        ) AS mint_amount,
+        COALESCE(
+            instruction :parsed :info :mintAuthority :: string,
+            instruction:parsed :info :multisigMintAuthority :: string,
+            instruction :parsed :info :authority :: string
+        ) AS mint_authority,
+        instruction :parsed :info :signers :: string AS signers,
+        _inserted_timestamp
+    FROM
+        {{ ref('silver__events_inner') }}
+    WHERE
+        instruction :parsed :type :: STRING IN (
+            'mintTo',
+            'initializeMint',
+            'mintToChecked',
+            'initializeMint2',
+            'initializeConfidentialTransferMint',
+            'initializeNonTransferableMint'
+        )
+        {% if is_incremental() %}
+            {% if execute %}
+            {{ get_batch_load_logic(this,30,'2023-02-14') }}
+            {% endif %}
+        {% else %}
+            AND _inserted_timestamp::date between '2022-08-12' and '2022-09-05'
         {% endif %}
-    {% else %}
-        AND _inserted_timestamp::date between '2022-08-12' and '2022-09-05'
-    {% endif %}
-),
-prefinal as (
-SELECT
-    block_id,
-    block_timestamp,
-    tx_id,
-    succeeded,
-    index,
-    null as inner_index,
-    event_type,
-    instruction :parsed :info :mint :: STRING AS mint,
-    instruction :parsed :info :account :: STRING as token_account, 
-    instruction :parsed :info :decimals :: INTEGER AS DECIMAL,
-    COALESCE(
-        instruction :parsed :info :amount :: INTEGER,
-        instruction :parsed :info :tokenAmount: amount :: INTEGER
-    ) AS mint_amount,
-    COALESCE(
-        instruction :parsed :info :mintAuthority :: string,
-        instruction :parsed :info :multisigMintAuthority :: string,
-        instruction :parsed :info :authority :: string
-    ) AS mint_authority,
-    instruction :parsed :info :signers :: string AS signers,
-    _inserted_timestamp
-FROM
-    base_events
-WHERE
-    event_type IN (
-        'mintTo',
-        'initializeMint',
-        'mintToChecked',
-        'initializeMint2',
-        'initializeConfidentialTransferMint',
-        'initializeNonTransferableMint'
-    )
-UNION
-SELECT
-    block_id,
-    block_timestamp,
-    tx_id,
-    succeeded,
-    e.index,
-    i.index as inner_index,
-    i.value :parsed :type :: STRING AS event_type,
-    i.value :parsed :info :mint :: STRING AS mint,
-    i.value :parsed :info :account :: STRING as token_account, 
-    i.value :parsed :info :decimals :: INTEGER AS DECIMAL,
-    COALESCE(
-        i.value :parsed :info :amount :: INTEGER,
-        i.value :parsed :info :tokenAmount: amount :: INTEGER
-    ) AS mint_amount,
-    COALESCE(
-        i.value :parsed :info :mintAuthority :: string,
-        i.value :parsed :info :multisigMintAuthority :: string,
-        i.value :parsed :info :authority :: string
-    ) AS mint_authority,
-    i.value :parsed :info :signers :: string AS signers,
-    _inserted_timestamp
-FROM
-    base_events e,
-    TABLE(FLATTEN(inner_instruction :instructions)) i
-WHERE
-    i.value :parsed :type :: STRING IN (
-        'mintTo',
-        'initializeMint',
-        'mintToChecked',
-        'initializeMint2',
-        'initializeConfidentialTransferMint',
-        'initializeNonTransferableMint'
-    )
 )
 
 SELECT