diff --git a/macros/create_sps.sql b/macros/create_sps.sql index 7de0232c..0284134d 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -5,4 +5,10 @@ {{ sp_create_prod_clone('_internal') }}; {% endif %} {% endif %} +{% endmacro %} + +{% macro enable_search_optimization(schema_name, table_name, condition = '') %} + {% if target.database == 'SOLANA' %} + ALTER TABLE {{ schema_name }}.{{ table_name }} ADD SEARCH OPTIMIZATION {{ condition }} + {% endif %} {% endmacro %} \ No newline at end of file diff --git a/macros/sp_create_prod_clone.sql b/macros/sp_create_prod_clone.sql index 6d79bdee..f0988221 100644 --- a/macros/sp_create_prod_clone.sql +++ b/macros/sp_create_prod_clone.sql @@ -12,6 +12,10 @@ $$ snowflake.execute({sqlText: `DROP SCHEMA IF EXISTS ${DESTINATION_DB_NAME}._INTERNAL`}); /* this only needs to be in prod */ snowflake.execute({sqlText: `GRANT USAGE ON DATABASE ${DESTINATION_DB_NAME} TO AWS_LAMBDA_SOLANA_API`}); + snowflake.execute({sqlText: `ALTER TABLE ${DESTINATION_DB_NAME}.SILVER.TRANSACTIONS DROP SEARCH OPTIMIZATION;`}); + snowflake.execute({sqlText: `ALTER TABLE ${DESTINATION_DB_NAME}.SILVER.VOTES DROP SEARCH OPTIMIZATION;`}); + snowflake.execute({sqlText: `ALTER TABLE ${DESTINATION_DB_NAME}.SILVER.EVENTS DROP SEARCH OPTIMIZATION;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL SCHEMAS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); @@ -27,6 +31,8 @@ $$ snowflake.execute({sqlText: `GRANT OWNERSHIP ON DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}) + + snowflake.execute({sqlText: `COMMIT;`}); } catch (err) { snowflake.execute({sqlText: `ROLLBACK;`}); diff --git a/models/silver/accounts/silver__token_account_owners.sql b/models/silver/accounts/silver__token_account_owners.sql index 490472c6..99eb56c6 100644 --- a/models/silver/accounts/silver__token_account_owners.sql +++ b/models/silver/accounts/silver__token_account_owners.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = ["account_address"], cluster_by = ['_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}'), full_refresh = false, enabled = false, ) }} diff --git a/models/silver/parser/silver__decoded_instructions.sql b/models/silver/parser/silver__decoded_instructions.sql index 93818c62..3f7f063d 100644 --- a/models/silver/parser/silver__decoded_instructions.sql +++ b/models/silver/parser/silver__decoded_instructions.sql @@ -4,7 +4,7 @@ materialized = 'incremental', unique_key = ["tx_id", "index" ], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE','program_id'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}'), merge_exclude_columns = ["inserted_timestamp"], tags = ['scheduled_non_core'] ) }} diff --git a/models/silver/rewards/silver__rewards_fee.sql b/models/silver/rewards/silver__rewards_fee.sql index b19e7bf2..30cb6151 100644 --- a/models/silver/rewards/silver__rewards_fee.sql +++ b/models/silver/rewards/silver__rewards_fee.sql @@ -3,7 +3,7 @@ unique_key = ["vote_pubkey","epoch_earned","block_id"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','floor(block_id,-6)','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(vote_pubkey, epoch_earned);", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(vote_pubkey, epoch_earned)'), tags = ['rewards', 'scheduled_non_core'] ) }} diff --git a/models/silver/rewards/silver__rewards_rent.sql b/models/silver/rewards/silver__rewards_rent.sql index 32998423..1e59e8a0 100644 --- a/models/silver/rewards/silver__rewards_rent.sql +++ b/models/silver/rewards/silver__rewards_rent.sql @@ -3,7 +3,7 @@ unique_key = ["vote_pubkey","epoch_earned","block_id"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','floor(block_id,-6)','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(vote_pubkey, epoch_earned);", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(vote_pubkey, epoch_earned)'), tags = ['rewards'], full_refresh = false, enabled = false diff --git a/models/silver/rewards/silver__rewards_staking.sql b/models/silver/rewards/silver__rewards_staking.sql index a625bbeb..7766fc4b 100644 --- a/models/silver/rewards/silver__rewards_staking.sql +++ b/models/silver/rewards/silver__rewards_staking.sql @@ -3,7 +3,7 @@ unique_key = ["stake_pubkey","epoch_earned","block_id"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','floor(block_id,-6)','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(stake_pubkey, epoch_earned);", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(stake_pubkey, epoch_earned)'), tags = ['rewards', 'scheduled_non_core'] ) }} diff --git a/models/silver/rewards/silver__rewards_voting.sql b/models/silver/rewards/silver__rewards_voting.sql index 7d444cfc..24c300ea 100644 --- a/models/silver/rewards/silver__rewards_voting.sql +++ b/models/silver/rewards/silver__rewards_voting.sql @@ -3,7 +3,7 @@ unique_key = ["vote_pubkey","epoch_earned","block_id"], merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','floor(block_id,-6)','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(vote_pubkey, epoch_earned);", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(vote_pubkey, epoch_earned)'), tags = ['rewards', 'scheduled_non_core'] ) }} diff --git a/models/silver/silver__events.sql b/models/silver/silver__events.sql index bb828ff4..02b230c2 100644 --- a/models/silver/silver__events.sql +++ b/models/silver/silver__events.sql @@ -3,7 +3,7 @@ unique_key = ['block_id','tx_id','index'], incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE','program_id'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}'), full_refresh = false, merge_exclude_columns = ["inserted_timestamp"], tags = ['scheduled_core'] diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql index 75efec14..dc54c628 100644 --- a/models/silver/silver__transactions.sql +++ b/models/silver/silver__transactions.sql @@ -3,7 +3,7 @@ unique_key = "tx_id", incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], cluster_by = ['block_timestamp::DATE','block_id','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}'), full_refresh = false, merge_exclude_columns = ["inserted_timestamp"], tags = ['scheduled_core'] diff --git a/models/silver/silver__transfers.sql b/models/silver/silver__transfers.sql index 6fc514c8..0dbbf101 100644 --- a/models/silver/silver__transfers.sql +++ b/models/silver/silver__transfers.sql @@ -3,7 +3,7 @@ unique_key = ["block_id","tx_id","index"], incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}'), full_refresh = false, merge_exclude_columns = ["inserted_timestamp"], tags = ['scheduled_core'] diff --git a/models/silver/silver__votes.sql b/models/silver/silver__votes.sql index 89530711..58b9a83c 100644 --- a/models/silver/silver__votes.sql +++ b/models/silver/silver__votes.sql @@ -3,7 +3,7 @@ unique_key = "tx_id", incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], cluster_by = ['block_timestamp::DATE','block_id','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}'), full_refresh = false, tags = ['scheduled_non_core'] ) }} diff --git a/models/silver/swaps/silver__swaps_intermediate_generic.sql b/models/silver/swaps/silver__swaps_intermediate_generic.sql index 90672069..0218a950 100644 --- a/models/silver/swaps/silver__swaps_intermediate_generic.sql +++ b/models/silver/swaps/silver__swaps_intermediate_generic.sql @@ -3,7 +3,7 @@ unique_key = ["block_id","tx_id","swap_index"], incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}'), tags = ['scheduled_non_core'] ) }} diff --git a/models/silver/swaps/silver__swaps_intermediate_jupiterv4.sql b/models/silver/swaps/silver__swaps_intermediate_jupiterv4.sql index e74f8c86..4c45092c 100644 --- a/models/silver/swaps/silver__swaps_intermediate_jupiterv4.sql +++ b/models/silver/swaps/silver__swaps_intermediate_jupiterv4.sql @@ -3,7 +3,7 @@ unique_key = ["block_id","tx_id","swap_index"], incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}'), tags = ['scheduled_non_core'] ) }} diff --git a/models/silver/swaps/silver__swaps_intermediate_orca.sql b/models/silver/swaps/silver__swaps_intermediate_orca.sql index 1d987b38..b98e4f57 100644 --- a/models/silver/swaps/silver__swaps_intermediate_orca.sql +++ b/models/silver/swaps/silver__swaps_intermediate_orca.sql @@ -3,7 +3,7 @@ unique_key = ["block_id","tx_id","swap_index"], incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}'), tags = ['scheduled_non_core'] ) }} diff --git a/models/streamline/parser/streamline__complete_decoded_instructions.sql b/models/streamline/parser/streamline__complete_decoded_instructions.sql index 2274635a..95fd95bf 100644 --- a/models/streamline/parser/streamline__complete_decoded_instructions.sql +++ b/models/streamline/parser/streamline__complete_decoded_instructions.sql @@ -3,7 +3,7 @@ materialized = "incremental", unique_key = "id", cluster_by = "ROUND(block_id, -3)", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)", + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(id)'), tags = ['streamline'], ) }}