hourly schedule for ez dex swaps and upstream (#815)

* hourly schedule for ez dex swaps and upstream

* hourly schedule for ez dex swaps and upstream

* cleanup

.github/workflows/dbt_run_incremental_non_core_hourly.yml

@@ -0,0 +1,42 @@
+name: dbt_run_scheduled_non_core_hourly
+run-name: dbt_run_scheduled_non_core_hourly
+
+on:
+  workflow_dispatch:
+    
+env:
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "${{ vars.PYTHON_VERSION }}"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -s "solana_models,tag:scheduled_non_core_hourly"

data/github_actions__workflows.csv

@@ -3,6 +3,7 @@ dbt_run_decode_instructions_orchestrator,"*/5 * * * *"
 dbt_run_decode_logs_orchestrator,"*/15 * * * *"
 dbt_run_incremental,"16,46 1-3,5-7,9-11,13-15,17-19,21-23 * * *"
 dbt_run_incremental_non_core,"1 */4 * * *"
+dbt_run_incremental_non_core_hourly,"1 1-3,5-7,9-11,13-15,17-19,21-23 * * *"
 dbt_run_nft_compressed_mints_realtime,"*/15 * * * *"
 dbt_run_nft_compressed_mints_sales_realtime,"55 * * * *"
 dbt_run_streamline_blocks,"*/5 * * * *"

models/gold/defi/defi__ez_dex_swaps.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'view',
     meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }} },
-    tags = ['scheduled_non_core','exclude_change_tracking']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly','exclude_change_tracking']
 ) }}
 
 WITH swaps AS (

models/gold/defi/defi__fact_swaps.sql

@@ -6,7 +6,7 @@
     cluster_by = ['block_timestamp::DATE','swap_program'],
     merge_exclude_columns = ["inserted_timestamp"],
     post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id, swapper, swap_from_mint, swap_to_mint, program_id, fact_swaps_id)'),
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 {% if execute %}

models/silver/labels/silver__labels.sql

@@ -5,7 +5,7 @@
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = 'modified_timestamp::DATE',
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(address); DELETE FROM {{ this }} WHERE _is_deleted = TRUE;",
-    tags=['scheduled_non_core'],
+    tags=['scheduled_non_core','scheduled_non_core_hourly'],
 ) }}
 
 SELECT

models/silver/metadata/silver__decoded_metadata.sql

@@ -6,7 +6,7 @@
     incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 {% if execute %}

models/silver/non_core/silver__mint_actions.sql

@@ -4,7 +4,7 @@
     incremental_strategy = 'delete+insert',
     incremental_predicates = ['block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 WITH base_events AS (

models/silver/parser/silver__decoded_instructions.sql

@@ -8,7 +8,7 @@
     unique_key = "decoded_instructions_id",
     cluster_by = ['block_timestamp::DATE', '_inserted_timestamp::DATE', 'program_id'],
     merge_exclude_columns = ["inserted_timestamp"],
-    tags = ['scheduled_non_core'],
+    tags = ['scheduled_non_core','scheduled_non_core_hourly'],
     full_refresh = false,
 ) }}
 

models/silver/parser/silver__decoded_instructions_combined.sql

@@ -9,7 +9,7 @@
         'ON EQUALITY(tx_id, event_type, decoded_instructions_combined_id)'
     ),
     merge_exclude_columns = ["inserted_timestamp"],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 -- depends_on: {{ ref('silver__decoded_instructions') }}
 /* run incremental timestamp value first then use it as a static value */

models/silver/parser/silver__decoded_logs.sql

@@ -16,7 +16,7 @@
         'ON EQUALITY(tx_id, event_type, decoded_logs_id)'
     ),
     full_refresh = false,
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 {% set CUTOVER_DATETIME = modules.datetime.datetime.strptime("2024-07-16 17:00:00", "%Y-%m-%d %H:%M:%S") %}

models/silver/prices/silver__complete_native_prices.sql

@@ -3,7 +3,7 @@
     incremental_strategy = 'delete+insert',
     unique_key = 'complete_native_prices_id',
     cluster_by = ['HOUR::DATE'],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 SELECT

models/silver/prices/silver__complete_token_prices.sql

@@ -4,7 +4,7 @@
     unique_key = 'complete_token_prices_id',
     cluster_by = ['HOUR::DATE'],
     post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(token_address,symbol)'),
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 SELECT

models/silver/swaps/bonkswap/silver__swaps_intermediate_bonkswap.sql

@@ -6,7 +6,7 @@
     incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 {% if execute %}

models/silver/swaps/dooar/silver__swaps_intermediate_dooar.sql

@@ -5,7 +5,7 @@
     incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 {% if execute %}

models/silver/swaps/generic/silver__swaps_intermediate_generic.sql

@@ -8,7 +8,7 @@
         '{{this.identifier}}',
         'ON EQUALITY(tx_id, program_id, swapper, from_mint, to_mint)'
     ),
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 WITH base_events AS(

models/silver/swaps/meteora/silver__swaps_intermediate_meteora.sql

@@ -6,7 +6,7 @@
     incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 {% if execute %}

models/silver/swaps/orca/silver__swaps_intermediate_orca.sql

@@ -8,7 +8,7 @@
         '{{this.identifier}}',
         'ON EQUALITY(tx_id, swapper, from_mint, to_mint)'
     ),
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 WITH base_events AS(

models/silver/swaps/phoenix/silver__swaps_intermediate_phoenix.sql

@@ -6,7 +6,7 @@
     incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
-    tags = ['scheduled_non_core'],
+    tags = ['scheduled_non_core','scheduled_non_core_hourly'],
 ) }}
 
 {% if execute %}

models/silver/swaps/pumpfun/silver__swaps_pumpfun.sql

@@ -11,7 +11,7 @@
         '{{this.identifier}}',
         'ON EQUALITY(tx_id, swapper, from_mint, to_mint)'
     ),
-    tags = ['scheduled_non_core'],
+    tags = ['scheduled_non_core','scheduled_non_core_hourly'],
 ) }}
 
 {% if execute %}

models/silver/swaps/raydium/silver__swaps_intermediate_raydium_clmm.sql

@@ -6,7 +6,7 @@
     incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 {% if execute %}

models/silver/swaps/raydium/silver__swaps_intermediate_raydium_cpmm.sql

@@ -6,7 +6,7 @@
     incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 {% if execute %}

models/silver/swaps/raydium/silver__swaps_intermediate_raydium_stable.sql

@@ -5,7 +5,7 @@
     incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 {% if execute %}

models/silver/swaps/raydium/silver__swaps_intermediate_raydium_v4_amm.sql

@@ -5,7 +5,7 @@
     incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 {% if execute %}

models/silver/swaps/silver__swaps.sql

@@ -4,7 +4,7 @@
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
     merge_exclude_columns = ["inserted_timestamp"],
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core','scheduled_non_core_hourly']
 ) }}
 
 WITH base AS (