diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml index 1ddcd2ac..ee14a51f 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -19,7 +19,7 @@ env: SCHEMA: "${{ vars.SCHEMA }}" concurrency: - group: ${{ github.workflow }} + group: "dbt_run_incremental_core" jobs: run_dbt_jobs: diff --git a/.github/workflows/dbt_run_incremental_intermediate.yml b/.github/workflows/dbt_run_incremental_intermediate.yml new file mode 100644 index 00000000..208b5bf6 --- /dev/null +++ b/.github/workflows/dbt_run_incremental_intermediate.yml @@ -0,0 +1,46 @@ +name: dbt_run_scheduled_intermediate +run-name: dbt_run_scheduled_intermediate + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: "dbt_run_incremental_core" + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_refresh_external_tables_full + dbt run -s "solana_models,tag:scheduled_core" + diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 17744285..89d1549f 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -2,6 +2,7 @@ workflow_name,workflow_schedule dbt_run_decode_instructions_orchestrator,"*/5 * * * *" dbt_run_decode_logs_orchestrator,"*/15 * * * *" dbt_run_incremental,"16,46 1-3,5-7,9-11,13-15,17-19,21-23 * * *" +dbt_run_incremental_intermediate,"46 */4 * * *" dbt_run_incremental_non_core,"1 */4 * * *" dbt_run_incremental_non_core_hourly,"1 1-3,5-7,9-11,13-15,17-19,21-23 * * *" dbt_run_nft_compressed_mints_realtime,"*/15 * * * *"