resume logic

.github/workflows/dbt_run_template_alter_all_gha_tasks.yml

@@ -65,28 +65,9 @@ jobs:
           pip install -r requirements.txt
           dbt deps
       
-      - name: Generate task names from existing workflows
+      - name: Alter all GHA tasks
         run: |
-          TASK_NAMES=""
-          for file in $(find .github/workflows -name "*.yml" -type f); do
-            filename=$(basename "$file" .yml)
-            task="TRIGGER_$(echo $filename | tr '[:lower:]' '[:upper:]')"
-            if [ -z "$TASK_NAMES" ]; then
-              TASK_NAMES="$task"
-            else
-              TASK_NAMES="$TASK_NAMES,$task"
-            fi
-          done
-          echo "TASK_NAMES=$TASK_NAMES" >> $GITHUB_ENV
-          echo "Found task names: $TASK_NAMES"
-
-      - name: Run DBT Jobs
-        run: |
-          if [ -n "$TASK_NAMES" ]; then
-            dbt run-operation fsc_evm.alter_gha_tasks --args '{"task_names": "${{ env.TASK_NAMES }}", "task_action": "${{ inputs.task_action }}"}' 
-          else
-            echo "No workflow files found. No tasks to alter."
-          fi
+          dbt run-operation fsc_evm.alter_all_gha_tasks --args '{"task_action": "${{ inputs.task_action }}"}'
 
   notify-failure:
     needs: [run_dbt_jobs]

.github/workflows/dbt_run_template_deploy_tasks.yml

@@ -1,91 +0,0 @@
-name: dbt_run_template_deploy_tasks
-
-on:
-  workflow_call:
-    inputs:
-      target:
-        type: string
-        required: false
-        default: prod
-
-jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_secrets
-
-    steps:
-      - uses: actions/checkout@v3
-      
-      - name: Extract project & profile names from dbt_project.yml
-        id: project-name
-        run: |
-          PROFILE_NAME=$(grep "^profile:" dbt_project.yml | sed 's/^profile:[[:space:]]*"//' | sed 's/".*$//')
-          PROJECT_NAME=$(grep "^name:" dbt_project.yml | sed 's/^name:[[:space:]]*"//' | sed 's/".*$//')
-          echo "PROFILE_NAME=$PROFILE_NAME" >> $GITHUB_ENV
-          echo "PROJECT_NAME=$PROJECT_NAME" >> $GITHUB_ENV
-          echo "PROFILE_NAME: $PROFILE_NAME"
-          echo "PROJECT_NAME: $PROJECT_NAME"
-      
-      - name: Set environment variables
-        run: |
-          echo "TARGET=${{ inputs.target }}" >> $GITHUB_ENV
-          echo "ACCOUNT=vna27887.us-east-1" >> $GITHUB_ENV
-          echo "REGION=us-east-1" >> $GITHUB_ENV
-          echo "SCHEMA=ADMIN" >> $GITHUB_ENV
-          echo "USER=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV  
-          echo "PASSWORD=${{ secrets.PASSWORD }}" >> $GITHUB_ENV
-          
-          if [[ "${{ inputs.target }}" == *"prod"* ]]; then
-            echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
-            echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
-            echo "WAREHOUSE=DBT_CLOUD" >> $GITHUB_ENV
-          elif [[ "${{ inputs.target }}" == *"test"* ]]; then
-            echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
-            echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
-            echo "WAREHOUSE=DBT_TEST" >> $GITHUB_ENV
-          else
-            echo "DATABASE=${PROFILE_NAME}_DEV" >> $GITHUB_ENV
-            echo "ROLE=INTERNAL_DEV" >> $GITHUB_ENV
-            echo "WAREHOUSE=DBT_CLOUD" >> $GITHUB_ENV
-          fi
-          
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: Install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-      
-      - name: Create stored procedure
-        run: |
-          dbt run-operation sp_update_workflow_table
-      
-      - name: Generate workflow names
-        id: generate-workflows
-        run: |
-          WORKFLOW_NAMES=""
-          for file in $(find .github/workflows -name "*.yml" -type f); do
-            filename=$(basename "$file" .yml)
-            if [ -z "$WORKFLOW_NAMES" ]; then
-              WORKFLOW_NAMES="$filename"
-            else
-              WORKFLOW_NAMES="$WORKFLOW_NAMES,$filename"
-            fi
-          done
-          echo "WORKFLOW_NAMES=$WORKFLOW_NAMES" >> $GITHUB_ENV
-          echo "Found workflows: $WORKFLOW_NAMES"
-      
-      - name: Call Snowflake stored procedure
-        run: |
-          dbt run-operation run_query --args "{\"query\": \"CALL github_actions.update_workflow_table('${{ env.WORKFLOW_NAMES }}');\"}"
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: ./.github/workflows/slack_notify.yml
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

macros/global/github_actions/workflow_tasks.sql

@@ -44,6 +44,9 @@
     -- Normalize database name
     {% set prod_db = target.database.lower().replace('_dev', '') %}
     
+    -- Track created tasks for potential resuming
+    {% set created_tasks = [] %}
+    
     -- Create tasks
     {% for result in results_list %}
         {% set task_name = result[0] %}
@@ -67,7 +70,24 @@
         END;
         {% endset %}
         {% do run_query(create_task_sql) %}
+        
+        {# Add task to list of created tasks #}
+        {% do created_tasks.append(task_name) %}
     {% endfor %}
+    
+    -- Optionally resume tasks if the variable is set
+    {% if var('RESUME_GHA_TASKS', false) %}
+        {% do log("Tasks created in RESUME state. Set var 'RESUME_GHA_TASKS': false and re-run to suspend them.", info=true) %}
+        {% for task_name in created_tasks %}
+            {% set resume_task_sql %}
+            ALTER TASK IF EXISTS github_actions.{{ task_name }} RESUME;
+            {% endset %}
+            {% do run_query(resume_task_sql) %}
+            {% do log("Resumed task: " ~ task_name, info=true) %}
+        {% endfor %}
+    {% else %}
+        {% do log("Tasks created in SUSPENDED state. Set var 'RESUME_GHA_TASKS': true and re-run to resume them.", info=true) %}
+    {% endif %}
 {% endmacro %}
 
 {% macro alter_gha_tasks(