From 4b9fcfb90e5effb7be014a85903c2ba2f5edcfe4 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 25 Apr 2025 14:16:16 -0600 Subject: [PATCH] resume logic --- .../dbt_run_template_alter_all_gha_tasks.yml | 23 +---- .../dbt_run_template_deploy_tasks.yml | 91 ------------------- .../global/github_actions/workflow_tasks.sql | 20 ++++ 3 files changed, 22 insertions(+), 112 deletions(-) delete mode 100644 .github/workflows/dbt_run_template_deploy_tasks.yml diff --git a/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml b/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml index 040b88f7..bc27ba8e 100644 --- a/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml +++ b/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml @@ -65,28 +65,9 @@ jobs: pip install -r requirements.txt dbt deps - - name: Generate task names from existing workflows + - name: Alter all GHA tasks run: | - TASK_NAMES="" - for file in $(find .github/workflows -name "*.yml" -type f); do - filename=$(basename "$file" .yml) - task="TRIGGER_$(echo $filename | tr '[:lower:]' '[:upper:]')" - if [ -z "$TASK_NAMES" ]; then - TASK_NAMES="$task" - else - TASK_NAMES="$TASK_NAMES,$task" - fi - done - echo "TASK_NAMES=$TASK_NAMES" >> $GITHUB_ENV - echo "Found task names: $TASK_NAMES" - - - name: Run DBT Jobs - run: | - if [ -n "$TASK_NAMES" ]; then - dbt run-operation fsc_evm.alter_gha_tasks --args '{"task_names": "${{ env.TASK_NAMES }}", "task_action": "${{ inputs.task_action }}"}' - else - echo "No workflow files found. No tasks to alter." - fi + dbt run-operation fsc_evm.alter_all_gha_tasks --args '{"task_action": "${{ inputs.task_action }}"}' notify-failure: needs: [run_dbt_jobs] diff --git a/.github/workflows/dbt_run_template_deploy_tasks.yml b/.github/workflows/dbt_run_template_deploy_tasks.yml deleted file mode 100644 index 450d1542..00000000 --- a/.github/workflows/dbt_run_template_deploy_tasks.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: dbt_run_template_deploy_tasks - -on: - workflow_call: - inputs: - target: - type: string - required: false - default: prod - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_secrets - - steps: - - uses: actions/checkout@v3 - - - name: Extract project & profile names from dbt_project.yml - id: project-name - run: | - PROFILE_NAME=$(grep "^profile:" dbt_project.yml | sed 's/^profile:[[:space:]]*"//' | sed 's/".*$//') - PROJECT_NAME=$(grep "^name:" dbt_project.yml | sed 's/^name:[[:space:]]*"//' | sed 's/".*$//') - echo "PROFILE_NAME=$PROFILE_NAME" >> $GITHUB_ENV - echo "PROJECT_NAME=$PROJECT_NAME" >> $GITHUB_ENV - echo "PROFILE_NAME: $PROFILE_NAME" - echo "PROJECT_NAME: $PROJECT_NAME" - - - name: Set environment variables - run: | - echo "TARGET=${{ inputs.target }}" >> $GITHUB_ENV - echo "ACCOUNT=vna27887.us-east-1" >> $GITHUB_ENV - echo "REGION=us-east-1" >> $GITHUB_ENV - echo "SCHEMA=ADMIN" >> $GITHUB_ENV - echo "USER=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV - echo "PASSWORD=${{ secrets.PASSWORD }}" >> $GITHUB_ENV - - if [[ "${{ inputs.target }}" == *"prod"* ]]; then - echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV - echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV - echo "WAREHOUSE=DBT_CLOUD" >> $GITHUB_ENV - elif [[ "${{ inputs.target }}" == *"test"* ]]; then - echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV - echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV - echo "WAREHOUSE=DBT_TEST" >> $GITHUB_ENV - else - echo "DATABASE=${PROFILE_NAME}_DEV" >> $GITHUB_ENV - echo "ROLE=INTERNAL_DEV" >> $GITHUB_ENV - echo "WAREHOUSE=DBT_CLOUD" >> $GITHUB_ENV - fi - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: Install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Create stored procedure - run: | - dbt run-operation sp_update_workflow_table - - - name: Generate workflow names - id: generate-workflows - run: | - WORKFLOW_NAMES="" - for file in $(find .github/workflows -name "*.yml" -type f); do - filename=$(basename "$file" .yml) - if [ -z "$WORKFLOW_NAMES" ]; then - WORKFLOW_NAMES="$filename" - else - WORKFLOW_NAMES="$WORKFLOW_NAMES,$filename" - fi - done - echo "WORKFLOW_NAMES=$WORKFLOW_NAMES" >> $GITHUB_ENV - echo "Found workflows: $WORKFLOW_NAMES" - - - name: Call Snowflake stored procedure - run: | - dbt run-operation run_query --args "{\"query\": \"CALL github_actions.update_workflow_table('${{ env.WORKFLOW_NAMES }}');\"}" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: ./.github/workflows/slack_notify.yml - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/macros/global/github_actions/workflow_tasks.sql b/macros/global/github_actions/workflow_tasks.sql index 44a2cd0d..04f514ec 100644 --- a/macros/global/github_actions/workflow_tasks.sql +++ b/macros/global/github_actions/workflow_tasks.sql @@ -44,6 +44,9 @@ -- Normalize database name {% set prod_db = target.database.lower().replace('_dev', '') %} + -- Track created tasks for potential resuming + {% set created_tasks = [] %} + -- Create tasks {% for result in results_list %} {% set task_name = result[0] %} @@ -67,7 +70,24 @@ END; {% endset %} {% do run_query(create_task_sql) %} + + {# Add task to list of created tasks #} + {% do created_tasks.append(task_name) %} {% endfor %} + + -- Optionally resume tasks if the variable is set + {% if var('RESUME_GHA_TASKS', false) %} + {% do log("Tasks created in RESUME state. Set var 'RESUME_GHA_TASKS': false and re-run to suspend them.", info=true) %} + {% for task_name in created_tasks %} + {% set resume_task_sql %} + ALTER TASK IF EXISTS github_actions.{{ task_name }} RESUME; + {% endset %} + {% do run_query(resume_task_sql) %} + {% do log("Resumed task: " ~ task_name, info=true) %} + {% endfor %} + {% else %} + {% do log("Tasks created in SUSPENDED state. Set var 'RESUME_GHA_TASKS': true and re-run to resume them.", info=true) %} + {% endif %} {% endmacro %} {% macro alter_gha_tasks(