diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index 1cf5195..b4b38c9 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -21,12 +21,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: @@ -41,10 +39,10 @@ jobs: run: | pip install -r requirements.txt dbt deps - + - name: refresh ddl for datashare run: | - cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; + cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; - name: checkout docs branch run: | git checkout -B docs origin/main @@ -71,4 +69,11 @@ jobs: git commit -am "Auto-update docs" - name: push changes to docs run: | - git push -f --set-upstream origin docs \ No newline at end of file + git push -f --set-upstream origin docs + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_abi_refresh.yml b/.github/workflows/dbt_run_abi_refresh.yml index 84d3643..81302b2 100644 --- a/.github/workflows/dbt_run_abi_refresh.yml +++ b/.github/workflows/dbt_run_abi_refresh.yml @@ -5,8 +5,8 @@ on: workflow_dispatch: schedule: # Runs “At minute 49 past every 12th hour.” (see https://crontab.guru) - - cron: '49 */12 * * *' - + - cron: "49 */12 * * *" + env: DBT_PROFILES_DIR: ./ @@ -22,12 +22,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: @@ -45,4 +43,10 @@ jobs: - name: Run DBT Jobs run: | dbt run -m "kaia_models,tag:abis" - \ No newline at end of file + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 3b2d63f..b4e2a4b 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -7,7 +7,7 @@ on: - "main" inputs: environment: - type: choice + type: choice description: DBT Run Environment required: true options: @@ -15,9 +15,9 @@ on: - prod default: dev warehouse: - type: choice + type: choice description: Snowflake warehouse - required: true + required: true options: - DBT - DBT_CLOUD @@ -26,9 +26,9 @@ on: default: DBT dbt_command: type: string - description: 'DBT Run Command' + description: "DBT Run Command" required: true - + env: DBT_PROFILES_DIR: ./ @@ -44,12 +44,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_${{ inputs.environment }} steps: @@ -66,4 +64,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - ${{ inputs.dbt_command }} \ No newline at end of file + ${{ inputs.dbt_command }} + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml deleted file mode 100644 index f30daa9..0000000 --- a/.github/workflows/dbt_run_deployment.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: dbt_run_deployment -run-name: ${{ inputs.dbt_command }} - -on: - workflow_dispatch: - branches: - - "main" - inputs: - warehouse: - type: choice - description: Snowflake warehouse - required: true - options: - - DBT - - DBT_CLOUD - - DBT_EMERGENCY - default: DBT - dbt_command: - type: string - description: 'DBT Run Command' - required: true - -env: - DBT_PROFILES_DIR: ./ - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ inputs.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - - - -jobs: - dbt: - runs-on: ubuntu-latest - environment: - name: workflow_prod - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - ${{ inputs.dbt_command }} - - name: Run datashare model - run: | - cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -gt 0 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; - dbt run-operation run_query --args "{sql: call admin.datashare.sp_grant_share_permissions('${{ env.DATABASE }}')}" - - name: Store logs - uses: actions/upload-artifact@v3 - with: - name: dbt-logs - path: | - logs - target \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 80e3eff..893bb21 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: schedule: # Runs "at 10:00 UTC" (see https://crontab.guru) - - cron: '0 10 * * *' + - cron: "0 10 * * *" env: DBT_PROFILES_DIR: ./ @@ -22,8 +22,6 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs_refresh: runs-on: ubuntu-latest @@ -46,6 +44,13 @@ jobs: run: | dbt run-operation run_sp_create_prod_clone + notify-failure: + needs: [run_dbt_jobs_refresh] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run_dbt_jobs_udfs: runs-on: ubuntu-latest needs: run_dbt_jobs_refresh @@ -67,4 +72,11 @@ jobs: - name: Run Recreate UDFs run: | dbt run-operation create_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev - dbt run -s livequery_models.deploy.core._live --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev \ No newline at end of file + dbt run -s livequery_models.deploy.core._live --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev + + notify-failure2: + needs: [run_dbt_jobs_udfs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml index 6af1343..299e594 100644 --- a/.github/workflows/dbt_run_full_observability.yml +++ b/.github/workflows/dbt_run_full_observability.yml @@ -5,8 +5,8 @@ on: workflow_dispatch: schedule: # Runs “At 17:00 on day-of-month 1.” (see https://crontab.guru) - - cron: '30 17 1 * *' - + - cron: "30 17 1 * *" + env: DBT_PROFILES_DIR: ./ @@ -22,12 +22,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod_2xl steps: @@ -45,6 +43,10 @@ jobs: - name: Run DBT Jobs run: | dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m "kaia_models,tag:observability" - - + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_heal_models.yml b/.github/workflows/dbt_run_heal_models.yml index 84c0992..9bcfc14 100644 --- a/.github/workflows/dbt_run_heal_models.yml +++ b/.github/workflows/dbt_run_heal_models.yml @@ -5,8 +5,8 @@ on: workflow_dispatch: schedule: # Runs at 02:00 on Wednesday (see https://crontab.guru) - - cron: '0 2 * * 3' - + - cron: "0 2 * * 3" + env: DBT_PROFILES_DIR: ./ @@ -22,12 +22,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: @@ -44,4 +42,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "kaia_models,tag:heal" --vars '{"HEAL_MODEL":True}' \ No newline at end of file + dbt run -m "kaia_models,tag:heal" --vars '{"HEAL_MODEL":True}' + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_operation_reorg.yml b/.github/workflows/dbt_run_operation_reorg.yml index 1752a38..6e3ec4f 100644 --- a/.github/workflows/dbt_run_operation_reorg.yml +++ b/.github/workflows/dbt_run_operation_reorg.yml @@ -5,8 +5,8 @@ on: workflow_dispatch: schedule: # Runs at minute 50 every 8 hours (see https://crontab.guru) - - cron: '50 */8 * * *' - + - cron: "50 */8 * * *" + env: DBT_PROFILES_DIR: ./ @@ -22,12 +22,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: @@ -48,7 +46,14 @@ jobs: run: | reorg_model_list=$(dbt list --select "kaia_models,tag:reorg" --resource-type model --output name | grep '__' | awk -F'.' '{print $NF}' | tr '\n' ',' | sed 's/,$//') echo "model_list=$reorg_model_list" >> $GITHUB_OUTPUT - + - name: Execute block_reorg macro run: | - dbt run-operation fsc_utils.block_reorg --args "{reorg_model_list: '${{ steps.list_models.outputs.model_list }}', hours: '12'}" && awk '/SQL status/ {print; next} /DELETE FROM/{getline; print} /\/\* {/ {print}' logs/dbt.log \ No newline at end of file + dbt run-operation fsc_utils.block_reorg --args "{reorg_model_list: '${{ steps.list_models.outputs.model_list }}', hours: '12'}" && awk '/SQL status/ {print; next} /DELETE FROM/{getline; print} /\/\* {/ {print}' logs/dbt.log + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_scheduled_curated.yml b/.github/workflows/dbt_run_scheduled_curated.yml index 36a1f11..f530d09 100644 --- a/.github/workflows/dbt_run_scheduled_curated.yml +++ b/.github/workflows/dbt_run_scheduled_curated.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: branches: - "main" - + env: DBT_PROFILES_DIR: ./ @@ -21,12 +21,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: @@ -43,4 +41,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "kaia_models,tag:curated" --exclude "models/bronze/api_udf/bronze_api__contract_abis.sql" \ No newline at end of file + dbt run -m "kaia_models,tag:curated" --exclude "models/bronze/api_udf/bronze_api__contract_abis.sql" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_scheduled_non_realtime.yml b/.github/workflows/dbt_run_scheduled_non_realtime.yml index 8c310f5..99f17b7 100644 --- a/.github/workflows/dbt_run_scheduled_non_realtime.yml +++ b/.github/workflows/dbt_run_scheduled_non_realtime.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: branches: - "main" - + env: DBT_PROFILES_DIR: ./ @@ -21,12 +21,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: @@ -43,4 +41,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "kaia_models,tag:non_realtime" \ No newline at end of file + dbt run -m "kaia_models,tag:non_realtime" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_streamline_decoder.yml b/.github/workflows/dbt_run_streamline_decoder.yml index 1f25843..d439ade 100644 --- a/.github/workflows/dbt_run_streamline_decoder.yml +++ b/.github/workflows/dbt_run_streamline_decoder.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: branches: - "main" - + env: DBT_PROFILES_DIR: ./ @@ -21,12 +21,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: @@ -43,4 +41,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "kaia_models,tag:decoded_logs" \ No newline at end of file + dbt run -m "kaia_models,tag:decoded_logs" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_trace_backfill.yml b/.github/workflows/dbt_run_trace_backfill.yml index 077578b..9b5fa94 100644 --- a/.github/workflows/dbt_run_trace_backfill.yml +++ b/.github/workflows/dbt_run_trace_backfill.yml @@ -5,8 +5,8 @@ on: workflow_dispatch: schedule: # Runs “At Minute 50 every hour” (see https://crontab.guru) - - cron: '50 * * * *' - + - cron: "50 * * * *" + env: DBT_PROFILES_DIR: ./ @@ -22,12 +22,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod_2xl steps: @@ -46,3 +44,9 @@ jobs: run: | dbt run --select silver__fact_traces2 + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index f9bca4b..520387d 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -5,8 +5,8 @@ on: workflow_dispatch: schedule: # Runs “At minute 5 past every 4th hour.” (see https://crontab.guru) - - cron: '10 */4 * * *' - + - cron: "10 */4 * * *" + env: DBT_PROFILES_DIR: ./ @@ -22,12 +22,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_test steps: @@ -44,4 +42,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt test -m "kaia_models,tag:recent_test" \ No newline at end of file + dbt test -m "kaia_models,tag:recent_test" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/slack_notify.yml b/.github/workflows/slack_notify.yml new file mode 100644 index 0000000..2dc6d14 --- /dev/null +++ b/.github/workflows/slack_notify.yml @@ -0,0 +1,27 @@ +name: Slack Notification +on: + workflow_call: + secrets: + SLACK_WEBHOOK_URL: + required: true + +jobs: + notify: + runs-on: ubuntu-latest + environment: workflow_prod + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install dependencies + run: pip install requests + + - name: Send Slack notification + run: python python/slack_alert.py + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/python/slack_alert.py b/python/slack_alert.py new file mode 100644 index 0000000..11ad94a --- /dev/null +++ b/python/slack_alert.py @@ -0,0 +1,74 @@ +import requests +import os +import sys + +def create_message(): + """Creates a simple failure notification message with repo, workflow name, and URL""" + + # Get GitHub environment variables + repository = os.environ.get('GITHUB_REPOSITORY', 'Unknown repository') + repo_name = repository.split('/')[-1] if '/' in repository else repository + workflow_name = os.environ.get('GITHUB_WORKFLOW', 'Unknown workflow') + run_id = os.environ.get('GITHUB_RUN_ID', '') + server_url = os.environ.get('GITHUB_SERVER_URL', 'https://github.com') + + # Build the workflow URL + workflow_url = f"{server_url}/{repository}/actions/runs/{run_id}" + + message_body = { + "text": f"Failure in {repo_name}", + "attachments": [ + { + "color": "#f44336", # Red color for failures + "fields": [ + { + "title": "Repository", + "value": repository, + "short": True + }, + { + "title": "Workflow", + "value": workflow_name, + "short": True + } + ], + "actions": [ + { + "type": "button", + "text": "View Workflow Run", + "style": "primary", + "url": workflow_url + } + ], + "footer": "GitHub Actions" + } + ] + } + + return message_body + +def send_alert(webhook_url): + """Sends a failure notification to Slack""" + + message = create_message() + + try: + response = requests.post(webhook_url, json=message) + + if response.status_code == 200: + print("Successfully sent Slack notification") + else: + print(f"Failed to send Slack notification: {response.status_code} {response.text}") + sys.exit(1) + except Exception as e: + print(f"Error sending Slack notification: {str(e)}") + sys.exit(1) + +if __name__ == '__main__': + webhook_url = os.environ.get("SLACK_WEBHOOK_URL") + + if not webhook_url: + print("ERROR: SLACK_WEBHOOK_URL environment variable is required") + sys.exit(1) + + send_alert(webhook_url) \ No newline at end of file