From 1a5de6cd3c6895fbeac66ed1755d8eec2cc2b0bd Mon Sep 17 00:00:00 2001 From: tarikceric <46071768+tarikceric@users.noreply.github.com> Date: Mon, 12 May 2025 11:00:25 -0700 Subject: [PATCH] update alerts (#61) * update alerts * move yml * rename --- .github/workflows/dbt_docs_update.yml | 7 ++ .github/workflows/dbt_run_adhoc.yml | 9 ++- .github/workflows/dbt_run_deploy_seeds.yml | 8 +- .../workflows/dbt_run_incremental_core.yml | 8 +- .../dbt_run_incremental_non_core.yml | 8 +- .github/workflows/dbt_run_observability.yml | 9 ++- .../workflows/dbt_run_observability_full.yml | 9 ++- .../dbt_run_streamline_block_txs_realtime.yml | 7 ++ ...bt_run_streamline_block_txs_realtime_2.yml | 9 ++- .../dbt_run_streamline_blocks_realtime.yml | 7 ++ .github/workflows/slack_notify.yml | 27 +++++++ python/slack_alert.py | 74 +++++++++++++++++++ 12 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/slack_notify.yml create mode 100644 python/slack_alert.py diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index 49fab51..8f2b720 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -69,3 +69,10 @@ jobs: - name: push changes to docs run: | git push -f --set-upstream origin docs + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 435a87d..986a193 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -64,4 +64,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - ${{ inputs.dbt_command }} \ No newline at end of file + ${{ inputs.dbt_command }} + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_deploy_seeds.yml b/.github/workflows/dbt_run_deploy_seeds.yml index b6a62af..6e3c4ee 100644 --- a/.github/workflows/dbt_run_deploy_seeds.yml +++ b/.github/workflows/dbt_run_deploy_seeds.yml @@ -45,4 +45,10 @@ jobs: - name: Run DBT Jobs run: | dbt seed -s "eclipse_models,./data" --full-refresh - + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_incremental_core.yml b/.github/workflows/dbt_run_incremental_core.yml index fa7e7a9..6158761 100644 --- a/.github/workflows/dbt_run_incremental_core.yml +++ b/.github/workflows/dbt_run_incremental_core.yml @@ -42,4 +42,10 @@ jobs: - name: Run DBT Jobs run: | dbt run -s "eclipse_models,tag:scheduled_core" - + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_incremental_non_core.yml b/.github/workflows/dbt_run_incremental_non_core.yml index 77b91cd..32fc68c 100644 --- a/.github/workflows/dbt_run_incremental_non_core.yml +++ b/.github/workflows/dbt_run_incremental_non_core.yml @@ -41,4 +41,10 @@ jobs: - name: Run DBT Jobs run: | dbt run -s "eclipse_models,tag:scheduled_core" "eclipse_models,tag:scheduled_non_core" - + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml index 52bbdf5..a927dde 100644 --- a/.github/workflows/dbt_run_observability.yml +++ b/.github/workflows/dbt_run_observability.yml @@ -42,4 +42,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt build -s "eclipse_models,tag:observability" \ No newline at end of file + dbt build -s "eclipse_models,tag:observability" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_observability_full.yml b/.github/workflows/dbt_run_observability_full.yml index 2e1baaf..1b30af2 100644 --- a/.github/workflows/dbt_run_observability_full.yml +++ b/.github/workflows/dbt_run_observability_full.yml @@ -42,4 +42,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt build --vars '{"OBSERV_FULL_TEST":True}' -s "eclipse_models,tag:observability" \ No newline at end of file + dbt build --vars '{"OBSERV_FULL_TEST":True}' -s "eclipse_models,tag:observability" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_block_txs_realtime.yml b/.github/workflows/dbt_run_streamline_block_txs_realtime.yml index bd873f2..65af5d7 100644 --- a/.github/workflows/dbt_run_streamline_block_txs_realtime.yml +++ b/.github/workflows/dbt_run_streamline_block_txs_realtime.yml @@ -42,3 +42,10 @@ jobs: - name: Run DBT Jobs run: | dbt run -s streamline__block_txs_complete streamline__block_txs_realtime --vars '{STREAMLINE_INVOKE_STREAMS: True}' + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_block_txs_realtime_2.yml b/.github/workflows/dbt_run_streamline_block_txs_realtime_2.yml index 2aa0950..a04593f 100644 --- a/.github/workflows/dbt_run_streamline_block_txs_realtime_2.yml +++ b/.github/workflows/dbt_run_streamline_block_txs_realtime_2.yml @@ -41,4 +41,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -s streamline__block_txs_complete_2 streamline__block_txs_realtime_2 --vars '{STREAMLINE_INVOKE_STREAMS: True}' \ No newline at end of file + dbt run -s streamline__block_txs_complete_2 streamline__block_txs_realtime_2 --vars '{STREAMLINE_INVOKE_STREAMS: True}' + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_blocks_realtime.yml b/.github/workflows/dbt_run_streamline_blocks_realtime.yml index bd7faf0..1305118 100644 --- a/.github/workflows/dbt_run_streamline_blocks_realtime.yml +++ b/.github/workflows/dbt_run_streamline_blocks_realtime.yml @@ -42,3 +42,10 @@ jobs: - name: Run DBT Jobs run: | dbt run -s streamline__blocks_complete streamline__blocks_realtime --vars '{STREAMLINE_INVOKE_STREAMS: True}' + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/slack_notify.yml b/.github/workflows/slack_notify.yml new file mode 100644 index 0000000..e4afc12 --- /dev/null +++ b/.github/workflows/slack_notify.yml @@ -0,0 +1,27 @@ +name: Slack Notification +on: + workflow_call: + secrets: + SLACK_WEBHOOK_URL: + required: true + +jobs: + notify: + runs-on: ubuntu-latest + environment: workflow_prod + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install dependencies + run: pip install requests + + - name: Send Slack notification + run: python python/slack_alert.py + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/python/slack_alert.py b/python/slack_alert.py new file mode 100644 index 0000000..11ad94a --- /dev/null +++ b/python/slack_alert.py @@ -0,0 +1,74 @@ +import requests +import os +import sys + +def create_message(): + """Creates a simple failure notification message with repo, workflow name, and URL""" + + # Get GitHub environment variables + repository = os.environ.get('GITHUB_REPOSITORY', 'Unknown repository') + repo_name = repository.split('/')[-1] if '/' in repository else repository + workflow_name = os.environ.get('GITHUB_WORKFLOW', 'Unknown workflow') + run_id = os.environ.get('GITHUB_RUN_ID', '') + server_url = os.environ.get('GITHUB_SERVER_URL', 'https://github.com') + + # Build the workflow URL + workflow_url = f"{server_url}/{repository}/actions/runs/{run_id}" + + message_body = { + "text": f"Failure in {repo_name}", + "attachments": [ + { + "color": "#f44336", # Red color for failures + "fields": [ + { + "title": "Repository", + "value": repository, + "short": True + }, + { + "title": "Workflow", + "value": workflow_name, + "short": True + } + ], + "actions": [ + { + "type": "button", + "text": "View Workflow Run", + "style": "primary", + "url": workflow_url + } + ], + "footer": "GitHub Actions" + } + ] + } + + return message_body + +def send_alert(webhook_url): + """Sends a failure notification to Slack""" + + message = create_message() + + try: + response = requests.post(webhook_url, json=message) + + if response.status_code == 200: + print("Successfully sent Slack notification") + else: + print(f"Failed to send Slack notification: {response.status_code} {response.text}") + sys.exit(1) + except Exception as e: + print(f"Error sending Slack notification: {str(e)}") + sys.exit(1) + +if __name__ == '__main__': + webhook_url = os.environ.get("SLACK_WEBHOOK_URL") + + if not webhook_url: + print("ERROR: SLACK_WEBHOOK_URL environment variable is required") + sys.exit(1) + + send_alert(webhook_url) \ No newline at end of file