From 603ae9f16d3c9441e1c4bb9504fd9cd7db34fc04 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 7 Aug 2025 16:01:34 +0900 Subject: [PATCH] Enhance Slack Alert Macros with AI Analysis Features - Updated README documentation to clarify AI provider options and added new parameters for model selection and custom prompts. - Modified SQL macros to support the new `model_name` and `ai_prompt` parameters for improved AI analysis capabilities. - Adjusted UDF signatures and comments to reflect the changes in AI provider functionality and requirements. - Improved test cases to validate the new features and ensure robust performance of the Slack alert macros. --- macros/alerts/README.md | 54 +++++++---- macros/alerts/slack_alert.sql | 6 +- .../marketplace/github/actions_udtfs.yml.sql | 90 ++++--------------- .../github/github_actions__github_utils.yml | 23 ++--- 4 files changed, 66 insertions(+), 107 deletions(-) diff --git a/macros/alerts/README.md b/macros/alerts/README.md index 943327f..687add8 100644 --- a/macros/alerts/README.md +++ b/macros/alerts/README.md @@ -72,7 +72,9 @@ The `failed_gha_slack_alert` macro is ready to use immediately - no deployment r | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `enable_ai_analysis` | boolean | `true` | Enable AI failure analysis | -| `ai_provider` | string | `'cortex'` | AI provider: `'cortex'` or `'claude'` | +| `ai_provider` | string | `'cortex'` | AI provider: `'cortex'` (Snowflake built-in AI) | +| `model_name` | string | `'mistral-large'` | **Required for Cortex**: `'mistral-large'`, `'mistral-7b'`, `'llama2-70b-chat'`, `'mixtral-8x7b'` | +| `ai_prompt` | string | `''` | Custom AI analysis prompt (leave empty for default) | ### Threading & Appearance @@ -106,13 +108,14 @@ dbt run-operation failed_gha_slack_alert --vars '{ "run_id": "16729602656", "slack_channel": "C087GJQ1ZHQ", "enable_ai_analysis": true, - "ai_provider": "claude", + "ai_provider": "cortex", + "model_name": "mistral-7b", "username": "CI/CD Alert Bot", "icon_emoji": ":robot_face:" }' --target dev ``` -### Auto-Threading with Detailed Logs +### Auto-Threading with Custom Prompt ```bash dbt run-operation failed_gha_slack_alert --vars '{ @@ -122,6 +125,8 @@ dbt run-operation failed_gha_slack_alert --vars '{ "slack_channel": "C087GJQ1ZHQ", "enable_ai_analysis": true, "ai_provider": "cortex", + "model_name": "mixtral-8x7b", + "ai_prompt": "Focus on dependency issues and provide quick fixes:", "enable_auto_threading": true, "username": "Pipeline Monitor", "icon_emoji": ":stellar:" @@ -137,6 +142,8 @@ dbt run-operation failed_gha_slack_alert --vars '{ "run_id": "16729602656", "webhook_secret_name": "prod-alerts", "enable_ai_analysis": true, + "ai_provider": "cortex", + "model_name": "mistral-large", "username": "Production Monitor", "icon_emoji": ":package:" }' --target dev @@ -155,6 +162,7 @@ dbt run-operation failed_gha_slack_alert --vars '{ "slack_channel": "C087GJQ1ZHQ", "enable_ai_analysis": true, "ai_provider": "cortex", + "model_name": "mistral-large", "enable_auto_threading": true, "username": "GitHub Actions", "icon_emoji": ":github:" @@ -163,7 +171,7 @@ dbt run-operation failed_gha_slack_alert --vars '{ ## Message Format -### Failure Messages Include: +### Failure Messages Include - **๐Ÿ”ด Red Sidebar**: Visual failure indicator - **Header**: Repository name with failure indicator (โŒ) @@ -172,7 +180,7 @@ dbt run-operation failed_gha_slack_alert --vars '{ - **๐Ÿ”— Action Button**: Direct link to workflow run - **๐Ÿงต Threading** (if enabled): Individual job details and logs as thread replies -### Success Messages Include: +### Success Messages Include - **๐ŸŸข Green Sidebar**: Visual success indicator - **Header**: Repository name with success indicator (โœ…) @@ -181,30 +189,33 @@ dbt run-operation failed_gha_slack_alert --vars '{ ## AI Analysis -The macro supports two AI providers for intelligent failure analysis: +The macro supports Snowflake's Cortex AI for intelligent failure analysis: ### Cortex (Default) -- Uses Snowflake's built-in Cortex AI -- No additional configuration required -- Automatically analyzes logs and provides insights -### Claude -- Uses Anthropic's Claude AI via Livequery integration -- Provides sophisticated reasoning and code analysis -- Better at understanding complex failure patterns +- Uses Snowflake's built-in Cortex AI +- **Requires `model_name` parameter** to specify which model to use +- Available models: `'mistral-large'`, `'mistral-7b'`, `'llama2-70b-chat'`, `'mixtral-8x7b'` +- Automatically analyzes logs and provides insights +- Custom prompts supported via `ai_prompt` parameter Enable AI analysis with: + ```yaml "enable_ai_analysis": true, -"ai_provider": "claude" # or "cortex" +"ai_provider": "cortex", +"model_name": "mistral-large", # Required! +"ai_prompt": "Focus on the most critical issues:" # Optional ``` ## Environment Variables & Vault Setup ### Webhook Method + - `SLACK_WEBHOOK_URL`: Your Slack webhook URL (GitHub secret) ### Bot Token Method + - **No environment variables required!** - Bot tokens are stored in Livequery vault at: `_FSC_SYS/SLACK/{bot_secret_name}` - Channel ID provided as parameter in macro call @@ -212,10 +223,12 @@ Enable AI analysis with: ### Vault Paths for Bot Tokens Store your bot tokens in these vault locations: -- `_FSC_SYS/SLACK/intelligence` (default) -- `_FSC_SYS/SLACK/alerts` (custom) -- `_FSC_SYS/SLACK/ci-cd` (custom) -- `_FSC_SYS/SLACK/prod` (custom) + +- `prod/livequery/slack/intelligence` (default) +- `prod/livequery/alerts` (custom) +- `prod/livequery/` (custom) + +** The `_FSC/SYS/..` will not work anymore, because we are not able to access studio to store `CREDENTIALS` anymore. So the context + `_FSC/SYS/...` is deprecated. It's in the sql code for backward compatability. ### How to Get Slack Channel IDs @@ -254,10 +267,10 @@ Add this step before the notification to debug issues: echo "Channel: C1234567890" # Your actual channel ID ``` - ### Channel ID Validation Test if your channel ID is valid: + ```sql SELECT slack_utils.validate_channel('C1234567890') as is_valid; -- Should return true for valid channel IDs @@ -291,18 +304,21 @@ Ensure these UDFs are deployed before using the notification macro. ## Performance & Benefits ### โšก **Lightning Fast Execution** + - **Pure SQL**: No Python interpreter overhead - **Direct UDF calls**: Leverages Livequery's optimized marketplace functions - **Single transaction**: All operations in one dbt run-operation call - **Instant feedback**: Real-time execution with immediate Slack delivery ### ๐ŸŽฏ **Production Ready** + - **Reliable**: Battle-tested with GitHub Actions workflows - **Scalable**: Handles multiple failed jobs with threading - **Secure**: Vault-based credential management - **Flexible**: Supports both webhook and bot token methods ### ๐Ÿค– **Intelligent Analysis** + - **AI-Powered**: Cortex and Claude integration for failure analysis - **Actionable Insights**: Common patterns, root causes, and prioritized action items - **Context-Aware**: Includes job names, workflow details, and error logs diff --git a/macros/alerts/slack_alert.sql b/macros/alerts/slack_alert.sql index 808a1e7..515dc88 100644 --- a/macros/alerts/slack_alert.sql +++ b/macros/alerts/slack_alert.sql @@ -7,6 +7,8 @@ {%- set slack_channel = var('slack_channel', none) -%} {%- set enable_ai_analysis = var('enable_ai_analysis', true) -%} {%- set ai_provider = var('ai_provider', 'cortex') -%} + {%- set model_name = var('model_name', 'mistral-large') -%} + {%- set ai_prompt = var('ai_prompt', '') -%} {%- set enable_auto_threading = var('enable_auto_threading', false) -%} {%- set bot_secret_name = var('bot_secret_name', 'intelligence') -%} {%- set webhook_secret_name = var('webhook_secret_name', none) -%} @@ -27,7 +29,7 @@ {{ return("") }} {%- endif -%} - {%- if enable_ai_analysis and ai_provider in ['cortex', 'claude'] -%} + {%- if enable_ai_analysis -%} {# Get failure data with AI analysis #} {% set failure_query %} SELECT @@ -35,7 +37,7 @@ ai_analysis, total_failures, failure_metadata - FROM TABLE(github_actions.tf_failure_analysis_with_ai('{{ owner }}', '{{ repo }}', '{{ run_id }}', '{{ ai_provider }}', '{{ var("model_name", "") }}')) + FROM TABLE(github_actions.tf_failure_analysis_with_ai('{{ owner }}', '{{ repo }}', '{{ run_id }}', '{{ ai_provider }}', '{{ model_name }}', '{{ ai_prompt }}')) {% endset %} {%- set failure_results = run_query(failure_query) -%} diff --git a/macros/marketplace/github/actions_udtfs.yml.sql b/macros/marketplace/github/actions_udtfs.yml.sql index e7fc15c..512978c 100644 --- a/macros/marketplace/github/actions_udtfs.yml.sql +++ b/macros/marketplace/github/actions_udtfs.yml.sql @@ -290,15 +290,16 @@ - name: {{ schema_name -}}.tf_failure_analysis_with_ai signature: - - [owner, "TEXT"] - - [repo, "TEXT"] - - [run_id, "TEXT"] - - [ai_provider, "TEXT"] - - [model_name, "STRING"] + - [owner, "TEXT", "GitHub repository owner/organization name"] + - [repo, "TEXT", "GitHub repository name"] + - [run_id, "TEXT", "GitHub Actions run ID to analyze"] + - [ai_provider, "TEXT", "AI provider to use: 'cortex' (Snowflake built-in AI)"] + - [model_name, "STRING", "Model name (required): 'mistral-large', 'mistral-7b', 'llama2-70b-chat', 'mixtral-8x7b'"] + - [ai_prompt, "STRING", "Custom AI analysis prompt. Leave empty to use default failure analysis prompt."] return_type: - "TABLE(run_id STRING, ai_analysis STRING, total_failures NUMBER, failure_metadata ARRAY)" options: | - COMMENT = $$Gets GitHub Actions failure analysis with configurable AI providers (cortex, claude, groq) for Slack notifications.$$ + COMMENT = $$Gets GitHub Actions failure analysis using Snowflake Cortex AI with custom prompts for Slack notifications.$$ sql: | WITH failure_data AS ( SELECT @@ -328,68 +329,16 @@ ) SELECT run_id::STRING, - CASE - WHEN LOWER(COALESCE(ai_provider, 'cortex')) = 'cortex' THEN - snowflake.cortex.complete( - 'mistral-large', - CONCAT( - 'Analyze these ', total_failures, ' GitHub Actions failures for run ', run_id, ' and provide:\n', - 'Keep it concise with 1-2 sentences per section\n', - 'Return the analysis in markdown format\n', - '1. Common failure patterns\n', - '2. Root cause analysis\n', - '3. Prioritized action items\n\n', - job_details - ) - ) - WHEN LOWER(ai_provider) = 'claude' THEN - ( - SELECT COALESCE( - response:data:content[0]:text::STRING, - response:data:error:message::STRING, - 'Claude analysis failed' - ) - FROM ( - SELECT claude.post_messages( - COALESCE(NULLIF(model_name, ''), 'claude-3-5-sonnet-20241022'), - ARRAY_CONSTRUCT( - OBJECT_CONSTRUCT( - 'role', 'user', - 'content', CONCAT( - 'Analyze these ', total_failures, ' GitHub Actions failures for run ', run_id, ' and provide:\n', - 'Keep it concise with 1-2 sentences per section\n', - 'Return the analysis in markdown format\n', - '1. Common failure patterns\n', - '2. Root cause analysis\n', - '3. Prioritized action items\n\n', - job_details - ) - ) - ), - 4096 - ) as response - ) - ) - WHEN LOWER(ai_provider) = 'groq' THEN - ( - SELECT groq.extract_response_text( - groq.quick_chat( - CONCAT( - 'Analyze these ', total_failures, ' GitHub Actions failures for run ', run_id, ' and provide:\n', - 'Keep it concise with 1-2 sentences\n', - 'Return the analysis in markdown format\n', - '1. Common failure patterns\n', - '2. Root cause analysis\n', - '3. Prioritized action items\n\n', - job_details - ), - COALESCE(NULLIF(model_name, ''), 'llama3-8b-8192') - ) - ) - ) - ELSE - CONCAT('Unsupported AI provider: ', COALESCE(ai_provider, 'null')) - END as ai_analysis, + snowflake.cortex.complete( + model_name, + CONCAT( + COALESCE( + NULLIF(ai_prompt, ''), + 'Analyze these GitHub Actions failures and provide:\n1. Common failure patterns\n2. Root cause analysis\n3. Prioritized action items\n\nKeep it concise with 1-2 sentences per section in markdown format.\n\n' + ), + job_details + ) + ) as ai_analysis, total_failures, failure_metadata FROM failure_data @@ -399,12 +348,11 @@ - [owner, "TEXT"] - [repo, "TEXT"] - [run_id, "TEXT"] - - [ai_provider, "TEXT"] return_type: - "TABLE(run_id STRING, ai_analysis STRING, total_failures NUMBER, failure_metadata ARRAY)" options: | - COMMENT = $$Gets GitHub Actions failure analysis with configurable AI providers (cortex, claude, groq) for Slack notifications. Uses default groq model.$$ + COMMENT = $$Gets GitHub Actions failure analysis with default AI provider (cortex) for Slack notifications.$$ sql: | - SELECT * FROM TABLE({{ schema_name -}}.tf_failure_analysis_with_ai(owner, repo, run_id, ai_provider, '')) + SELECT * FROM TABLE({{ schema_name -}}.tf_failure_analysis_with_ai(owner, repo, run_id, 'cortex', 'mistral-large', '')) {% endmacro %} diff --git a/models/deploy/marketplace/github/github_actions__github_utils.yml b/models/deploy/marketplace/github/github_actions__github_utils.yml index 4cf7249..e82d108 100644 --- a/models/deploy/marketplace/github/github_actions__github_utils.yml +++ b/models/deploy/marketplace/github/github_actions__github_utils.yml @@ -246,26 +246,20 @@ models: 'FlipsideCrypto', 'admin-models', '12345678', - 'cortex' + 'cortex', + 'mistral-large', + '' assertions: - row_count >= 0 - test_udtf: - name: test_github_actions__tf_failure_analysis_with_ai_claude + name: test_github_actions__tf_failure_analysis_with_ai_custom_prompt args: > 'FlipsideCrypto', 'admin-models', '12345678', - 'claude' - assertions: - - row_count >= 0 - - test_udtf: - name: test_github_actions__tf_failure_analysis_with_ai_groq - args: > - 'FlipsideCrypto', - 'admin-models', - '12345678', - 'groq', - 'llama3-8b-8192' + 'cortex', + 'mistral-7b', + 'Analyze these failures and provide concise recommendations:' assertions: - row_count >= 0 - test_udtf: @@ -273,8 +267,7 @@ models: args: > 'FlipsideCrypto', 'admin-models', - '12345678', - 'cortex' + '12345678' assertions: - row_count >= 0