livequery-models/macros/marketplace/github/utils_udfs.yaml.sql
Jensen Yap 8138951db2
[STREAM-1324] Add Slack, Github Action Logs & Slack AI Alerts (#127)
* Update GitHub Actions workflow to reduce thread count and add extensive API integration documentation

- Changed thread count from 24 to 5 in GitHub Actions workflows for improved performance.
- Added comprehensive README files for various API integrations including Alchemy, NBA All Day, API Layer, Binance, and more.
- Introduced new UDFs and UDTFs for Groq and Slack API integrations, enhancing functionality and usability.
- Implemented tests for new UDFs and UDTFs to ensure reliability and correctness.
- Updated existing UDF definitions and added new tests for enhanced coverage and robustness.

* Refactor Slack UDFs to use webhook secret names and improve error handling

- Updated UDF definitions to replace WEBHOOK_URL and BOT_TOKEN with WEBHOOK_SECRET_NAME for enhanced security.
- Improved error messages for required parameters in the SQL logic.
- Standardized comments for clarity and consistency across UDFs.
- Ensured proper handling of user context for accessing secrets in the vault.

* update test

* fix test

* update slack

* remove test

* fix tests

* fix

* fix test args

* fix

* add .gitignore

* Add Slack Notification Macros and Enhance UDFs

- Introduced a new dbt macro for sending Slack notifications from GitHub Actions with AI-powered failure analysis.
- Added comprehensive README documentation for the new macro, detailing setup, configuration options, and usage examples.
- Implemented a SQL macro to handle Slack message formatting and sending, including support for AI analysis and threading.
- Updated existing UDFs to utilize webhook secret names for improved security and added detailed comments for clarity.
- Enhanced error handling and logging within the macros to ensure robust operation and easier debugging.

* update slack alerts

* update

* remove groq

* Enhance Slack Alert Macros with AI Analysis Features

- Updated README documentation to clarify AI provider options and added new parameters for model selection and custom prompts.
- Modified SQL macros to support the new `model_name` and `ai_prompt` parameters for improved AI analysis capabilities.
- Adjusted UDF signatures and comments to reflect the changes in AI provider functionality and requirements.
- Improved test cases to validate the new features and ensure robust performance of the Slack alert macros.

* update slack_alert

* change secret path to data_platform

* add backward compatibility for udf_api_v2

* revert to Object return type

* update type
2025-08-08 23:42:24 +09:00

140 lines
4.7 KiB
SQL

{% macro config_github_utils_udfs(schema_name = "github_utils", utils_schema_name = "github_utils") -%}
{#
This macro is used to generate the Github API Calls
#}
- name: {{ schema_name -}}.octocat
signature:
- []
return_type:
- "VARIANT"
options: |
COMMENT = $$Verify token [Authenticating to the REST API](https://docs.github.com/en/rest/overview/authenticating-to-the-rest-api?apiVersion=2022-11-28).$$
sql: |
SELECT
{% set v2_exists = check_udf_api_v2_exists() %}
{% if v2_exists -%}
live.udf_api_v2(
'GET',
'https://api.github.com/octocat',
{'Authorization': 'Bearer {TOKEN}', 'X-GitHub-Api-Version': '2022-11-28', 'fsc-quantum-execution-mode': 'async'},
{},
IFF(_utils.udf_whoami() <> CURRENT_USER(), '_FSC_SYS/GITHUB', 'Vault/github/api'),
TRUE
)
{%- else -%}
live.udf_api(
'GET',
'https://api.github.com/octocat',
{'Authorization': 'Bearer {TOKEN}', 'X-GitHub-Api-Version': '2022-11-28'},
{},
IFF(_utils.udf_whoami() <> CURRENT_USER(), '_FSC_SYS/GITHUB', 'Vault/github/api')
)
{%- endif %}
as response
- name: {{ schema_name -}}.headers
signature: []
return_type:
- "TEXT"
options: |
NOT NULL
IMMUTABLE
MEMOIZABLE
sql: |
SELECT '{"Authorization": "Bearer {TOKEN}",
"X-GitHub-Api-Version": "2022-11-28",
"Accept": "application/vnd.github+json"
}'
- name: {{ schema_name -}}.get_api
signature:
- [route, "TEXT"]
- [query, "OBJECT"]
return_type:
- "VARIANT"
options: |
COMMENT = $$List all workflow runs for a workflow. You can replace workflow_id with the workflow file name. You can use parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow).$$
sql: |
SELECT
{% set v2_exists = check_udf_api_v2_exists() %}
{% if v2_exists -%}
live.udf_api_v2(
'GET',
CONCAT_WS('/', 'https://api.github.com', route || '?') || utils.udf_urlencode(query),
PARSE_JSON({{ schema_name -}}.headers()),
{},
IFF(_utils.udf_whoami() <> CURRENT_USER(), '_FSC_SYS/GITHUB', 'Vault/github/api'),
TRUE
)
{%- else -%}
live.udf_api(
'GET',
CONCAT_WS('/', 'https://api.github.com', route || '?') || utils.udf_urlencode(query),
PARSE_JSON({{ schema_name -}}.headers()),
{},
IFF(_utils.udf_whoami() <> CURRENT_USER(), '_FSC_SYS/GITHUB', 'Vault/github/api')
)
{%- endif %}
as response
- name: {{ schema_name -}}.post_api
signature:
- [route, "TEXT"]
- [data, "OBJECT"]
return_type:
- "VARIANT"
options: |
COMMENT = $$List all workflow runs for a workflow. You can replace workflow_id with the workflow file name. You can use parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow).$$
sql: |
SELECT
{% set v2_exists = check_udf_api_v2_exists() %}
{% if v2_exists -%}
live.udf_api_v2(
'POST',
CONCAT_WS('/', 'https://api.github.com', route),
PARSE_JSON({{ schema_name -}}.headers()),
data,
IFF(_utils.udf_whoami() <> CURRENT_USER(), '_FSC_SYS/GITHUB', 'Vault/github/api'),
TRUE
)
{%- else -%}
live.udf_api(
'POST',
CONCAT_WS('/', 'https://api.github.com', route),
PARSE_JSON({{ schema_name -}}.headers()),
data,
IFF(_utils.udf_whoami() <> CURRENT_USER(), '_FSC_SYS/GITHUB', 'Vault/github/api')
)
{%- endif %}
as response
- name: {{ schema_name -}}.put_api
signature:
- [route, "TEXT"]
- [data, "OBJECT"]
return_type:
- "VARIANT"
options: |
COMMENT = $$List all workflow runs for a workflow. You can replace workflow_id with the workflow file name. You can use parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow).$$
sql: |
SELECT
{% set v2_exists = check_udf_api_v2_exists() %}
{% if v2_exists -%}
live.udf_api_v2(
'PUT',
CONCAT_WS('/', 'https://api.github.com', route),
PARSE_JSON({{ schema_name -}}.headers()),
data,
IFF(_utils.udf_whoami() <> CURRENT_USER(), '_FSC_SYS/GITHUB', 'Vault/github/api'),
TRUE
)
{%- else -%}
live.udf_api(
'PUT',
CONCAT_WS('/', 'https://api.github.com', route),
PARSE_JSON({{ schema_name -}}.headers()),
data,
IFF(_utils.udf_whoami() <> CURRENT_USER(), '_FSC_SYS/GITHUB', 'Vault/github/api')
)
{%- endif %}
as response
{% endmacro %}