STREAM-649/github-actions-api (#50)

This commit is contained in:
Julius Remigio 2023-09-29 13:25:47 -07:00 committed by GitHub
parent 8806013181
commit 04ebabb296
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 472 additions and 8 deletions

View File

@ -67,6 +67,7 @@ vars:
STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False
REST_API_PREFIX_PROD: bqco8lkjsb.execute-api.us-east-1.amazonaws.com/prod/
REST_API_PREFIX_DEV: u5z0tu43sc.execute-api.us-east-1.amazonaws.com/stg/
API_INTEGRATION: AWS_LIVE_QUERY{{ '_STG' if target.name != 'prod' else '' }}
API_INTEGRATION: AWS_LIVE_QUERY{{ "_STG" if target.name != "prod" else "" }}
AWS_REGION: us-east-1
STUDIO_TEST_USER_ID: 98d15c30-9fa5-43cd-9c69-3d4c0bb269f5
STUDIO_TEST_USER_ID: '{{ env_var("STUDIO_TEST_USER_ID", "98d15c30-9fa5-43cd-9c69-3d4c0bb269f5") }}'

View File

@ -148,6 +148,7 @@
options: |
NULL
LANGUAGE PYTHON
RETURNS NULL ON NULL INPUT
IMMUTABLE
RUNTIME_VERSION = '3.8'
COMMENT=$$Pthon (function)[https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode] to convert an object to a URL query string.$$
@ -162,6 +163,7 @@
options: |
NULL
LANGUAGE PYTHON
RETURNS NULL ON NULL INPUT
IMMUTABLE
RUNTIME_VERSION = '3.8'
COMMENT=$$Pthon (function)[https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode] to convert an array to a URL query string.$$

View File

@ -162,9 +162,8 @@
{%- do log("Deploy core udfs: " ~ this.database ~ "." ~ this.schema, true) -%}
{%- endif -%}
{%- do run_query(sql ~ apply_grants_by_schema(this.schema)) -%}
{% else -%}
SELECT '{{ model.schema }}' as schema_
{%- endif -%}
SELECT '{{ model.schema }}' as schema_
{%- endmacro -%}
{% macro ephemeral_deploy(configs) %}
@ -187,9 +186,8 @@
{%- do log("Deploy partner udfs: " ~ this.database ~ "." ~ schema, true) -%}
{%- endif -%}
{%- do run_query(sql ~ apply_grants_by_schema(schema)) -%}
{% else -%}
SELECT '{{ model.schema }}' as schema_
{%- endif -%}
SELECT '{{ model.schema }}' as schema_
{%- endmacro -%}
{% macro ephemeral_deploy_marketplace(configs) %}
@ -211,8 +209,7 @@
{%- do log("Deploy marketplace udfs: " ~ this.database ~ "." ~ schema, true) -%}
{%- endif -%}
{%- do run_query(sql ~ apply_grants_by_schema(schema)) -%}
{% else -%}
SELECT '{{ model.schema }}' as schema_
{%- endif -%}
SELECT '{{ model.schema }}' as schema_
{%- endmacro -%}

View File

@ -0,0 +1,148 @@
{% macro config_github_actions_udfs(schema_name = "github_actions", utils_schema_name = "github_utils") -%}
{#
This macro is used to generate the Github API Calls
#}
- name: {{ schema_name -}}.workflows
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [query, "OBJECT"]
return_type:
- "OBJECT"
options: |
COMMENT = $$[List repository workflows](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#list-repository-workflows).$$
sql: |
SELECT
{{ utils_schema_name }}.GET(
CONCAT_WS('/', 'repos', owner, repo, 'actions/workflows'),
query
):data::OBJECT
- name: {{ schema_name -}}.workflows
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
return_type:
- "OBJECT"
options: |
COMMENT = $$[List repository workflows](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#list-repository-workflows).$$
sql: |
SELECT
{{ schema_name -}}.workflows(owner, repo, {})
- name: {{ schema_name -}}.runs
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [query, "OBJECT"]
return_type:
- "OBJECT"
options: |
COMMENT = $$Lists all workflow runs for a repository. You can use query parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository).$$
sql: |
SELECT
{{ utils_schema_name }}.GET(
CONCAT_WS('/', 'repos', owner, repo, 'actions/runs'),
query
):data::OBJECT
- name: {{ schema_name -}}.runs
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
return_type:
- "OBJECT"
options: |
COMMENT = $$Lists all workflow runs for a repository. You can use query parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository).$$
sql: |
SELECT
{{ schema_name -}}.runs(owner, repo, {})
- name: {{ schema_name -}}.workflow_runs
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [workflow_id, "TEXT"]
- [query, "OBJECT"]
return_type:
- "OBJECT"
options: |
COMMENT = $$List all workflow runs for a workflow. You can replace workflow_id with the workflow file name. You can use query parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow).$$
sql: |
SELECT
{{ utils_schema_name }}.GET(
CONCAT_WS('/', 'repos', owner, repo, 'actions/workflows', workflow_id, 'runs'),
query
):data::OBJECT
- name: {{ schema_name -}}.workflow_runs
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [workflow_id, "TEXT"]
return_type:
- "OBJECT"
options: |
COMMENT = $$List all workflow runs for a workflow. You can replace workflow_id with the workflow file name. You can use query parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow).$$
sql: |
SELECT
{{ schema_name -}}.workflow_runs(owner, repo, workflow_id, {})
- name: {{ schema_name -}}.workflow_dispatches
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [workflow_id, "TEXT"]
- [body, "OBJECT"]
return_type:
- "OBJECT"
options: |
COMMENT = $$You can use this endpoint to manually trigger a GitHub Actions workflow run. You can replace workflow_id with the workflow file name. For example, you could use main.yaml. [Docs](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#create-a-workflow-dispatch-event).$$
sql: |
SELECT
{{ utils_schema_name }}.POST(
CONCAT_WS('/', 'repos', owner, repo, 'actions/workflows', workflow_id, 'dispatches'),
COALESCE(body, {'ref': 'main'})::OBJECT
)::OBJECT
- name: {{ schema_name -}}.workflow_dispatches
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [workflow_id, "TEXT"]
return_type:
- "OBJECT"
options: |
COMMENT = $$You can use this endpoint to manually trigger a GitHub Actions workflow run. You can replace workflow_id with the workflow file name. For example, you could use main.yaml. [Docs](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#create-a-workflow-dispatch-event).$$
sql: |
SELECT
{{ schema_name -}}.workflow_dispatches(owner, repo, workflow_id, NULL)
- name: {{ schema_name -}}.workflow_enable
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [workflow_id, "TEXT"]
return_type:
- "OBJECT"
options: |
COMMENT = $$Enables a workflow. You can replace workflow_id with the workflow file name. For example, you could use main.yaml. [Docs](https://docs.github.com/en/rest/reference/actions#enable-a-workflow).$$
sql: |
SELECT
{{ utils_schema_name }}.PUT(
CONCAT_WS('/', 'repos', owner, repo, 'actions/workflows', workflow_id, 'enable'),
{}
)::OBJECT
- name: {{ schema_name -}}.workflow_disable
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [workflow_id, "TEXT"]
return_type:
- "OBJECT"
options: |
COMMENT = $$Disables a workflow. You can replace workflow_id with the workflow file name. For example, you could use main.yaml. [Docs](https://docs.github.com/en/rest/reference/actions#disable-a-workflow).$$
sql: |
SELECT
{{ utils_schema_name }}.PUT(
CONCAT_WS('/', 'repos', owner, repo, 'actions/workflows', workflow_id, 'disable'),
{}
)::OBJECT
{% endmacro %}

View File

@ -0,0 +1,169 @@
{% macro config_github_actions_udtfs(schema_name = "github_actions", utils_schema_name = "github_utils") -%}
{#
This macro is used to generate the Github API Calls
#}
- name: {{ schema_name -}}.tf_workflows
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [query, "OBJECT"]
return_type:
- "TABLE(id INTEGER, badge_url STRING, created_at TIMESTAMP, html_url STRING, name STRING, node_id STRING, path STRING, state STRING, updated_at TIMESTAMP, url STRING)"
options: |
COMMENT = $$[List repository workflows](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#list-repository-workflows).$$
sql: |
WITH workflows AS
(
SELECT
github_actions.workflows(OWNER, REPO, QUERY) AS response
)
SELECT
value:id::INTEGER AS id
,value:badge_url::STRING AS badge_url
,value:created_at::TIMESTAMP AS created_at
,value:html_url::STRING AS html_url
,value:name::STRING AS name
,value:node_id::STRING AS node_id
,value:path::STRING AS path
,value:state::STRING AS state
,value:updated_at::TIMESTAMP AS updated_at
,value:url::STRING AS url
FROM workflows, LATERAL FLATTEN( input=> response:workflows)
- name: {{ schema_name -}}.tf_workflows
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
return_type:
- "TABLE(id INTEGER, badge_url STRING, created_at TIMESTAMP, html_url STRING, name STRING, node_id STRING, path STRING, state STRING, updated_at TIMESTAMP, url STRING)"
options: |
COMMENT = $$[List repository workflows](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#list-repository-workflows).$$
sql: |
SELECT *
FROM TABLE({{ schema_name -}}.tf_workflows(owner, repo, {}))
- name: {{ schema_name -}}.tf_runs
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [query, "OBJECT"]
return_type:
- "TABLE(id NUMBER, name STRING, node_id STRING, check_suite_id NUMBER, check_suite_node_id STRING, head_branch STRING, head_sha STRING, run_number NUMBER, event STRING, display_title STRING, status STRING, conclusion STRING, workflow_id NUMBER, url STRING, html_url STRING, pull_requests STRING, created_at TIMESTAMP, updated_at TIMESTAMP, actor OBJECT, run_attempt STRING, run_started_at TIMESTAMP, triggering_actor OBJECT, jobs_url STRING, logs_url STRING, check_suite_url STRING, artifacts_url STRING, cancel_url STRING, rerun_url STRING, workflow_url STRING, head_commit OBJECT, repository OBJECT, head_repository OBJECT)"
options: |
COMMENT = $$Lists all workflow runs for a repository. You can use query parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository).$$
sql: |
WITH response AS
(
SELECT
github_actions.runs(OWNER, REPO, QUERY) AS response
)
SELECT
value:id::NUMBER AS id
,value:name::STRING AS name
,value:node_id::STRING AS node_id
,value:check_suite_id::NUMBER AS check_suite_id
,value:check_suite_node_id::STRING AS check_suite_node_id
,value:head_branch::STRING AS head_branch
,value:head_sha::STRING AS head_sha
,value:run_number::NUMBER AS run_number
,value:event::STRING AS event
,value:display_title::STRING AS display_title
,value:status::STRING AS status
,value:conclusion::STRING AS conclusion
,value:workflow_id::NUMBER AS workflow_id
,value:url::STRING AS url
,value:html_url::STRING AS html_url
,value:pull_requests::STRING AS pull_requests
,value:created_at::TIMESTAMP AS created_at
,value:updated_at::TIMESTAMP AS updated_at
,value:actor::OBJECT AS actor
,value:run_attempt::STRING AS run_attempt
,value:run_started_at::TIMESTAMP AS run_started_at
,value:triggering_actor::OBJECT AS triggering_actor
,value:jobs_url::STRING AS jobs_url
,value:logs_url::STRING AS logs_url
,value:check_suite_url::STRING AS check_suite_url
,value:artifacts_url::STRING AS artifacts_url
,value:cancel_url::STRING AS cancel_url
,value:rerun_url::STRING AS rerun_url
,value:workflow_url::STRING AS workflow_url
,value:head_commit::OBJECT AS head_commit
,value:repository::OBJECT AS repository
,value:head_repository::OBJECT AS head_repository
FROM response, LATERAL FLATTEN( input=> response:workflow_runs)
- name: {{ schema_name -}}.tf_runs
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
return_type:
- "TABLE(id NUMBER, name STRING, node_id STRING, check_suite_id NUMBER, check_suite_node_id STRING, head_branch STRING, head_sha STRING, run_number NUMBER, event STRING, display_title STRING, status STRING, conclusion STRING, workflow_id NUMBER, url STRING, html_url STRING, pull_requests STRING, created_at TIMESTAMP, updated_at TIMESTAMP, actor OBJECT, run_attempt STRING, run_started_at TIMESTAMP, triggering_actor OBJECT, jobs_url STRING, logs_url STRING, check_suite_url STRING, artifacts_url STRING, cancel_url STRING, rerun_url STRING, workflow_url STRING, head_commit OBJECT, repository OBJECT, head_repository OBJECT)"
options: |
COMMENT = $$Lists all workflow runs for a repository. You can use query parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository).$$
sql: |
SELECT *
FROM TABLE({{ schema_name -}}.tf_runs(owner, repo, {}))
- name: {{ schema_name -}}.tf_workflow_runs
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [workflkow_id, "TEXT"]
- [query, "OBJECT"]
return_type:
- "TABLE(id NUMBER, name STRING, node_id STRING, check_suite_id NUMBER, check_suite_node_id STRING, head_branch STRING, head_sha STRING, run_number NUMBER, event STRING, display_title STRING, status STRING, conclusion STRING, workflow_id NUMBER, url STRING, html_url STRING, pull_requests STRING, created_at TIMESTAMP, updated_at TIMESTAMP, actor OBJECT, run_attempt STRING, run_started_at TIMESTAMP, triggering_actor OBJECT, jobs_url STRING, logs_url STRING, check_suite_url STRING, artifacts_url STRING, cancel_url STRING, rerun_url STRING, workflow_url STRING, head_commit OBJECT, repository OBJECT, head_repository OBJECT)"
options: |
COMMENT = $$Lists all workflow runs for a repository. You can use query parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository).$$
sql: |
WITH response AS
(
SELECT
github_actions.workflow_runs(OWNER, REPO, WORKFLKOW_ID, QUERY) AS response
)
SELECT
value:id::NUMBER AS id
,value:name::STRING AS name
,value:node_id::STRING AS node_id
,value:check_suite_id::NUMBER AS check_suite_id
,value:check_suite_node_id::STRING AS check_suite_node_id
,value:head_branch::STRING AS head_branch
,value:head_sha::STRING AS head_sha
,value:run_number::NUMBER AS run_number
,value:event::STRING AS event
,value:display_title::STRING AS display_title
,value:status::STRING AS status
,value:conclusion::STRING AS conclusion
,value:workflow_id::NUMBER AS workflow_id
,value:url::STRING AS url
,value:html_url::STRING AS html_url
,value:pull_requests::STRING AS pull_requests
,value:created_at::TIMESTAMP AS created_at
,value:updated_at::TIMESTAMP AS updated_at
,value:actor::OBJECT AS actor
,value:run_attempt::STRING AS run_attempt
,value:run_started_at::TIMESTAMP AS run_started_at
,value:triggering_actor::OBJECT AS triggering_actor
,value:jobs_url::STRING AS jobs_url
,value:logs_url::STRING AS logs_url
,value:check_suite_url::STRING AS check_suite_url
,value:artifacts_url::STRING AS artifacts_url
,value:cancel_url::STRING AS cancel_url
,value:rerun_url::STRING AS rerun_url
,value:workflow_url::STRING AS workflow_url
,value:head_commit::OBJECT AS head_commit
,value:repository::OBJECT AS repository
,value:head_repository::OBJECT AS head_repository
FROM response, LATERAL FLATTEN( input=> response:workflow_runs)
- name: {{ schema_name -}}.tf_workflow_runs
signature:
- [owner, "TEXT"]
- [repo, "TEXT"]
- [workflkow_id, "TEXT"]
return_type:
- "TABLE(id NUMBER, name STRING, node_id STRING, check_suite_id NUMBER, check_suite_node_id STRING, head_branch STRING, head_sha STRING, run_number NUMBER, event STRING, display_title STRING, status STRING, conclusion STRING, workflow_id NUMBER, url STRING, html_url STRING, pull_requests STRING, created_at TIMESTAMP, updated_at TIMESTAMP, actor OBJECT, run_attempt STRING, run_started_at TIMESTAMP, triggering_actor OBJECT, jobs_url STRING, logs_url STRING, check_suite_url STRING, artifacts_url STRING, cancel_url STRING, rerun_url STRING, workflow_url STRING, head_commit OBJECT, repository OBJECT, head_repository OBJECT)"
options: |
COMMENT = $$Lists all workflow runs for a repository. You can use query parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository).$$
sql: |
SELECT *
FROM TABLE({{ schema_name -}}.tf_workflow_runs(owner, repo, WORKFLKOW_ID, {}))
{% endmacro %}

View File

@ -0,0 +1,87 @@
{% macro config_github_utils_udfs(schema_name = "github_utils", utils_schema_name = "github_utils") -%}
{#
This macro is used to generate the Github API Calls
#}
- name: {{ schema_name -}}.octocat
signature:
- []
return_type:
- "VARIANT"
options: |
COMMENT = $$Verify token [Authenticating to the REST API](https://docs.github.com/en/rest/overview/authenticating-to-the-rest-api?apiVersion=2022-11-28).$$
sql: |
SELECT
live.udf_api(
'GET',
'https://api.github.com/octocat',
{'Authorization': 'Bearer {TOKEN}',
'X-GitHub-Api-Version': '2022-11-28'},
{},
'_FSC_SYS/GITHUB'
) as response
- name: {{ schema_name -}}.headers
signature: []
return_type:
- "TEXT"
options: |
NOT NULL
IMMUTABLE
MEMOIZABLE
sql: |
SELECT '{"Authorization": "Bearer {TOKEN}",
"X-GitHub-Api-Version": "2022-11-28",
"Accept": "application/vnd.github+json"}'
- name: {{ schema_name -}}.get
signature:
- [route, "TEXT"]
- [query, "OBJECT"]
return_type:
- "VARIANT"
options: |
COMMENT = $$List all workflow runs for a workflow. You can replace workflow_id with the workflow file name. You can use parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow).$$
sql: |
SELECT
live.udf_api(
'GET',
CONCAT_WS('/', 'https://api.github.com', route || '?') || utils.udf_urlencode(query),
PARSE_JSON({{ schema_name -}}.headers()),
{},
'_FSC_SYS/GITHUB'
)
- name: {{ schema_name -}}.post
signature:
- [route, "TEXT"]
- [data, "OBJECT"]
return_type:
- "VARIANT"
options: |
COMMENT = $$List all workflow runs for a workflow. You can replace workflow_id with the workflow file name. You can use parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow).$$
sql: |
SELECT
live.udf_api(
'POST',
CONCAT_WS('/', 'https://api.github.com', route),
PARSE_JSON({{ schema_name -}}.headers()),
data,
'_FSC_SYS/GITHUB'
)
- name: {{ schema_name -}}.put
signature:
- [route, "TEXT"]
- [data, "OBJECT"]
return_type:
- "VARIANT"
options: |
COMMENT = $$List all workflow runs for a workflow. You can replace workflow_id with the workflow file name. You can use parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow).$$
sql: |
SELECT
live.udf_api(
'PUT',
CONCAT_WS('/', 'https://api.github.com', route),
PARSE_JSON({{ schema_name -}}.headers()),
data,
'_FSC_SYS/GITHUB'
)
{% endmacro %}

View File

@ -0,0 +1,7 @@
-- depends_on: {{ ref('live') }}
-- depends_on: {{ ref('github_utils__github_utils') }}
{%- set configs = [
config_github_actions_udfs,
config_github_actions_udtfs,
] -%}
{{- ephemeral_deploy_marketplace(configs) -}}

View File

@ -0,0 +1,37 @@
version: 2
models:
- name: github_actions__github_utils
columns:
- name: workflows
tests:
- test_udf:
name: test_github_actions__workflows_status_200
args: >
'FlipsideCrypto',
'admin-models'
assertions:
- result:status_code = 200
- result:error IS NULL
- name: runs
tests:
- test_udf:
name: test_github_actions__runs_status_200
args: >
'FlipsideCrypto',
'admin-models',
{}
assertions:
- result:status_code = 200
- result:error IS NULL
- name: workflow_runs
tests:
- test_udf:
name: test_github_actions__workflow_runs_status_200
args: >
'FlipsideCrypto',
'admin-models',
'dbt_run_dev_refresh.yml',
{}
assertions:
- result:status_code = 200
- result:error IS NULL

View File

@ -0,0 +1,5 @@
-- depends_on: {{ ref('live') }}
{%- set configs = [
config_github_utils_udfs,
] -%}
{{- ephemeral_deploy_marketplace(configs) -}}

View File

@ -0,0 +1,11 @@
version: 2
models:
- name: github_utils__github_utils
columns:
- name: octocat
tests:
- test_udf:
name: test_github_utils__octocat_status_200
assertions:
- result:status_code = 200
- result:error IS NULL