diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml new file mode 100644 index 0000000..da996d8 --- /dev/null +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -0,0 +1,46 @@ +name: dbt_alter_gha_task +run-name: dbt_alter_gha_task + +on: + workflow_dispatch: + branches: + - "main" + inputs: + workflow_name: + type: string + description: Name of the workflow to perform the action on, no .yml extension + required: true + task_action: + type: choice + description: Action to perform + required: true + options: + - SUSPEND + - RESUME + default: SUSPEND + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main + with: + workflow_name: | + ${{ inputs.workflow_name }} + task_action: | + ${{ inputs.task_action }} + environment: workflow_prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_lq_load.yml b/.github/workflows/dbt_run_lq_load.yml deleted file mode 100644 index 68a1c18..0000000 --- a/.github/workflows/dbt_run_lq_load.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: dbt_run_scheduled_lq_load -run-name: dbt_run_scheduled_lq_load - -on: - workflow_dispatch: - schedule: - # “At minute 3 past every 2nd hour.” - - cron: '3 */2 * * *' - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main - with: - dbt_command: > - dbt run-operation load_blocks_lq; dbt run-operation load_blocks_lq; dbt run-operation load_txs_lq; dbt run-operation load_txs_lq - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml index 44e3e9c..ab67e32 100644 --- a/.github/workflows/dbt_run_observability.yml +++ b/.github/workflows/dbt_run_observability.yml @@ -3,8 +3,8 @@ run-name: dbt_run_observability on: workflow_dispatch: - schedule: - - cron: '15 0,4,8,12,17,20 * * *' + # schedule: + # - cron: '15 0,4,8,12,17,20 * * *' env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index dd35b7d..c6a1934 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -3,9 +3,9 @@ run-name: dbt_run_scheduled on: workflow_dispatch: - schedule: - # Runs 0700 daily (see https://crontab.guru) - - cron: '20,50 * * * *' + # schedule: + # # Runs 0700 daily (see https://crontab.guru) + # - cron: '20,50 * * * *' env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_streamline.yml b/.github/workflows/dbt_run_streamline.yml index 1c36c55..8ca515e 100644 --- a/.github/workflows/dbt_run_streamline.yml +++ b/.github/workflows/dbt_run_streamline.yml @@ -3,9 +3,9 @@ run-name: dbt_run_streamline on: workflow_dispatch: - schedule: - # Runs "every hour at the 0,30 minute" (see https://crontab.guru) - - cron: '0,30 */1 * * *' + # schedule: + # # Runs "every hour at the 0,30 minute" (see https://crontab.guru) + # - cron: '0,30 */1 * * *' env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml new file mode 100644 index 0000000..976440b --- /dev/null +++ b/.github/workflows/dbt_test_tasks.yml @@ -0,0 +1,27 @@ +name: dbt_test_tasks +run-name: dbt_test_tasks + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main + secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv new file mode 100644 index 0000000..d2b8af5 --- /dev/null +++ b/data/github_actions__workflows.csv @@ -0,0 +1,4 @@ +workflow_name,workflow_schedule +dbt_run_observability,"15 0,4,8,12,17,20 * * *" +dbt_run_scheduled,"6,18,30,42,54 * * * *" +dbt_run_streamline,"0,12,24,36,48 * * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 11b5712..a3c29ac 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -56,6 +56,7 @@ vars: UPDATE_UDFS_AND_SPS: False UPDATE_SNOWFLAKE_TAGS: True OBSERV_FULL_TEST: FALSE + START_GHA_TASKS: False dispatch: diff --git a/models/github_actions/github_actions__current_task_status.sql b/models/github_actions/github_actions__current_task_status.sql new file mode 100644 index 0000000..577a226 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.sql @@ -0,0 +1,6 @@ +{{ config( + materialized = 'view', + tags = ['gha_tasks'] +) }} + +{{ fsc_utils.gha_task_current_status_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__current_task_status.yml b/models/github_actions/github_actions__current_task_status.yml new file mode 100644 index 0000000..e80ad12 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.yml @@ -0,0 +1,17 @@ +version: 2 +models: + - name: github_actions__current_task_status + columns: + - name: PIPELINE_ACTIVE + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - TRUE + - name: SUCCESSES + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - 204 + config: + severity: warn + warn_if: ">0" \ No newline at end of file diff --git a/models/github_actions/github_actions__task_history.sql b/models/github_actions/github_actions__task_history.sql new file mode 100644 index 0000000..9c35ce7 --- /dev/null +++ b/models/github_actions/github_actions__task_history.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_history_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_performance.sql b/models/github_actions/github_actions__task_performance.sql new file mode 100644 index 0000000..117ded5 --- /dev/null +++ b/models/github_actions/github_actions__task_performance.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_performance_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_schedule.sql b/models/github_actions/github_actions__task_schedule.sql new file mode 100644 index 0000000..ff95a44 --- /dev/null +++ b/models/github_actions/github_actions__task_schedule.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_schedule_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__tasks.sql b/models/github_actions/github_actions__tasks.sql new file mode 100644 index 0000000..feab82a --- /dev/null +++ b/models/github_actions/github_actions__tasks.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_tasks_view() }} \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index b60eace..621988c 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -98,4 +98,9 @@ sources: database: crosschain schema: core tables: - - name: dim_date_hours \ No newline at end of file + - name: dim_date_hours + - name: github_actions + database: terra + schema: github_actions + tables: + - name: workflows \ No newline at end of file diff --git a/packages.yml b/packages.yml index 9ceecf7..d22b56e 100644 --- a/packages.yml +++ b/packages.yml @@ -4,6 +4,6 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: "v1.9.3" + revision: "v1.11.0" - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"] \ No newline at end of file