From 2734d36d7cb66b47c3edc3a4460ab51fb72b157b Mon Sep 17 00:00:00 2001 From: desmond-hui <97470747+desmond-hui@users.noreply.github.com> Date: Tue, 7 Nov 2023 08:17:36 -0800 Subject: [PATCH] Dispatch gha via snowflake (#388) * update package * setup GHA dispatches via Snowflake tasks * remove unused workflows * add model qualifier to only run models in solana * include task status monitoring workflow --- .github/workflows/dbt_alter_gha_tasks.yml | 46 +++++++++++++++++++ ...dbt_backfill_instruction_decode_hourly.yml | 38 --------------- .github/workflows/dbt_run_batch_backfill.yml | 44 ------------------ .github/workflows/dbt_run_incremental.yml | 7 ++- .github/workflows/dbt_test.yml | 2 +- .github/workflows/dbt_test_tasks.yml | 27 +++++++++++ data/github_actions__workflows.csv | 3 ++ dbt_project.yml | 7 +-- .../github_actions__current_task_status.sql | 6 +++ .../github_actions__current_task_status.yml | 17 +++++++ .../github_actions__task_history.sql | 5 ++ .../github_actions__task_performance.sql | 5 ++ .../github_actions__task_schedule.sql | 5 ++ .../github_actions/github_actions__tasks.sql | 5 ++ models/sources.yml | 7 ++- packages.yml | 2 +- 16 files changed, 134 insertions(+), 92 deletions(-) create mode 100644 .github/workflows/dbt_alter_gha_tasks.yml delete mode 100644 .github/workflows/dbt_backfill_instruction_decode_hourly.yml delete mode 100644 .github/workflows/dbt_run_batch_backfill.yml create mode 100644 .github/workflows/dbt_test_tasks.yml create mode 100644 data/github_actions__workflows.csv create mode 100644 models/github_actions/github_actions__current_task_status.sql create mode 100644 models/github_actions/github_actions__current_task_status.yml create mode 100644 models/github_actions/github_actions__task_history.sql create mode 100644 models/github_actions/github_actions__task_performance.sql create mode 100644 models/github_actions/github_actions__task_schedule.sql create mode 100644 models/github_actions/github_actions__tasks.sql diff --git a/.github/workflows/dbt_alter_gha_tasks.yml b/.github/workflows/dbt_alter_gha_tasks.yml new file mode 100644 index 00000000..d138700c --- /dev/null +++ b/.github/workflows/dbt_alter_gha_tasks.yml @@ -0,0 +1,46 @@ +name: dbt_run_alter_gha_task +run-name: dbt_run_alter_gha_task + +on: + workflow_dispatch: + branches: + - "main" + inputs: + workflow_name: + type: string + description: Name of the workflow to perform the action on, no .yml extension + required: true + task_action: + type: choice + description: Action to perform + required: true + options: + - SUSPEND + - RESUME + default: SUSPEND + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main + with: + workflow_name: | + ${{ inputs.workflow_name }} + task_action: | + ${{ inputs.task_action }} + environment: workflow_prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_backfill_instruction_decode_hourly.yml b/.github/workflows/dbt_backfill_instruction_decode_hourly.yml deleted file mode 100644 index c817f8b9..00000000 --- a/.github/workflows/dbt_backfill_instruction_decode_hourly.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: dbt_backfill_instruction_decode_hourly -run-name: dbt_backfill_instruction_decode_hourly - -on: - workflow_dispatch: - schedule: - # Run at xx:8 UTC hourly (see https://crontab.guru) - - cron: '8 * * * *' - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main - with: - dbt_command: | - dbt run-operation run_sp_bulk_decode_instructions; - dbt run-operation run_sp_bulk_decode_instructions; - dbt run-operation run_sp_bulk_decode_instructions; - dbt run-operation run_sp_bulk_decode_instructions; - dbt run-operation run_sp_bulk_decode_instructions; - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit diff --git a/.github/workflows/dbt_run_batch_backfill.yml b/.github/workflows/dbt_run_batch_backfill.yml deleted file mode 100644 index 589d0cab..00000000 --- a/.github/workflows/dbt_run_batch_backfill.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_scheduled_batch_backfill -run-name: dbt_run_scheduled_batch_backfill - -on: - workflow_dispatch: - # schedule: - # # Runs every 5 mins, adjust to appropriate schedule as needed - # - cron: '*/5 * * * *' - -env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: DBT_EMERGENCY - SCHEMA: "${{ secrets.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run -s models/silver/accounts diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml index e0291750..85ff4401 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -3,9 +3,8 @@ run-name: dbt_run_scheduled on: workflow_dispatch: - schedule: - # Runs "every hour" (see https://crontab.guru) - - cron: '1,16,31,46 * * * *' + branches: + - "main" env: DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" @@ -42,5 +41,5 @@ jobs: - name: Run DBT Jobs run: | dbt run-operation run_sp_refresh_external_tables_full - dbt run -s ./models --exclude tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql models/silver/validator models/silver/_observability models/silver/nfts/silver__nft_compressed_mints_onchain.sql models/silver/nfts/silver__nft_compressed_mints.sql models/gold/core/core__ez_signers.sql tag:helius tag:bronze_api + dbt run -s "solana_models,./models" --exclude tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql models/silver/validator models/silver/_observability models/silver/nfts/silver__nft_compressed_mints_onchain.sql models/silver/nfts/silver__nft_compressed_mints.sql models/gold/core/core__ez_signers.sql tag:helius tag:bronze_api models/github_actions diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index 24c07e99..71e04bcb 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt test -s ./models \ No newline at end of file + dbt test -s "solana_models,./models" \ No newline at end of file diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml new file mode 100644 index 00000000..976440bd --- /dev/null +++ b/.github/workflows/dbt_test_tasks.yml @@ -0,0 +1,27 @@ +name: dbt_test_tasks +run-name: dbt_test_tasks + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main + secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv new file mode 100644 index 00000000..0207ece5 --- /dev/null +++ b/data/github_actions__workflows.csv @@ -0,0 +1,3 @@ +workflow_name,workflow_schedule +dbt_run_scheduled,"1,16,31,46 * * * *" +dbt_test_tasks,"0,30 * * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 8c98fdff..bcdc16db 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -69,9 +69,10 @@ models: vars: "dbt_date:time_zone": GMT - UPDATE_SNOWFLAKE_TAGS: True - UPDATE_UDFS_AND_SPS: False + OBSERV_FULL_TEST: False + START_GHA_TASKS: False STREAMLINE_INVOKE_STREAMS: False STREAMLINE_RUN_HISTORY: False STREAMLINE_RETRY_UNKNOWN: False - OBSERV_FULL_TEST: False \ No newline at end of file + UPDATE_SNOWFLAKE_TAGS: True + UPDATE_UDFS_AND_SPS: False \ No newline at end of file diff --git a/models/github_actions/github_actions__current_task_status.sql b/models/github_actions/github_actions__current_task_status.sql new file mode 100644 index 00000000..577a2266 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.sql @@ -0,0 +1,6 @@ +{{ config( + materialized = 'view', + tags = ['gha_tasks'] +) }} + +{{ fsc_utils.gha_task_current_status_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__current_task_status.yml b/models/github_actions/github_actions__current_task_status.yml new file mode 100644 index 00000000..e80ad122 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.yml @@ -0,0 +1,17 @@ +version: 2 +models: + - name: github_actions__current_task_status + columns: + - name: PIPELINE_ACTIVE + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - TRUE + - name: SUCCESSES + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - 204 + config: + severity: warn + warn_if: ">0" \ No newline at end of file diff --git a/models/github_actions/github_actions__task_history.sql b/models/github_actions/github_actions__task_history.sql new file mode 100644 index 00000000..9c35ce78 --- /dev/null +++ b/models/github_actions/github_actions__task_history.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_history_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_performance.sql b/models/github_actions/github_actions__task_performance.sql new file mode 100644 index 00000000..117ded55 --- /dev/null +++ b/models/github_actions/github_actions__task_performance.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_performance_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_schedule.sql b/models/github_actions/github_actions__task_schedule.sql new file mode 100644 index 00000000..ff95a44b --- /dev/null +++ b/models/github_actions/github_actions__task_schedule.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_schedule_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__tasks.sql b/models/github_actions/github_actions__tasks.sql new file mode 100644 index 00000000..feab82a8 --- /dev/null +++ b/models/github_actions/github_actions__tasks.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_tasks_view() }} \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index e0f2ba64..94a1b81f 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -92,4 +92,9 @@ sources: database: solana schema: bronze_api tables: - - name: parse_compressed_nft_mints_requests \ No newline at end of file + - name: parse_compressed_nft_mints_requests + - name: github_actions + database: solana + schema: github_actions + tables: + - name: workflows \ No newline at end of file diff --git a/packages.yml b/packages.yml index ff6d559a..cc492744 100644 --- a/packages.yml +++ b/packages.yml @@ -6,6 +6,6 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: "v1.3" + revision: "v1.11.0" - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"]