From 9c5faa8dbff6ab24be8c93f1e46e096f6ffc72dc Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 5 Jan 2023 11:47:16 -0700 Subject: [PATCH] workflow config (#100) * workflow config * github actions workflow * small tweak * env update for ad hoc * daily job --- {.dbt => .github/workflows/.dbt}/profiles.yml | 10 ++- .../{docs_update.yml => dbt_docs_update.yml} | 30 ++++----- .github/workflows/dbt_run_adhoc.yml | 65 +++++++++++++++++++ .../workflows/dbt_run_daily_dev_refresh.yml | 42 ++++++++++++ .github/workflows/dbt_run_hourly.yml | 44 +++++++++++++ .../workflows/dbt_run_moments_metadata.yml | 43 ++++++++++++ .github/workflows/dbt_test.yml | 41 ++++++++++++ .gitignore | 2 +- 8 files changed, 253 insertions(+), 24 deletions(-) rename {.dbt => .github/workflows/.dbt}/profiles.yml (71%) rename .github/workflows/{docs_update.yml => dbt_docs_update.yml} (69%) create mode 100644 .github/workflows/dbt_run_adhoc.yml create mode 100644 .github/workflows/dbt_run_daily_dev_refresh.yml create mode 100644 .github/workflows/dbt_run_hourly.yml create mode 100644 .github/workflows/dbt_run_moments_metadata.yml create mode 100644 .github/workflows/dbt_test.yml diff --git a/.dbt/profiles.yml b/.github/workflows/.dbt/profiles.yml similarity index 71% rename from .dbt/profiles.yml rename to .github/workflows/.dbt/profiles.yml index ad81399..bbb4d98 100644 --- a/.dbt/profiles.yml +++ b/.github/workflows/.dbt/profiles.yml @@ -8,12 +8,11 @@ flow: user: "{{ env_var('USER') }}" password: "{{ env_var('PASSWORD') }}" region: "{{ env_var('REGION') }}" - database: "{{ env_var('DATABASE_DEV') }}" - warehouse: "{{ env_var('WAREHOUSE_DEV') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" schema: SILVER threads: 4 client_session_keep_alive: False - query_tag: gh_actions prod: type: snowflake account: "{{ env_var('ACCOUNT') }}" @@ -21,9 +20,8 @@ flow: user: "{{ env_var('USER') }}" password: "{{ env_var('PASSWORD') }}" region: "{{ env_var('REGION') }}" - database: "{{ env_var('DATABASE_PROD') }}" - warehouse: "{{ env_var('WAREHOUSE_PROD') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" schema: silver threads: 4 client_session_keep_alive: False - query_tag: gh_actions diff --git a/.github/workflows/docs_update.yml b/.github/workflows/dbt_docs_update.yml similarity index 69% rename from .github/workflows/docs_update.yml rename to .github/workflows/dbt_docs_update.yml index 85a91f3..64e42b6 100644 --- a/.github/workflows/docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -1,4 +1,4 @@ -name: docs_update +name: dbt_docs_update on: push: @@ -6,20 +6,23 @@ on: - "main" env: + DBT_PROFILES_DIR: .dbt/ + ACCOUNT: "${{ secrets.ACCOUNT }}" ROLE: "${{ secrets.ROLE }}" USER: "${{ secrets.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" REGION: "${{ secrets.REGION }}" - DATABASE_DEV: "${{ secrets.DATABASE_DEV }}" - DATABASE_PROD: "${{ secrets.DATABASE_PROD }}" - WAREHOUSE_DEV: "${{ secrets.WAREHOUSE_DEV }}" - WAREHOUSE_PROD: "${{ secrets.WAREHOUSE_PROD }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" jobs: scheduled_run: name: docs_update runs-on: ubuntu-latest + environment: + name: workflow_prod steps: - uses: actions/checkout@v3 @@ -30,25 +33,19 @@ jobs: - name: install dependencies run: | - pip install dbt-snowflake + pip install dbt-snowflake==1.3.0 dbt deps - - - name: delete existing docs branch - run: git push origin --delete docs || true - - name: checkout docs branch run: | - git checkout -b docs - git push --set-upstream origin docs - + git checkout -B docs origin/main + - name: generate dbt docs - run: dbt docs generate -t prod --profiles-dir ./.dbt + run: dbt docs generate -t prod - name: move files to docs directory run: | mkdir -p ./docs cp target/{catalog.json,manifest.json,index.html} docs/ - - name: clean up target directory run: dbt clean @@ -63,7 +60,6 @@ jobs: git config user.email "abc@xyz" git config user.name "github-actions" git commit -am "Auto-update docs" - - name: push changes to docs run: | - git push + git push -f --set-upstream origin docs diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml new file mode 100644 index 0000000..1f85644 --- /dev/null +++ b/.github/workflows/dbt_run_adhoc.yml @@ -0,0 +1,65 @@ +name: dbt_run_adhoc +run-name: dbt_run_adhoc + +on: + workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: choice + description: DBT Run Environment + required: true + options: + - dev + - prod + default: dev + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + - DBT_EMERGENCY + dbt_command: + type: string + description: 'DBT Run Command' + required: true + +env: + DBT_PROFILES_DIR: .dbt/ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ inputs.warehouse }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_${{ inputs.environment }} + strategy: + matrix: + command: ${{fromJson(inputs.dbt_command)}} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + ${{ matrix.command }} diff --git a/.github/workflows/dbt_run_daily_dev_refresh.yml b/.github/workflows/dbt_run_daily_dev_refresh.yml new file mode 100644 index 0000000..bed0ec5 --- /dev/null +++ b/.github/workflows/dbt_run_daily_dev_refresh.yml @@ -0,0 +1,42 @@ +name: dbt_run_scheduled_daily +run-name: dbt_run_scheduled_daily + +on: + workflow_dispatch: + schedule: + # Runs 08:00 daily (see https://crontab.guru) + - cron: '0 8 * * *' + +env: + DBT_PROFILES_DIR: .dbt/ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_create_prod_clone diff --git a/.github/workflows/dbt_run_hourly.yml b/.github/workflows/dbt_run_hourly.yml new file mode 100644 index 0000000..1be983c --- /dev/null +++ b/.github/workflows/dbt_run_hourly.yml @@ -0,0 +1,44 @@ +name: dbt_run_scheduled +run-name: dbt_run_scheduled + +on: + workflow_dispatch: + schedule: + # Runs "every hour" (see https://crontab.guru) + - cron: '0 */1 * * *' + +env: + DBT_PROFILES_DIR: .dbt/ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation stage_external_sources --vars "ext_full_refresh: true" + dbt seed + dbt run --exclude streamline__all_topshot_moments_minted_metadata_needed streamline__allday_moments_metadata_needed diff --git a/.github/workflows/dbt_run_moments_metadata.yml b/.github/workflows/dbt_run_moments_metadata.yml new file mode 100644 index 0000000..f1b1f5b --- /dev/null +++ b/.github/workflows/dbt_run_moments_metadata.yml @@ -0,0 +1,43 @@ +name: dbt_run_moments_metadata +run-name: dbt_run_moments_metadata + +on: + workflow_dispatch: + schedule: + # Runs 0000 UTC daily (see https://crontab.guru) + - cron: '0 0 * * *' + +env: + DBT_PROFILES_DIR: .dbt/ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation run_bulk_get_topshot_moments_metadata + dbt run-operation run_bulk_get_nfl_allday_moments_metadata diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml new file mode 100644 index 0000000..013714e --- /dev/null +++ b/.github/workflows/dbt_test.yml @@ -0,0 +1,41 @@ +name: dbt_test_scheduled +run-name: dbt_test_scheduled + +on: + workflow_dispatch: + schedule: + # Run at 1600 UTC daily (see https://crontab.guru) + - cron: '0 16 * * *' +env: + DBT_PROFILES_DIR: .dbt/ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt test diff --git a/.gitignore b/.gitignore index c1efebc..939d28d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,4 @@ logs/ .vscode/ .dbt/.user.yml -.env \ No newline at end of file +.env