From 62aa4a61e3c43609c99ed4f5ccb35511b22a0e31 Mon Sep 17 00:00:00 2001 From: desmond-hui <97470747+desmond-hui@users.noreply.github.com> Date: Thu, 8 Dec 2022 15:25:36 -0800 Subject: [PATCH] An 2510/solana incrementals (#183) * standardize env var names, allow env choice, multi cmd workaround * rename to match jobs from other chains * update command * query tags defaulted by project config * migrate more jobs from dbt cloud * add doc update workflow --- .github/workflows/dbt_docs_update.yml | 68 +++++++++++++++++++ .github/workflows/dbt_run_adhoc.yml | 2 +- .github/workflows/dbt_run_daily.yml | 42 ++++++++++++ ...ncremental.yml => dbt_run_incremental.yml} | 6 +- .github/workflows/dbt_test.yml | 42 ++++++++++++ dbt_project.yml | 2 +- profiles.yml | 2 - 7 files changed, 157 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/dbt_docs_update.yml create mode 100644 .github/workflows/dbt_run_daily.yml rename .github/workflows/{dbt_scheduled_incremental.yml => dbt_run_incremental.yml} (94%) create mode 100644 .github/workflows/dbt_test.yml diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml new file mode 100644 index 00000000..f9ab694a --- /dev/null +++ b/.github/workflows/dbt_docs_update.yml @@ -0,0 +1,68 @@ +name: docs_update + +on: + push: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + +jobs: + scheduled_run: + name: docs_update + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip install dbt-snowflake==1.2.0 + dbt deps + - name: delete existing docs branch + run: git push origin --delete docs || true + + - name: checkout docs branch + run: | + git checkout -b docs + git push --set-upstream origin docs + - name: generate dbt docs + run: dbt docs generate -t prod + + - name: move files to docs directory + run: | + mkdir -p ./docs + cp target/{catalog.json,manifest.json,index.html} docs/ + - name: clean up target directory + run: dbt clean + + - name: check for changes + run: git status + + - name: stage changed files + run: git add . + + - name: commit changed files + run: | + git config user.email "abc@xyz" + git config user.name "github-actions" + git commit -am "Auto-update docs" + - name: push changes to docs + run: | + git push \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 91af3cec..1f7a881a 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -58,7 +58,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake cli_passthrough requests click + pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml new file mode 100644 index 00000000..c2a68061 --- /dev/null +++ b/.github/workflows/dbt_run_daily.yml @@ -0,0 +1,42 @@ +name: dbt_run_scheduled_daily +run-name: dbt_run_scheduled_daily + +on: + workflow_dispatch: + schedule: + # Runs 00:35 daily (see https://crontab.guru) + - cron: '35 0 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run -s models/silver/silver__daily_signers.sql+ diff --git a/.github/workflows/dbt_scheduled_incremental.yml b/.github/workflows/dbt_run_incremental.yml similarity index 94% rename from .github/workflows/dbt_scheduled_incremental.yml rename to .github/workflows/dbt_run_incremental.yml index 1586fc89..1ce73a2a 100644 --- a/.github/workflows/dbt_scheduled_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -3,9 +3,9 @@ run-name: dbt_run_scheduled on: workflow_dispatch: - # schedule: - # # Runs "every hour" (see https://crontab.guru) - # - cron: '0 */1 * * *' + schedule: + # Runs "every hour" (see https://crontab.guru) + - cron: '0 */1 * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml new file mode 100644 index 00000000..83b692b7 --- /dev/null +++ b/.github/workflows/dbt_test.yml @@ -0,0 +1,42 @@ +name: dbt_test_scheduled +run-name: dbt_test_scheduled + +on: + workflow_dispatch: + schedule: + # Run at 9am UTC daily (see https://crontab.guru) + - cron: '0 9 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt test -s ./models \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 805da01c..5991a7ce 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -49,7 +49,7 @@ models: relation: true columns: true +on_schema_change: "append_new_columns" - +query_tag: "{{ 'solana_dbt_prod' if target.database == 'SOLANA' else 'solana_dbt_dev' }}" + +query_tag: "{{ 'dbt_solana_prod' if target.database == 'SOLANA' else 'dbt_solana_dev' }}" vars: "dbt_date:time_zone": GMT diff --git a/profiles.yml b/profiles.yml index 83dec2e4..ea3df084 100644 --- a/profiles.yml +++ b/profiles.yml @@ -13,7 +13,6 @@ solana: warehouse: "{{ env_var('WAREHOUSE') }}" threads: 8 client_session_keep_alive: False - query_tag: solana_dbt_dev prod: type: snowflake account: "{{ env_var('ACCOUNT') }}" @@ -26,6 +25,5 @@ solana: warehouse: "{{ env_var('WAREHOUSE') }}" threads: 8 client_session_keep_alive: False - query_tag: solana_dbt_prod config: send_anonymous_usage_stats: False \ No newline at end of file