From 70c25ac007f35399e900630c17a0eca77074cb8c Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 5 Jan 2023 10:50:07 -0800 Subject: [PATCH 1/5] allow each repo to sepcify execution env, dbt ver, dbt profiles --- .github/workflows/dbt_run_template.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml index cd03cf0..609e13b 100644 --- a/.github/workflows/dbt_run_template.yml +++ b/.github/workflows/dbt_run_template.yml @@ -9,24 +9,22 @@ on: required: true env: - DBT_PROFILES_DIR: ./ + DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" ACCOUNT: "${{ secrets.ACCOUNT }}" ROLE: "${{ secrets.ROLE }}" USER: "${{ secrets.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" REGION: "${{ secrets.REGION }}" - DATABASE_DEV: "${{ secrets.DATABASE_DEV }}" - DATABASE_PROD: "${{ secrets.DATABASE_PROD }}" - WAREHOUSE_DEV: "${{ secrets.WAREHOUSE_DEV }}" - WAREHOUSE_PROD: "${{ secrets.WAREHOUSE_PROD }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" SCHEMA: "${{ secrets.SCHEMA }}" jobs: run_dbt_jobs: runs-on: ubuntu-latest environment: - name: workflow + name: "${{ secrets.GITHUB_ENVIRONMENT }}" strategy: matrix: command: ${{fromJson(inputs.dbt_command)}} @@ -40,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake cli_passthrough requests click + pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | From b53ae157036428530185a635d6878edc633c8323 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 5 Jan 2023 13:19:26 -0800 Subject: [PATCH 2/5] env should be input param --- .github/workflows/dbt_run_template.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml index 609e13b..8ab22e0 100644 --- a/.github/workflows/dbt_run_template.yml +++ b/.github/workflows/dbt_run_template.yml @@ -7,6 +7,9 @@ on: type: string description: 'dbt commands to run' required: true + environment: + type: string + description: 'github environment to get configuration values' env: DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" @@ -24,7 +27,7 @@ jobs: run_dbt_jobs: runs-on: ubuntu-latest environment: - name: "${{ secrets.GITHUB_ENVIRONMENT }}" + name: "${{ inputs.environment }}" strategy: matrix: command: ${{fromJson(inputs.dbt_command)}} From 90033bf7eccfd15aec56f6f5ae01900f2ae90475 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 5 Jan 2023 17:52:12 -0800 Subject: [PATCH 3/5] use warehouse env as default if non provided --- .github/workflows/dbt_run_template.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml index 8ab22e0..3b04b8e 100644 --- a/.github/workflows/dbt_run_template.yml +++ b/.github/workflows/dbt_run_template.yml @@ -10,6 +10,10 @@ on: environment: type: string description: 'github environment to get configuration values' + warehouse: + type: string + description: 'dbt warehouse' + default: "${{ secrets.WAREHOUSE }}" env: DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" @@ -20,7 +24,7 @@ env: PASSWORD: "${{ secrets.PASSWORD }}" REGION: "${{ secrets.REGION }}" DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" + WAREHOUSE: "${{ inputs.warehouse }}" SCHEMA: "${{ secrets.SCHEMA }}" jobs: From 9a46f0cfaecde1097de3a321be4e15c420f6f4d9 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 5 Jan 2023 17:56:21 -0800 Subject: [PATCH 4/5] doesnt like secrets for default value --- .github/workflows/dbt_run_template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml index 3b04b8e..a250362 100644 --- a/.github/workflows/dbt_run_template.yml +++ b/.github/workflows/dbt_run_template.yml @@ -13,7 +13,6 @@ on: warehouse: type: string description: 'dbt warehouse' - default: "${{ secrets.WAREHOUSE }}" env: DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" From d90c166d2ea9fde2200db762369f3995fa6ea545 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 12 Jan 2023 15:12:24 -0800 Subject: [PATCH 5/5] make required --- .github/workflows/dbt_run_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml index a250362..faa122d 100644 --- a/.github/workflows/dbt_run_template.yml +++ b/.github/workflows/dbt_run_template.yml @@ -10,9 +10,11 @@ on: environment: type: string description: 'github environment to get configuration values' + required: true warehouse: type: string description: 'dbt warehouse' + required: true env: DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"