diff --git a/.github/workflows/dbt_run_deployment_template.yml b/.github/workflows/dbt_run_deployment_template.yml index ee5ebbb..a1f00c2 100644 --- a/.github/workflows/dbt_run_deployment_template.yml +++ b/.github/workflows/dbt_run_deployment_template.yml @@ -11,18 +11,29 @@ on: type: string description: 'github environment to get configuration values' required: true - default: prod warehouse: type: string description: 'dbt warehouse' required: true - default: DBT_CLOUD +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + SCHEMA: "${{ vars.SCHEMA }}" + + WAREHOUSE: "${{ inputs.warehouse }}" jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: ${{ inputs.environment }} steps: @@ -43,7 +54,7 @@ jobs: echo "REGION=${{ vars.REGION }}" >> $GITHUB_ENV echo "DATABASE=${{ vars.DATABASE }}" >> $GITHUB_ENV echo "SCHEMA=${{ vars.SCHEMA }}" >> $GITHUB_ENV - + - name: install dependencies run: | pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click @@ -51,30 +62,8 @@ jobs: - name: Run DBT Jobs run: | ${{ inputs.dbt_command }} - + - name: Run datashare model run: | - cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; + cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; dbt run-operation run_query --args "{sql: call admin.datashare.sp_grant_share_permissions('${{ env.DATABASE }}')}" - - dbt: - uses: ./.github/workflows/dbt.yml - secrets: inherit - with: - warehouse: ${{ inputs.warehouse }} - environment: workflow_${{ inputs.environment }} - command: ${{ inputs.dbt_command }} - - update_datashare: - uses: ./.github/workflows/dbt.yml - needs: dbt - secrets: inherit - with: - warehouse: ${{ inputs.warehouse }} - environment: workflow_${{ inputs.environment }} - command: | - dbt ls -m fsc_utils.datashare._datashare___create_gold - cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ) - if [ $cnt -eq 1 ]; then - dbt run -m fsc_utils.datashare._datashare___create_gold - fi \ No newline at end of file diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml index fa5e7ce..3f86856 100644 --- a/.github/workflows/dbt_run_template.yml +++ b/.github/workflows/dbt_run_template.yml @@ -11,18 +11,55 @@ on: type: string description: 'github environment to get configuration values' required: true - default: prod warehouse: type: string description: 'dbt warehouse' required: true - default: DBT_CLOUD + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_VERSION: "${{ secrets.DBT_VERSION }}" + DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + WAREHOUSE: "${{ inputs.warehouse }}" jobs: run_dbt_jobs: - uses: ./.github/workflows/dbt.yml - secrets: inherit - with: - warehouse: ${{ inputs.warehouse }} - environment: workflow_${{ inputs.environment }} - command: ${{ inputs.dbt_command }} \ No newline at end of file + runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: overwrite env vars + if: ${{ env.USE_VARS == 'TRUE' }} + run: | + echo "DBT_VERSION=${{ vars.DBT_VERSION }}" >> $GITHUB_ENV + echo "DBT_PROFILES_DIR=${{ vars.DBT_PROFILES_DIR }}" >> $GITHUB_ENV + echo "ACCOUNT=${{ vars.ACCOUNT }}" >> $GITHUB_ENV + echo "ROLE=${{ vars.ROLE }}" >> $GITHUB_ENV + echo "USER=${{ vars.USER }}" >> $GITHUB_ENV + echo "REGION=${{ vars.REGION }}" >> $GITHUB_ENV + echo "DATABASE=${{ vars.DATABASE }}" >> $GITHUB_ENV + echo "SCHEMA=${{ vars.SCHEMA }}" >> $GITHUB_ENV + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click + dbt deps + + - name: Run DBT Jobs + run: | + ${{ inputs.dbt_command }}