From 72d45581b9a12a837659511af8aa3cd374618858 Mon Sep 17 00:00:00 2001
From: Julius Remigio <14811322+juls858@users.noreply.github.com>
Date: Thu, 14 Sep 2023 14:43:34 -0700
Subject: [PATCH] - add dbt.yml - refactor workflows to use dbt.yml

---
 .github/workflows/dbt.yml                     | 66 ++++++++++++++++++
 .../workflows/dbt_run_deployment_template.yml | 69 ++++++-------------
 .github/workflows/dbt_run_template.yml        | 53 +++-----------
 3 files changed, 96 insertions(+), 92 deletions(-)
 create mode 100644 .github/workflows/dbt.yml

diff --git a/.github/workflows/dbt.yml b/.github/workflows/dbt.yml
new file mode 100644
index 0000000..a0998d2
--- /dev/null
+++ b/.github/workflows/dbt.yml
@@ -0,0 +1,66 @@
+name: dbt
+on:
+  workflow_call:
+    inputs:
+      warehouse:
+        required: false
+        type: string
+        default: DBT_CLOUD
+      environment:
+        required: true
+        type: string
+      command:
+          required: true
+          type: string
+      python_version:
+          required: false
+          type: string
+          default: "3.10"
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+jobs:
+  dbt:
+    runs-on: ubuntu-latest
+    env:
+      DBT_PROFILES_DIR: ./
+      ACCOUNT: "${{ vars.ACCOUNT }}"
+      ROLE: "${{ vars.ROLE }}"
+      USER: "${{ vars.USER }}"
+      PASSWORD: "${{ secrets.PASSWORD }}"
+      REGION: "${{ vars.REGION }}"
+      DATABASE: "${{ vars.DATABASE }}"
+      WAREHOUSE: ${{ inputs.warehouse }}
+      SCHEMA: "${{ vars.SCHEMA }}"
+    environment:
+      name: ${{ inputs.environment }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          ${{ inputs.command }}
+      - name: Store logs
+        uses: actions/upload-artifact@v3
+        with:
+          name: dbt-logs
+          path: |
+            logs
+            target
\ No newline at end of file
diff --git a/.github/workflows/dbt_run_deployment_template.yml b/.github/workflows/dbt_run_deployment_template.yml
index 26e0be8..6e0bc79 100644
--- a/.github/workflows/dbt_run_deployment_template.yml
+++ b/.github/workflows/dbt_run_deployment_template.yml
@@ -11,58 +11,33 @@ on:
         type: string
         description: 'github environment to get configuration values'
         required: true
+        default: prod
       warehouse:
         type: string
         description: 'dbt warehouse'
         required: true
+        default: DBT_CLOUD
 
-env:
-  USE_VARS: "${{ vars.USE_VARS }}"
-  DBT_VERSION: "${{ vars.DBT_VERSION }}"
-  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
-  WAREHOUSE: "${{ inputs.warehouse }}"
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment: 
-      name: ${{ inputs.environment }}
+  dbt:
+    uses: ./.github/workflows/dbt.yml
+    secrets: inherit
+    with:
+      warehouse: ${{ inputs.warehouse }}
+      environment: workflow_${{ inputs.environment }}
+      command: ${{ inputs.dbt_command }}
 
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v1
-        with:
-          python-version: "3.7.x"
-
-      - name: overwrite env vars
-        if: ${{ env.USE_VARS == 'TRUE' }}
-        run: |
-          echo "DBT_VERSION=${{ vars.DBT_VERSION }}" >> $GITHUB_ENV
-          echo "DBT_PROFILES_DIR=${{ vars.DBT_PROFILES_DIR }}" >> $GITHUB_ENV
-          echo "ACCOUNT=${{ vars.ACCOUNT }}" >> $GITHUB_ENV
-          echo "ROLE=${{ vars.ROLE }}" >> $GITHUB_ENV
-          echo "USER=${{ vars.USER }}" >> $GITHUB_ENV
-          echo "REGION=${{ vars.REGION }}" >> $GITHUB_ENV
-          echo "DATABASE=${{ vars.DATABASE }}" >> $GITHUB_ENV
-          echo "SCHEMA=${{ vars.SCHEMA }}" >> $GITHUB_ENV
-        
-      - name: install dependencies
-        run: |
-          pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click
-          dbt deps
-      - name: Run DBT Jobs
-        run: |
-          ${{ inputs.dbt_command }}
-      
-      - name: Run datashare model
-        run: |
-          cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; 
+  update_datashare:
+    uses: ./.github/workflows/dbt.yml
+    needs: dbt
+    secrets: inherit
+    with:
+      warehouse: ${{ inputs.warehouse }}
+      environment: workflow_${{ inputs.environment }}
+      command: |
+        dbt ls -m fsc_utils.datashare._datashare___create_gold
+        cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l )
+        if [ $cnt -eq 1 ]; then
+          dbt run -m fsc_utils.datashare._datashare___create_gold
+        fi
\ No newline at end of file
diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml
index 3f86856..fa5e7ce 100644
--- a/.github/workflows/dbt_run_template.yml
+++ b/.github/workflows/dbt_run_template.yml
@@ -11,55 +11,18 @@ on:
         type: string
         description: 'github environment to get configuration values'
         required: true
+        default: prod
       warehouse:
         type: string
         description: 'dbt warehouse'
         required: true
-
-env:
-  USE_VARS: "${{ vars.USE_VARS }}"
-  DBT_VERSION: "${{ secrets.DBT_VERSION }}"
-  DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
-  ACCOUNT: "${{ secrets.ACCOUNT }}"
-  ROLE: "${{ secrets.ROLE }}"
-  USER: "${{ secrets.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ secrets.REGION }}"
-  DATABASE: "${{ secrets.DATABASE }}"
-  SCHEMA: "${{ secrets.SCHEMA }}"
-
-  WAREHOUSE: "${{ inputs.warehouse }}"
+        default: DBT_CLOUD
 
 jobs:
   run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment: 
-      name: ${{ inputs.environment }}
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v1
-        with:
-          python-version: "3.7.x"
-
-      - name: overwrite env vars
-        if: ${{ env.USE_VARS == 'TRUE' }}
-        run: |
-          echo "DBT_VERSION=${{ vars.DBT_VERSION }}" >> $GITHUB_ENV
-          echo "DBT_PROFILES_DIR=${{ vars.DBT_PROFILES_DIR }}" >> $GITHUB_ENV
-          echo "ACCOUNT=${{ vars.ACCOUNT }}" >> $GITHUB_ENV
-          echo "ROLE=${{ vars.ROLE }}" >> $GITHUB_ENV
-          echo "USER=${{ vars.USER }}" >> $GITHUB_ENV
-          echo "REGION=${{ vars.REGION }}" >> $GITHUB_ENV
-          echo "DATABASE=${{ vars.DATABASE }}" >> $GITHUB_ENV
-          echo "SCHEMA=${{ vars.SCHEMA }}" >> $GITHUB_ENV
-        
-      - name: install dependencies
-        run: |
-          pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click
-          dbt deps
-
-      - name: Run DBT Jobs
-        run: |
-          ${{ inputs.dbt_command }}
+    uses: ./.github/workflows/dbt.yml
+    secrets: inherit
+    with:
+      warehouse: ${{ inputs.warehouse }}
+      environment: workflow_${{ inputs.environment }}
+      command: ${{ inputs.dbt_command }}
\ No newline at end of file