Merge pull request #71 from FlipsideCrypto/update_docs

Update docs

.github/workflows/dbt_alter_gha_task.yml

@@ -1,46 +1,56 @@
-name: dbt_alter_gha_task
-run-name: dbt_alter_gha_task
+name: dbt_alter_gha_tasks
 
 on:
-  workflow_dispatch:
-    branches:
-      - "main"
+  workflow_call:
     inputs:
       workflow_name:
         type: string
-        description: Name of the workflow to perform the action on, no .yml extension
+        description: "Name of the workflow to perform the action on, no .yml extension"
         required: true
       task_action:
-        type: choice
-        description: Action to perform
+        type: string
+        description: "Action to perform"
+        required: true
+      environment:
+        type: string
+        description: "Environment to run the workflow in"
         required: true
-        options:
-          - SUSPEND
-          - RESUME
-        default: SUSPEND
 
 env:
-  DBT_PROFILES_DIR: ./
-
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
   ACCOUNT: "${{ vars.ACCOUNT }}"
   ROLE: "${{ vars.ROLE }}"
   USER: "${{ vars.USER }}"
   PASSWORD: "${{ secrets.PASSWORD }}"
   REGION: "${{ vars.REGION }}"
   DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
   SCHEMA: "${{ vars.SCHEMA }}"
-
-concurrency:
-  group: ${{ github.workflow }}
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
 
 jobs:
-  called_workflow_template:
-    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7
-    with:
-      workflow_name: |
-        ${{ inputs.workflow_name }}
-      task_action: |
-        ${{ inputs.task_action }}
-      environment: workflow_prod
-    secrets: inherit
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.environment }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip3 install -r requirements.txt
+          dbt deps
+
+      - name: Set up workflow name
+        run: echo "WORKFLOW_NAME_UPPER=$(echo '${{ inputs.workflow_name }}' | tr '[:lower:]' '[:upper:]')" >> $GITHUB_ENV
+
+      - name: Run DBT Jobs
+        run: |
+          dbt run-operation fsc_utils.alter_gha_task --args '{ "task_name": "TRIGGER_${{ env.WORKFLOW_NAME_UPPER }}", "task_action": "${{ inputs.task_action }}" }'

.github/workflows/dbt_docs_update.yml

@@ -22,6 +22,51 @@ concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  called_workflow_template:
-    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@AN-4374/upgrade-dbt-1.7
-    secrets: inherit
+  dbt:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_prod
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+
+      - name: refresh ddl for datashare
+        run: |
+          cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi;
+
+      - name: checkout docs branch
+        run: |
+          git checkout -B docs origin/main
+
+      - name: generate dbt docs
+        run: dbt docs generate -t prod
+
+      - name: move files to docs directory
+        run: |
+          mkdir -p ./docs
+          cp target/{catalog.json,manifest.json,index.html} docs/
+      - name: clean up target directory
+        run: dbt clean
+
+      - name: check for changes
+        run: git status
+
+      - name: stage changed files
+        run: git add .
+
+      - name: commit changed files
+        run: |
+          git config user.email "abc@xyz"
+          git config user.name "github-actions"
+          git commit -am "Auto-update docs"
+      - name: push changes to docs
+        run: |
+          git push -f --set-upstream origin docs

.github/workflows/dbt_run_deployment.yml

@@ -37,10 +37,35 @@ concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  called_workflow_template:
-    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@AN-4374/upgrade-dbt-1.7
-    with:
-      dbt_command: |
-        ${{ inputs.dbt_command }}
-      warehouse: ${{ inputs.WAREHOUSE }}
-    secrets: inherit
+  dbt:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_prod
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+
+      - name: Run DBT Jobs
+        run: |
+          ${{ inputs.dbt_command }}
+
+      - name: Run datashare model
+        run: |
+          cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; 
+          dbt run-operation run_query --args "{sql: call admin.datashare.sp_grant_share_permissions('${{ env.DATABASE }}')}"
+
+      - name: Store logs
+        uses: actions/upload-artifact@v3
+        with:
+          name: dbt-logs
+          path: |
+            logs
+            target

.github/workflows/dbt_test_tasks.yml

@@ -1,27 +1,40 @@
 name: dbt_test_tasks
-run-name: dbt_test_tasks
 
 on:
-  workflow_dispatch:
-    branches:
-      - "main"
+  workflow_call:
 
 env:
-  DBT_PROFILES_DIR: ./
-
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
   ACCOUNT: "${{ vars.ACCOUNT }}"
   ROLE: "${{ vars.ROLE }}"
   USER: "${{ vars.USER }}"
   PASSWORD: "${{ secrets.PASSWORD }}"
   REGION: "${{ vars.REGION }}"
   DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
   SCHEMA: "${{ vars.SCHEMA }}"
-
-concurrency:
-  group: ${{ github.workflow }}
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
 
 jobs:
-  called_workflow_template:
-    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7
-    secrets: inherit
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+
+      - name: Run DBT Jobs
+        run: |
+          dbt test -m models/github_actions/github_actions__current_task_status.sql