workflow config (#100)

* workflow config

* github actions workflow

* small tweak

* env update for ad hoc

* daily job

.github/workflows/.dbt/profiles.yml

@@ -8,12 +8,11 @@ flow:
       user: "{{ env_var('USER') }}"
       password: "{{ env_var('PASSWORD') }}"
       region: "{{ env_var('REGION') }}"
-      database: "{{ env_var('DATABASE_DEV') }}"
-      warehouse: "{{ env_var('WAREHOUSE_DEV') }}"
+      database: "{{ env_var('DATABASE') }}"
+      warehouse: "{{ env_var('WAREHOUSE') }}"
       schema: SILVER
       threads: 4
       client_session_keep_alive: False
-      query_tag: gh_actions
     prod:
       type: snowflake
       account: "{{ env_var('ACCOUNT') }}"
@@ -21,9 +20,8 @@ flow:
       user: "{{ env_var('USER') }}"
       password: "{{ env_var('PASSWORD') }}"
       region: "{{ env_var('REGION') }}"
-      database: "{{ env_var('DATABASE_PROD') }}"
-      warehouse: "{{ env_var('WAREHOUSE_PROD') }}"
+      database: "{{ env_var('DATABASE') }}"
+      warehouse: "{{ env_var('WAREHOUSE') }}"
       schema: silver
       threads: 4
       client_session_keep_alive: False
-      query_tag: gh_actions

.github/workflows/dbt_docs_update.yml

@@ -1,4 +1,4 @@
-name: docs_update
+name: dbt_docs_update
 
 on:
   push:
@@ -6,20 +6,23 @@ on:
       - "main"
 
 env:
+  DBT_PROFILES_DIR: .dbt/
+
   ACCOUNT: "${{ secrets.ACCOUNT }}"
   ROLE: "${{ secrets.ROLE }}"
   USER: "${{ secrets.USER }}"
   PASSWORD: "${{ secrets.PASSWORD }}"
   REGION: "${{ secrets.REGION }}"
-  DATABASE_DEV: "${{ secrets.DATABASE_DEV }}"
-  DATABASE_PROD: "${{ secrets.DATABASE_PROD }}"
-  WAREHOUSE_DEV: "${{ secrets.WAREHOUSE_DEV }}"
-  WAREHOUSE_PROD: "${{ secrets.WAREHOUSE_PROD }}"
+  DATABASE: "${{ secrets.DATABASE }}"
+  WAREHOUSE: "${{ secrets.WAREHOUSE }}"
+  SCHEMA: "${{ secrets.SCHEMA }}"
 
 jobs:
   scheduled_run:
     name: docs_update
     runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
 
     steps:
       - uses: actions/checkout@v3
@@ -30,25 +33,19 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip install dbt-snowflake
+          pip install dbt-snowflake==1.3.0
           dbt deps
-
-      - name: delete existing docs branch
-        run: git push origin --delete docs || true
-
       - name: checkout docs branch
         run: |
-          git checkout -b docs
-          git push --set-upstream origin docs
-
+          git checkout -B docs origin/main
+          
       - name: generate dbt docs
-        run: dbt docs generate -t prod --profiles-dir ./.dbt
+        run: dbt docs generate -t prod
 
       - name: move files to docs directory
         run: |
           mkdir -p ./docs
           cp target/{catalog.json,manifest.json,index.html} docs/
-
       - name: clean up target directory
         run: dbt clean
 
@@ -63,7 +60,6 @@ jobs:
           git config user.email "abc@xyz"
           git config user.name "github-actions"
           git commit -am "Auto-update docs"
-
       - name: push changes to docs
         run: |
-          git push
+          git push -f --set-upstream origin docs

.github/workflows/dbt_run_adhoc.yml

@@ -0,0 +1,65 @@
+name: dbt_run_adhoc
+run-name: dbt_run_adhoc
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    inputs:
+      environment:
+        type: choice 
+        description: DBT Run Environment
+        required: true
+        options:
+          - dev
+          - prod
+        default: dev
+      warehouse:
+        type: choice 
+        description: Snowflake warehouse
+        required: true 
+        options:
+          - DBT
+          - DBT_CLOUD
+          - DBT_EMERGENCY
+      dbt_command:
+        type: string
+        description: 'DBT Run Command'
+        required: true
+
+env:
+  DBT_PROFILES_DIR: .dbt/
+
+  ACCOUNT: "${{ secrets.ACCOUNT }}"
+  ROLE: "${{ secrets.ROLE }}"
+  USER: "${{ secrets.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ secrets.REGION }}"
+  DATABASE: "${{ secrets.DATABASE }}"
+  WAREHOUSE: "${{ inputs.warehouse }}"
+  SCHEMA: "${{ secrets.SCHEMA }}"
+
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_${{ inputs.environment }}
+    strategy:
+      matrix:
+        command: ${{fromJson(inputs.dbt_command)}}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          ${{ matrix.command }}

.github/workflows/dbt_run_daily_dev_refresh.yml

@@ -0,0 +1,42 @@
+name: dbt_run_scheduled_daily
+run-name: dbt_run_scheduled_daily
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs 08:00 daily (see https://crontab.guru)
+    - cron: '0 8 * * *'
+
+env:
+  DBT_PROFILES_DIR: .dbt/
+
+  ACCOUNT: "${{ secrets.ACCOUNT }}"
+  ROLE: "${{ secrets.ROLE }}"
+  USER: "${{ secrets.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ secrets.REGION }}"
+  DATABASE: "${{ secrets.DATABASE }}"
+  WAREHOUSE: "${{ secrets.WAREHOUSE }}"
+  SCHEMA: "${{ secrets.SCHEMA }}"
+
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run-operation run_sp_create_prod_clone

.github/workflows/dbt_run_hourly.yml

@@ -0,0 +1,44 @@
+name: dbt_run_scheduled
+run-name: dbt_run_scheduled
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs "every hour" (see https://crontab.guru)
+    - cron: '0 */1 * * *'
+
+env:
+  DBT_PROFILES_DIR: .dbt/
+
+  ACCOUNT: "${{ secrets.ACCOUNT }}"
+  ROLE: "${{ secrets.ROLE }}"
+  USER: "${{ secrets.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ secrets.REGION }}"
+  DATABASE: "${{ secrets.DATABASE }}"
+  WAREHOUSE: "${{ secrets.WAREHOUSE }}"
+  SCHEMA: "${{ secrets.SCHEMA }}"
+
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
+          dbt seed
+          dbt run --exclude streamline__all_topshot_moments_minted_metadata_needed streamline__allday_moments_metadata_needed

.github/workflows/dbt_run_moments_metadata.yml

@@ -0,0 +1,43 @@
+name: dbt_run_moments_metadata
+run-name: dbt_run_moments_metadata
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs 0000 UTC daily (see https://crontab.guru)
+    - cron: '0 0 * * *'
+
+env:
+  DBT_PROFILES_DIR: .dbt/
+
+  ACCOUNT: "${{ secrets.ACCOUNT }}"
+  ROLE: "${{ secrets.ROLE }}"
+  USER: "${{ secrets.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ secrets.REGION }}"
+  DATABASE: "${{ secrets.DATABASE }}"
+  WAREHOUSE: "${{ secrets.WAREHOUSE }}"
+  SCHEMA: "${{ secrets.SCHEMA }}"
+
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run-operation run_bulk_get_topshot_moments_metadata
+          dbt run-operation run_bulk_get_nfl_allday_moments_metadata

.github/workflows/dbt_test.yml

@@ -0,0 +1,41 @@
+name: dbt_test_scheduled
+run-name: dbt_test_scheduled
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run at 1600 UTC daily (see https://crontab.guru)
+    - cron: '0 16 * * *'
+env:
+  DBT_PROFILES_DIR: .dbt/
+
+  ACCOUNT: "${{ secrets.ACCOUNT }}"
+  ROLE: "${{ secrets.ROLE }}"
+  USER: "${{ secrets.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ secrets.REGION }}"
+  DATABASE: "${{ secrets.DATABASE }}"
+  WAREHOUSE: "${{ secrets.WAREHOUSE }}"
+  SCHEMA: "${{ secrets.SCHEMA }}"
+
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt test

.gitignore

@@ -16,4 +16,4 @@ logs/
 .vscode/
 
 .dbt/.user.yml
-.env
+.env