An 2510/solana incrementals (#183)

* standardize env var names, allow env choice, multi cmd workaround

* rename to match jobs from other chains

* update command

* query tags defaulted by project config

* migrate more jobs from dbt cloud

* add doc update workflow

.github/workflows/dbt_docs_update.yml

@@ -0,0 +1,68 @@
+name: docs_update
+
+on:
+  push:
+    branches:
+      - "main"
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ secrets.ACCOUNT }}"
+  ROLE: "${{ secrets.ROLE }}"
+  USER: "${{ secrets.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ secrets.REGION }}"
+  DATABASE: "${{ secrets.DATABASE }}"
+  WAREHOUSE: "${{ secrets.WAREHOUSE }}"
+  SCHEMA: "${{ secrets.SCHEMA }}"
+
+jobs:
+  scheduled_run:
+    name: docs_update
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip install dbt-snowflake==1.2.0
+          dbt deps
+      - name: delete existing docs branch
+        run: git push origin --delete docs || true
+
+      - name: checkout docs branch
+        run: |
+          git checkout -b docs
+          git push --set-upstream origin docs
+      - name: generate dbt docs
+        run: dbt docs generate -t prod
+
+      - name: move files to docs directory
+        run: |
+          mkdir -p ./docs
+          cp target/{catalog.json,manifest.json,index.html} docs/
+      - name: clean up target directory
+        run: dbt clean
+
+      - name: check for changes
+        run: git status
+
+      - name: stage changed files
+        run: git add .
+
+      - name: commit changed files
+        run: |
+          git config user.email "abc@xyz"
+          git config user.name "github-actions"
+          git commit -am "Auto-update docs"
+      - name: push changes to docs
+        run: |
+          git push

.github/workflows/dbt_run_adhoc.yml

@@ -58,7 +58,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake cli_passthrough requests click
+          pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_daily.yml

@@ -0,0 +1,42 @@
+name: dbt_run_scheduled_daily
+run-name: dbt_run_scheduled_daily
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs 00:35 daily (see https://crontab.guru)
+    - cron: '35 0 * * *'
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ secrets.ACCOUNT }}"
+  ROLE: "${{ secrets.ROLE }}"
+  USER: "${{ secrets.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ secrets.REGION }}"
+  DATABASE: "${{ secrets.DATABASE }}"
+  WAREHOUSE: "${{ secrets.WAREHOUSE }}"
+  SCHEMA: "${{ secrets.SCHEMA }}"
+
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -s models/silver/silver__daily_signers.sql+

.github/workflows/dbt_run_incremental.yml

@@ -3,9 +3,9 @@ run-name: dbt_run_scheduled
 
 on:
   workflow_dispatch:
-  # schedule:
-  #   # Runs "every hour" (see https://crontab.guru)
-  #   - cron: '0 */1 * * *'
+  schedule:
+    # Runs "every hour" (see https://crontab.guru)
+    - cron: '0 */1 * * *'
     
 env:
   DBT_PROFILES_DIR: ./

.github/workflows/dbt_test.yml

@@ -0,0 +1,42 @@
+name: dbt_test_scheduled
+run-name: dbt_test_scheduled
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run at 9am UTC daily (see https://crontab.guru)
+    - cron: '0 9 * * *'
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ secrets.ACCOUNT }}"
+  ROLE: "${{ secrets.ROLE }}"
+  USER: "${{ secrets.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ secrets.REGION }}"
+  DATABASE: "${{ secrets.DATABASE }}"
+  WAREHOUSE: "${{ secrets.WAREHOUSE }}"
+  SCHEMA: "${{ secrets.SCHEMA }}"
+
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt test -s ./models

dbt_project.yml

@@ -49,7 +49,7 @@ models:
     relation: true
     columns: true
   +on_schema_change: "append_new_columns"
-  +query_tag: "{{ 'solana_dbt_prod' if target.database == 'SOLANA' else 'solana_dbt_dev' }}"
+  +query_tag: "{{ 'dbt_solana_prod' if target.database == 'SOLANA' else 'dbt_solana_dev' }}"
 
 vars:
   "dbt_date:time_zone": GMT

profiles.yml

@@ -13,7 +13,6 @@ solana:
       warehouse: "{{ env_var('WAREHOUSE') }}"
       threads: 8
       client_session_keep_alive: False
-      query_tag: solana_dbt_dev
     prod:
       type: snowflake
       account: "{{ env_var('ACCOUNT') }}"
@@ -26,6 +25,5 @@ solana:
       warehouse: "{{ env_var('WAREHOUSE') }}"
       threads: 8
       client_session_keep_alive: False
-      query_tag: solana_dbt_prod
   config:
     send_anonymous_usage_stats: False