Dispatch gha via snowflake (#388)

* update package

* setup GHA dispatches via Snowflake tasks

* remove unused workflows

* add model qualifier to only run models in solana

* include task status monitoring workflow

.github/workflows/dbt_alter_gha_tasks.yml

@@ -0,0 +1,46 @@
+name: dbt_run_alter_gha_task
+run-name: dbt_run_alter_gha_task
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    inputs:
+      workflow_name:
+        type: string
+        description: Name of the workflow to perform the action on, no .yml extension
+        required: true
+      task_action:
+        type: choice 
+        description: Action to perform
+        required: true
+        options:
+          - SUSPEND
+          - RESUME
+        default: SUSPEND
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main
+    with:
+      workflow_name: |
+         ${{ inputs.workflow_name }}
+      task_action: |
+         ${{ inputs.task_action }}
+      environment: workflow_prod
+    secrets: inherit

.github/workflows/dbt_backfill_instruction_decode_hourly.yml

@@ -1,38 +0,0 @@
-name: dbt_backfill_instruction_decode_hourly
-run-name: dbt_backfill_instruction_decode_hourly
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Run at xx:8 UTC hourly (see https://crontab.guru)
-    - cron: '8 * * * *'
-    
-env:
-  USE_VARS: "${{ vars.USE_VARS }}"
-  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
-  DBT_VERSION: "${{ vars.DBT_VERSION }}"
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
-concurrency:
-  group: ${{ github.workflow }}
-
-jobs:
-  called_workflow_template:
-    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
-    with:
-      dbt_command: |
-         dbt run-operation run_sp_bulk_decode_instructions; 
-         dbt run-operation run_sp_bulk_decode_instructions;  
-         dbt run-operation run_sp_bulk_decode_instructions; 
-         dbt run-operation run_sp_bulk_decode_instructions;  
-         dbt run-operation run_sp_bulk_decode_instructions; 
-      environment: workflow_prod
-      warehouse: ${{ vars.WAREHOUSE }}
-    secrets: inherit 

.github/workflows/dbt_run_batch_backfill.yml

@@ -1,44 +0,0 @@
-name: dbt_run_scheduled_batch_backfill
-run-name: dbt_run_scheduled_batch_backfill
-
-on:
-  workflow_dispatch:
-  # schedule:
-  #   # Runs every 5 mins, adjust to appropriate schedule as needed
-  #   - cron: '*/5 * * * *'
-    
-env:
-  DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
-
-  ACCOUNT: "${{ secrets.ACCOUNT }}"
-  ROLE: "${{ secrets.ROLE }}"
-  USER: "${{ secrets.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ secrets.REGION }}"
-  DATABASE: "${{ secrets.DATABASE }}"
-  WAREHOUSE: DBT_EMERGENCY
-  SCHEMA: "${{ secrets.SCHEMA }}"
-
-concurrency:
-  group: ${{ github.workflow }}
-
-jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment: 
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v1
-        with:
-          python-version: "3.7.x"
-
-      - name: install dependencies
-        run: |
-          pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click
-          dbt deps
-      - name: Run DBT Jobs
-        run: |
-          dbt run -s models/silver/accounts

.github/workflows/dbt_run_incremental.yml

@@ -3,9 +3,8 @@ run-name: dbt_run_scheduled
 
 on:
   workflow_dispatch:
-  schedule:
-    # Runs "every hour" (see https://crontab.guru)
-    - cron: '1,16,31,46 * * * *'
+    branches:
+      - "main"
     
 env:
   DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
@@ -42,5 +41,5 @@ jobs:
       - name: Run DBT Jobs
         run: |
           dbt run-operation run_sp_refresh_external_tables_full
-          dbt run -s ./models --exclude tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql models/silver/validator models/silver/_observability models/silver/nfts/silver__nft_compressed_mints_onchain.sql models/silver/nfts/silver__nft_compressed_mints.sql models/gold/core/core__ez_signers.sql tag:helius tag:bronze_api
+          dbt run -s "solana_models,./models" --exclude tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql models/silver/validator models/silver/_observability models/silver/nfts/silver__nft_compressed_mints_onchain.sql models/silver/nfts/silver__nft_compressed_mints.sql models/gold/core/core__ez_signers.sql tag:helius tag:bronze_api models/github_actions
           

.github/workflows/dbt_test.yml

@@ -41,4 +41,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt test -s ./models
+          dbt test -s "solana_models,./models"

.github/workflows/dbt_test_tasks.yml

@@ -0,0 +1,27 @@
+name: dbt_test_tasks
+run-name: dbt_test_tasks
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main
+    secrets: inherit

data/github_actions__workflows.csv

@@ -0,0 +1,3 @@
+workflow_name,workflow_schedule
+dbt_run_scheduled,"1,16,31,46 * * * *"
+dbt_test_tasks,"0,30 * * * *"

dbt_project.yml

@@ -69,9 +69,10 @@ models:
 
 vars:
   "dbt_date:time_zone": GMT
-  UPDATE_SNOWFLAKE_TAGS: True
-  UPDATE_UDFS_AND_SPS: False 
+  OBSERV_FULL_TEST: False
+  START_GHA_TASKS: False
   STREAMLINE_INVOKE_STREAMS: False
   STREAMLINE_RUN_HISTORY: False
   STREAMLINE_RETRY_UNKNOWN: False
-  OBSERV_FULL_TEST: False
+  UPDATE_SNOWFLAKE_TAGS: True
+  UPDATE_UDFS_AND_SPS: False 

models/github_actions/github_actions__current_task_status.sql

@@ -0,0 +1,6 @@
+{{ config(
+    materialized = 'view',
+    tags = ['gha_tasks']
+) }}
+
+{{ fsc_utils.gha_task_current_status_view() }}

models/github_actions/github_actions__current_task_status.yml

@@ -0,0 +1,17 @@
+version: 2
+models:
+  - name: github_actions__current_task_status
+    columns:
+      - name: PIPELINE_ACTIVE
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_set:
+              value_set:
+                - TRUE
+      - name: SUCCESSES
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_set:
+              value_set:
+                - 204
+              config: 
+                  severity: warn
+                  warn_if: ">0"

models/github_actions/github_actions__task_history.sql

@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_task_history_view() }}

models/github_actions/github_actions__task_performance.sql

@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_task_performance_view() }}

models/github_actions/github_actions__task_schedule.sql

@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_task_schedule_view() }}

models/github_actions/github_actions__tasks.sql

@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_tasks_view() }}

models/sources.yml

@@ -92,4 +92,9 @@ sources:
     database: solana
     schema: bronze_api
     tables:
-      - name: parse_compressed_nft_mints_requests
+      - name: parse_compressed_nft_mints_requests
+  - name: github_actions
+    database: solana
+    schema: github_actions
+    tables:
+      - name: workflows

packages.yml

@@ -6,6 +6,6 @@ packages:
   - package: dbt-labs/dbt_utils
     version: 1.0.0
   - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: "v1.3"
+    revision: "v1.11.0"
   - package: get-select/dbt_snowflake_query_tags
     version: [">=2.0.0", "<3.0.0"]