From 7bf5f5f2bf0962a8c45432873603c7b95fd06336 Mon Sep 17 00:00:00 2001
From: Austin <93135983+austinFlipside@users.noreply.github.com>
Date: Tue, 31 Oct 2023 16:29:42 -0400
Subject: [PATCH] An 4105/lq tasks (#157)

* stash

* LQ

* rename
---
 .github/workflows/dbt_alter_gha_task.yml      | 46 +++++++++++++++++++
 .github/workflows/dbt_run_abi_refresh.yml     |  2 +-
 .../workflows/dbt_run_full_observability.yml  |  2 +-
 .github/workflows/dbt_run_heal_models.yml     |  2 +-
 .github/workflows/dbt_run_operation_reorg.yml |  2 +-
 .../workflows/dbt_run_scheduled_curated.yml   |  5 +-
 .../dbt_run_scheduled_non_realtime.yml        |  7 ++-
 .../dbt_run_streamline_chainhead.yml          |  8 ++--
 .../workflows/dbt_run_streamline_decoder.yml  |  7 ++-
 .../dbt_run_streamline_decoder_history.yml    |  2 +-
 .../dbt_run_streamline_history_adhoc.yml      |  4 +-
 .github/workflows/dbt_test_daily.yml          |  2 +-
 .github/workflows/dbt_test_intraday.yml       |  4 +-
 .github/workflows/dbt_test_monthly.yml        |  2 +-
 .github/workflows/dbt_test_tasks.yml          | 27 +++++++++++
 data/github_actions__workflows.csv            |  6 +++
 dbt_project.yml                               |  3 +-
 .../github_actions__current_task_status.sql   |  6 +++
 .../github_actions__current_task_status.yml   | 17 +++++++
 .../github_actions__task_history.sql          |  5 ++
 .../github_actions__task_performance.sql      |  5 ++
 .../github_actions__task_schedule.sql         |  5 ++
 .../github_actions/github_actions__tasks.sql  |  5 ++
 models/sources.yml                            |  5 ++
 packages.yml                                  |  2 +-
 25 files changed, 152 insertions(+), 29 deletions(-)
 create mode 100644 .github/workflows/dbt_alter_gha_task.yml
 create mode 100644 .github/workflows/dbt_test_tasks.yml
 create mode 100644 data/github_actions__workflows.csv
 create mode 100644 models/github_actions/github_actions__current_task_status.sql
 create mode 100644 models/github_actions/github_actions__current_task_status.yml
 create mode 100644 models/github_actions/github_actions__task_history.sql
 create mode 100644 models/github_actions/github_actions__task_performance.sql
 create mode 100644 models/github_actions/github_actions__task_schedule.sql
 create mode 100644 models/github_actions/github_actions__tasks.sql

diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml
new file mode 100644
index 0000000..da996d8
--- /dev/null
+++ b/.github/workflows/dbt_alter_gha_task.yml
@@ -0,0 +1,46 @@
+name: dbt_alter_gha_task
+run-name: dbt_alter_gha_task
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    inputs:
+      workflow_name:
+        type: string
+        description: Name of the workflow to perform the action on, no .yml extension
+        required: true
+      task_action:
+        type: choice 
+        description: Action to perform
+        required: true
+        options:
+          - SUSPEND
+          - RESUME
+        default: SUSPEND
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main
+    with:
+      workflow_name: |
+         ${{ inputs.workflow_name }}
+      task_action: |
+         ${{ inputs.task_action }}
+      environment: workflow_prod
+    secrets: inherit
\ No newline at end of file
diff --git a/.github/workflows/dbt_run_abi_refresh.yml b/.github/workflows/dbt_run_abi_refresh.yml
index 831d39c..a0d2768 100644
--- a/.github/workflows/dbt_run_abi_refresh.yml
+++ b/.github/workflows/dbt_run_abi_refresh.yml
@@ -42,5 +42,5 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run -m tag:abis
+          dbt run -m "arbitrum_models,tag:abis"
   
\ No newline at end of file
diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml
index 7a17283..d63d443 100644
--- a/.github/workflows/dbt_run_full_observability.yml
+++ b/.github/workflows/dbt_run_full_observability.yml
@@ -42,5 +42,5 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m tag:observability
+          dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m "arbitrum_models,tag:observability"
   
\ No newline at end of file
diff --git a/.github/workflows/dbt_run_heal_models.yml b/.github/workflows/dbt_run_heal_models.yml
index c2617f8..0454f10 100644
--- a/.github/workflows/dbt_run_heal_models.yml
+++ b/.github/workflows/dbt_run_heal_models.yml
@@ -42,4 +42,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run -m tag:heal --var '{"HEAL_MODEL":True}'
\ No newline at end of file
+          dbt run -m "arbitrum_models,tag:heal" --var '{"HEAL_MODEL":True}'
\ No newline at end of file
diff --git a/.github/workflows/dbt_run_operation_reorg.yml b/.github/workflows/dbt_run_operation_reorg.yml
index c535025..e0588f4 100644
--- a/.github/workflows/dbt_run_operation_reorg.yml
+++ b/.github/workflows/dbt_run_operation_reorg.yml
@@ -44,7 +44,7 @@ jobs:
       - name: List reorg models
         id: list_models
         run: |
-          reorg_model_list=$(dbt list --select tag:reorg --resource-type model | awk -F'.' '{print $NF}' | tr '\n' ',' | sed 's/,$//')
+          reorg_model_list=$(dbt list --select "arbitrum_models,tag:reorg" --resource-type model | awk -F'.' '{print $NF}' | tr '\n' ',' | sed 's/,$//')
           echo "::set-output name=model_list::$reorg_model_list"
   
       - name: Execute block_reorg macro
diff --git a/.github/workflows/dbt_run_scheduled_curated.yml b/.github/workflows/dbt_run_scheduled_curated.yml
index e36c041..fc2819a 100644
--- a/.github/workflows/dbt_run_scheduled_curated.yml
+++ b/.github/workflows/dbt_run_scheduled_curated.yml
@@ -3,9 +3,8 @@ run-name: dbt_run_scheduled_curated
 
 on:
   workflow_dispatch:
-  schedule:
-    # Runs at minutes 20 and 50" (see https://crontab.guru)
-    - cron: '20,50 * * * *'
+    branches:
+      - "main"
     
 env:
   DBT_PROFILES_DIR: ./
diff --git a/.github/workflows/dbt_run_scheduled_non_realtime.yml b/.github/workflows/dbt_run_scheduled_non_realtime.yml
index 7d3d960..c8a44cf 100644
--- a/.github/workflows/dbt_run_scheduled_non_realtime.yml
+++ b/.github/workflows/dbt_run_scheduled_non_realtime.yml
@@ -3,9 +3,8 @@ run-name: dbt_run_scheduled_non_realtime
 
 on:
   workflow_dispatch:
-  schedule:
-    # Runs at minutes 3,15,27,39,51" (see https://crontab.guru)
-    - cron: '3,15,27,39,51 * * * *'
+    branches:
+      - "main"
     
 env:
   DBT_PROFILES_DIR: ./
@@ -42,4 +41,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m tag:non_realtime tag:streamline_decoded_logs_complete tag:streamline_decoded_logs_realtime
+          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "arbitrum_models,tag:non_realtime" "arbitrum_models,tag:streamline_decoded_logs_complete" "arbitrum_models,tag:streamline_decoded_logs_realtime"
\ No newline at end of file
diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml
index 64f892a..b0cd549 100644
--- a/.github/workflows/dbt_run_streamline_chainhead.yml
+++ b/.github/workflows/dbt_run_streamline_chainhead.yml
@@ -3,9 +3,8 @@ run-name: dbt_run_streamline_chainhead
 
 on:
   workflow_dispatch:
-  schedule:
-    # Runs "at minutes 9,21,33,45,57" (see https://crontab.guru)
-    - cron: '9,21,33,45,57 * * * *'
+    branches:
+      - "main"
     
 env:
   DBT_PROFILES_DIR: ./
@@ -42,5 +41,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m tag:streamline_core_complete tag:streamline_core_realtime
-          
\ No newline at end of file
+          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "arbitrum_models,tag:streamline_core_complete" "arbitrum_models,tag:streamline_core_realtime"
\ No newline at end of file
diff --git a/.github/workflows/dbt_run_streamline_decoder.yml b/.github/workflows/dbt_run_streamline_decoder.yml
index db5a9ae..3f00164 100644
--- a/.github/workflows/dbt_run_streamline_decoder.yml
+++ b/.github/workflows/dbt_run_streamline_decoder.yml
@@ -3,9 +3,8 @@ run-name: dbt_run_streamline_decoder
 
 on:
   workflow_dispatch:
-  schedule:
-    # Runs at minutes 0,12,24,36,48 (see https://crontab.guru)
-    - cron: '0,12,24,36,48 * * * *'
+    branches:
+      - "main"
     
 env:
   DBT_PROFILES_DIR: ./
@@ -42,5 +41,5 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run -m tag:decoded_logs
+          dbt run -m "arbitrum_models,tag:decoded_logs"
           
\ No newline at end of file
diff --git a/.github/workflows/dbt_run_streamline_decoder_history.yml b/.github/workflows/dbt_run_streamline_decoder_history.yml
index 9d42dde..0ca085f 100644
--- a/.github/workflows/dbt_run_streamline_decoder_history.yml
+++ b/.github/workflows/dbt_run_streamline_decoder_history.yml
@@ -42,5 +42,5 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120}' -m tag:streamline_decoded_logs_complete tag:streamline_decoded_logs_history
+          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120}' -m "arbitrum_models,tag:streamline_decoded_logs_complete" "arbitrum_models,tag:streamline_decoded_logs_history"
           
\ No newline at end of file
diff --git a/.github/workflows/dbt_run_streamline_history_adhoc.yml b/.github/workflows/dbt_run_streamline_history_adhoc.yml
index 77394c3..c2e9b89 100644
--- a/.github/workflows/dbt_run_streamline_history_adhoc.yml
+++ b/.github/workflows/dbt_run_streamline_history_adhoc.yml
@@ -29,8 +29,8 @@ on:
         description: 'DBT Run Command'
         required: true
         options:
-          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m tag:streamline_core_complete tag:streamline_core_history
-          - dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120}' -m tag:streamline_decoded_logs_complete tag:streamline_decoded_logs_history
+          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "arbitrum_models,tag:streamline_core_complete" "arbitrum_models,tag:streamline_core_history"
+          - dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120}' -m "arbitrum_models,tag:streamline_decoded_logs_complete" "arbitrum_models,tag:streamline_decoded_logs_history"
     
 env:
   DBT_PROFILES_DIR: ./
diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml
index d50ca8e..556b0fe 100644
--- a/.github/workflows/dbt_test_daily.yml
+++ b/.github/workflows/dbt_test_daily.yml
@@ -42,5 +42,5 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt test --exclude tag:full_test tag:recent_test
+          dbt test --exclude "arbitrum_models,tag:full_test" "arbitrum_models,tag:recent_test" livequery_models
   
\ No newline at end of file
diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml
index 56a1826..f6151de 100644
--- a/.github/workflows/dbt_test_intraday.yml
+++ b/.github/workflows/dbt_test_intraday.yml
@@ -42,6 +42,6 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run -m tag:observability
-          dbt test -m tag:recent_test
+          dbt run -m "arbitrum_models,tag:observability"
+          dbt test -m "arbitrum_models,tag:recent_test"
           
\ No newline at end of file
diff --git a/.github/workflows/dbt_test_monthly.yml b/.github/workflows/dbt_test_monthly.yml
index c535be4..359136c 100644
--- a/.github/workflows/dbt_test_monthly.yml
+++ b/.github/workflows/dbt_test_monthly.yml
@@ -42,5 +42,5 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt test -m tag:full_test
+          dbt test -m "arbitrum_models,tag:full_test"
           
\ No newline at end of file
diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml
new file mode 100644
index 0000000..976440b
--- /dev/null
+++ b/.github/workflows/dbt_test_tasks.yml
@@ -0,0 +1,27 @@
+name: dbt_test_tasks
+run-name: dbt_test_tasks
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main
+    secrets: inherit
\ No newline at end of file
diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv
new file mode 100644
index 0000000..88d6e19
--- /dev/null
+++ b/data/github_actions__workflows.csv
@@ -0,0 +1,6 @@
+workflow_name,workflow_schedule
+dbt_run_scheduled_non_realtime,"3,15,27,39,51 * * * *"
+dbt_run_streamline_chainhead,"9,21,33,45,57 * * * *"
+dbt_run_streamline_decoder,"0,12,24,36,48 * * * *"
+dbt_run_scheduled_curated,"20,50 * * * *"
+dbt_test_tasks,"5,35 * * * *"
\ No newline at end of file
diff --git a/dbt_project.yml b/dbt_project.yml
index d351f34..cb815d3 100644
--- a/dbt_project.yml
+++ b/dbt_project.yml
@@ -63,4 +63,5 @@ vars:
   UPDATE_SNOWFLAKE_TAGS: True
   WAIT: 0
   OBSERV_FULL_TEST: False
-  HEAL_MODEL: False
\ No newline at end of file
+  HEAL_MODEL: False
+  START_GHA_TASKS: False
\ No newline at end of file
diff --git a/models/github_actions/github_actions__current_task_status.sql b/models/github_actions/github_actions__current_task_status.sql
new file mode 100644
index 0000000..577a226
--- /dev/null
+++ b/models/github_actions/github_actions__current_task_status.sql
@@ -0,0 +1,6 @@
+{{ config(
+    materialized = 'view',
+    tags = ['gha_tasks']
+) }}
+
+{{ fsc_utils.gha_task_current_status_view() }}
\ No newline at end of file
diff --git a/models/github_actions/github_actions__current_task_status.yml b/models/github_actions/github_actions__current_task_status.yml
new file mode 100644
index 0000000..e28300d
--- /dev/null
+++ b/models/github_actions/github_actions__current_task_status.yml
@@ -0,0 +1,17 @@
+version: 2
+models:
+  - name: github_actions__current_task_status
+    columns:
+      - name: WAS_SUCCESSFUL
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_set:
+              value_set:
+                - TRUE
+      - name: RETURN_VALUE
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_set:
+              value_set:
+                - 204
+              config: 
+                  severity: warn
+                  warn_if: ">0"
diff --git a/models/github_actions/github_actions__task_history.sql b/models/github_actions/github_actions__task_history.sql
new file mode 100644
index 0000000..9c35ce7
--- /dev/null
+++ b/models/github_actions/github_actions__task_history.sql
@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_task_history_view() }}
\ No newline at end of file
diff --git a/models/github_actions/github_actions__task_performance.sql b/models/github_actions/github_actions__task_performance.sql
new file mode 100644
index 0000000..117ded5
--- /dev/null
+++ b/models/github_actions/github_actions__task_performance.sql
@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_task_performance_view() }}
\ No newline at end of file
diff --git a/models/github_actions/github_actions__task_schedule.sql b/models/github_actions/github_actions__task_schedule.sql
new file mode 100644
index 0000000..ff95a44
--- /dev/null
+++ b/models/github_actions/github_actions__task_schedule.sql
@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_task_schedule_view() }}
\ No newline at end of file
diff --git a/models/github_actions/github_actions__tasks.sql b/models/github_actions/github_actions__tasks.sql
new file mode 100644
index 0000000..feab82a
--- /dev/null
+++ b/models/github_actions/github_actions__tasks.sql
@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_tasks_view() }}
\ No newline at end of file
diff --git a/models/sources.yml b/models/sources.yml
index c29321f..6bcf749 100644
--- a/models/sources.yml
+++ b/models/sources.yml
@@ -53,3 +53,8 @@ sources:
     schema: bronze_api
     tables:
       - name: contract_abis
+  - name: github_actions
+    database: polygon
+    schema: github_actions
+    tables:
+      - name: workflows
diff --git a/packages.yml b/packages.yml
index f6e3a9f..7bb0ca9 100644
--- a/packages.yml
+++ b/packages.yml
@@ -6,6 +6,6 @@ packages:
   - package: dbt-labs/dbt_utils
     version: 1.0.0
   - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: v1.6.2
+    revision: v1.9.3
   - package: get-select/dbt_snowflake_query_tags
     version: [">=2.0.0", "<3.0.0"]
\ No newline at end of file