add tasks (#296)

.github/workflows/dbt_alter_gha_task.yml

@@ -0,0 +1,46 @@
+name: dbt_alter_gha_task
+run-name: dbt_alter_gha_task
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    inputs:
+      workflow_name:
+        type: string
+        description: Name of the workflow to perform the action on, no .yml extension
+        required: true
+      task_action:
+        type: choice 
+        description: Action to perform
+        required: true
+        options:
+          - SUSPEND
+          - RESUME
+        default: SUSPEND
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main
+    with:
+      workflow_name: |
+         ${{ inputs.workflow_name }}
+      task_action: |
+         ${{ inputs.task_action }}
+      environment: workflow_prod
+    secrets: inherit

.github/workflows/dbt_run_streamline_asset_metadata.yml

@@ -3,9 +3,8 @@ run-name: dbt_run_streamline_asset_metadata
 
 on:
   workflow_dispatch:
-  schedule:
-      # Runs at minute 5 every hour (see https://crontab.guru)
-      - cron: '5 * * * *'  
+    branches:
+      - "main"
     
 env:
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"

.github/workflows/dbt_run_streamline_prices_history.yml

@@ -3,9 +3,8 @@ run-name: dbt_run_streamline_prices_history
 
 on:
   workflow_dispatch:
-  schedule:
-      # Runs once daily (see https://crontab.guru)
-      - cron: '45 * */1 * *'  
+    branches:
+      - "main"
     
 env:
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"

.github/workflows/dbt_run_streamline_prices_realtime.yml

@@ -3,9 +3,8 @@ run-name: dbt_run_streamline_prices_realtime
 
 on:
   workflow_dispatch:
-  schedule:
-      # Runs at minute 25 every hour (see https://crontab.guru)
-      - cron: '25 * * * *'  
+    branches:
+      - "main"
     
 env:
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"

.github/workflows/dbt_test_tasks.yml

@@ -0,0 +1,27 @@
+name: dbt_test_tasks
+run-name: dbt_test_tasks
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main
+    secrets: inherit

data/github_actions__workflows.csv

@@ -0,0 +1,5 @@
+workflow_name,workflow_schedule
+dbt_run_streamline_prices_history,"45 * */1 * *"
+dbt_run_streamline_prices_realtime,"25 * * * *"
+dbt_run_streamline_asset_metadata,"5 * * * *"
+dbt_test_tasks,"15 * * * *"

dbt_project.yml

@@ -27,8 +27,6 @@ clean-targets: # directories to be removed by `dbt clean`
 on-run-start:
   - "{{ create_sps() }}"
   - '{{ create_udfs() }}'
-  - '{{ sp_create_bulk_get_coin_gecko_asset_metadata() }}'
-  - '{{ sp_create_bulk_get_coin_market_cap_asset_metadata() }}'
   - '{{ sp_refresh_external_table_by_recent_date() }}'
 
 

macros/create_udfs.sql

@@ -2,11 +2,6 @@
     {% if var("UPDATE_UDFS_AND_SPS", False) %}
         {% set sql %}
             CREATE schema if NOT EXISTS streamline;
-            {{ udf_bulk_get_coin_gecko_asset_metadata() }};
-            {{ () }};
-            {{ udf_bulk_get_coin_market_cap_hourly_ohlc() }};
-            {{ udf_bulk_get_coin_gecko_ohlc() }};
-            {{ udf_bulk_get_coin_gecko_asset_market_data_historical() }};
             {{ create_udf_hex_to_int(
             schema = "public"
         ) }}

models/silver/github_actions/github_actions__current_task_status.sql

@@ -0,0 +1,6 @@
+{{ config(
+    materialized = 'view',
+    tags = ['gha_tasks']
+) }}
+
+{{ fsc_utils.gha_task_current_status_view() }}

models/silver/github_actions/github_actions__current_task_status.yml

@@ -0,0 +1,16 @@
+version: 2
+models:
+  - name: github_actions__current_task_status
+    columns:
+      - name: PIPELINE_ACTIVE
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_set:
+              value_set:
+                - TRUE
+      - name: SUCCESSES
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_set:
+              value_set:
+                - 2
+              config: 
+                  severity: warn

models/silver/github_actions/github_actions__task_history.sql

@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_task_history_view() }}

models/silver/github_actions/github_actions__task_performance.sql

@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_task_performance_view() }}

models/silver/github_actions/github_actions__task_schedule.sql

@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_task_schedule_view() }}

models/silver/github_actions/github_actions__tasks.sql

@@ -0,0 +1,5 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+{{ fsc_utils.gha_tasks_view() }}