From 99c4d47ab2b80b30d3be045792804bcdab8b8254 Mon Sep 17 00:00:00 2001
From: Eric Laurello <eric.laurello@flipsidecrypto.com>
Date: Fri, 8 Sep 2023 13:54:45 -0400
Subject: [PATCH] inc logic

---
 .github/workflows/dbt_run_daily.yml           | 36 +++++++++++++++++++
 .github/workflows/dbt_run_incremental.yml     |  6 ++--
 .../gold/core/core__fact_daily_balances.sql   |  9 ++---
 models/silver/core/silver__daily_balances.sql |  1 +
 .../core/silver__latest_liquid_balances.sql   |  3 +-
 .../silver/core/silver__liquid_balances.sql   |  1 +
 6 files changed, 45 insertions(+), 11 deletions(-)
 create mode 100644 .github/workflows/dbt_run_daily.yml

diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml
new file mode 100644
index 0000000..070a95e
--- /dev/null
+++ b/.github/workflows/dbt_run_daily.yml
@@ -0,0 +1,36 @@
+name: dbt_run_daily
+run-name: dbt_run_daily
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '5 3 * * *'
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
+    with:
+      dbt_command: |
+          dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
+          dbt run -s tag:balances 
+          dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
+      environment: workflow_prod
+      warehouse: ${{ vars.WAREHOUSE }}
+    secrets: inherit 
+
diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml
index 1fccfbc..c8bbd65 100644
--- a/.github/workflows/dbt_run_incremental.yml
+++ b/.github/workflows/dbt_run_incremental.yml
@@ -1,5 +1,5 @@
-name: dbt_run_scheduled
-run-name: dbt_run_scheduled
+name: dbt_run_incremental
+run-name: dbt_run_incremental
 
 on:
   workflow_dispatch:
@@ -28,7 +28,7 @@ jobs:
     with:
       dbt_command: |
           dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
-          dbt run -s ./models --exclude models/silver/_observability/silver_observability*
+          dbt run -s ./models --exclude models/silver/_observability  tag:balances 
           dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
       environment: workflow_prod
       warehouse: ${{ vars.WAREHOUSE }}
diff --git a/models/gold/core/core__fact_daily_balances.sql b/models/gold/core/core__fact_daily_balances.sql
index b0bff94..6d1c16a 100644
--- a/models/gold/core/core__fact_daily_balances.sql
+++ b/models/gold/core/core__fact_daily_balances.sql
@@ -3,13 +3,8 @@
     unique_key = "CONCAT_WS('-', date, address, balance_type, currency)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['date'],
-      meta={
-        'database_tags':{
-            'table': {
-                'PURPOSE': 'BALANCES'
-            }
-        }
-      }
+    tags = ['balances'],
+    meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'BALANCES' }} }
 ) }}
 
 SELECT
diff --git a/models/silver/core/silver__daily_balances.sql b/models/silver/core/silver__daily_balances.sql
index 04e0e1a..38df63b 100644
--- a/models/silver/core/silver__daily_balances.sql
+++ b/models/silver/core/silver__daily_balances.sql
@@ -3,6 +3,7 @@
     unique_key = ["date", "address", "balance_type", "currency"],
     incremental_strategy = 'delete+insert',
     cluster_by = ['date'],
+    tags = ['balances']
 ) }}
 
 WITH
diff --git a/models/silver/core/silver__latest_liquid_balances.sql b/models/silver/core/silver__latest_liquid_balances.sql
index e962f74..2cbd99a 100644
--- a/models/silver/core/silver__latest_liquid_balances.sql
+++ b/models/silver/core/silver__latest_liquid_balances.sql
@@ -2,7 +2,8 @@
   materialized = 'incremental',
   unique_key = "address",
   incremental_strategy = 'delete+insert',
-  cluster_by = ['block_timestamp::date']
+  cluster_by = ['block_timestamp::date'],
+  tags = ['balances']
 ) }}
 
 WITH max_bn AS (
diff --git a/models/silver/core/silver__liquid_balances.sql b/models/silver/core/silver__liquid_balances.sql
index daf58b9..3951a7c 100644
--- a/models/silver/core/silver__liquid_balances.sql
+++ b/models/silver/core/silver__liquid_balances.sql
@@ -3,6 +3,7 @@
     unique_key = "CONCAT_WS('-', block_id, address, currency)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['balances']
 ) }}
 
 WITH base AS (