inc logic

.github/workflows/dbt_run_daily.yml

@@ -0,0 +1,36 @@
+name: dbt_run_daily
+run-name: dbt_run_daily
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '5 3 * * *'
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
+    with:
+      dbt_command: |
+          dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
+          dbt run -s tag:balances 
+          dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
+      environment: workflow_prod
+      warehouse: ${{ vars.WAREHOUSE }}
+    secrets: inherit 
+

.github/workflows/dbt_run_incremental.yml

@@ -1,5 +1,5 @@
-name: dbt_run_scheduled
-run-name: dbt_run_scheduled
+name: dbt_run_incremental
+run-name: dbt_run_incremental
 
 on:
   workflow_dispatch:
@@ -28,7 +28,7 @@ jobs:
     with:
       dbt_command: |
           dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
-          dbt run -s ./models --exclude models/silver/_observability/silver_observability*
+          dbt run -s ./models --exclude models/silver/_observability  tag:balances 
           dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
       environment: workflow_prod
       warehouse: ${{ vars.WAREHOUSE }}

models/gold/core/core__fact_daily_balances.sql

@@ -3,13 +3,8 @@
     unique_key = "CONCAT_WS('-', date, address, balance_type, currency)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['date'],
-      meta={
-        'database_tags':{
-            'table': {
-                'PURPOSE': 'BALANCES'
-            }
-        }
-      }
+    tags = ['balances'],
+    meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'BALANCES' }} }
 ) }}
 
 SELECT

models/silver/core/silver__daily_balances.sql

@@ -3,6 +3,7 @@
     unique_key = ["date", "address", "balance_type", "currency"],
     incremental_strategy = 'delete+insert',
     cluster_by = ['date'],
+    tags = ['balances']
 ) }}
 
 WITH

models/silver/core/silver__latest_liquid_balances.sql

@@ -2,7 +2,8 @@
   materialized = 'incremental',
   unique_key = "address",
   incremental_strategy = 'delete+insert',
-  cluster_by = ['block_timestamp::date']
+  cluster_by = ['block_timestamp::date'],
+  tags = ['balances']
 ) }}
 
 WITH max_bn AS (

models/silver/core/silver__liquid_balances.sql

@@ -3,6 +3,7 @@
     unique_key = "CONCAT_WS('-', block_id, address, currency)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['balances']
 ) }}
 
 WITH base AS (