tags and schedule

.github/workflows/dbt_run_daily.yml

@@ -29,7 +29,6 @@ jobs:
       dbt_command: |
           dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
           dbt run -s "osmosis_models,tag:balances" "osmosis_models,tag:daily"
-          dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
       environment: workflow_prod
       warehouse: ${{ vars.WAREHOUSE }}
     secrets: inherit 

.github/workflows/dbt_run_incremental_core.yml

@@ -1,5 +1,5 @@
-name: dbt_run_incremental
-run-name: dbt_run_incremental
+name: dbt_run_incremental_core
+run-name: dbt_run_incremental_core
 
 on:
   workflow_dispatch:
@@ -28,8 +28,7 @@ jobs:
     with:
       dbt_command: |
           dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
-          dbt run -s "osmosis_models,./models" --exclude models/silver/_observability models/streamline tag:balances  tag:daily
-          dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
+          dbt run -s "osmosis_models,tag:core" 
       environment: workflow_prod
       warehouse: ${{ vars.WAREHOUSE }}
     secrets: inherit 

.github/workflows/dbt_run_incremental_non_core.yml

@@ -0,0 +1,34 @@
+name: dbt_run_incremental_non_core
+run-name: dbt_run_incremental_non_core
+
+on:
+  workflow_dispatch:
+  # schedule:
+  #   - cron: '25,55 * * * *'
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
+    with:
+      dbt_command: |
+          dbt run -s "osmosis_models,tag:noncore" 
+      environment: workflow_prod
+      warehouse: ${{ vars.WAREHOUSE }}
+    secrets: inherit 
+

data/github_actions__workflows.csv

@@ -1,5 +1,6 @@
 workflow_name,workflow_schedule
-dbt_run_incremental,"25,55 * * * *"
+dbt_run_incremental_core,"25,55 * * * *"
+dbt_run_incremental_non_core,"33 * * * *"
 dbt_run_streamline_blocks_txcount_realtime,"0,30 * * * *"
 dbt_run_streamline_transactions_realtime,"8,38 * * * *"
 dbt_test_tasks,"0,30 * * * *"

models/bronze/bronze__transactions_2.sql

@@ -3,7 +3,8 @@
   incremental_strategy = 'delete+insert',
   unique_key = 'id',
   cluster_by = ['_inserted_timestamp::date','block_timestamp::date'],
-  post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
+  post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
+  tags = ['core']
 ) }}
 -- depends_on: {{ ref('bronze__streamline_transactions') }}
 

models/bronze/core/bronze__streamline_FR_blocks.sql

@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['core']
 ) }}
 
 {% set model = this.identifier.split("_") [-1] %}

models/bronze/core/bronze__streamline_FR_transactions.sql

@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['core']
 ) }}
 
 {% set model = this.identifier.split("_") [-1] %}

models/bronze/core/bronze__streamline_FR_txcount.sql

@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['core']
 ) }}
 
 {% set model = this.identifier.split("_") [-1] %}

models/bronze/core/bronze__streamline_blocks.sql

@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['core']
 ) }}
 
 {% set model = this.identifier.split("_") [-1] %}

models/bronze/core/bronze__streamline_transactions.sql

@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['core']
 ) }}
 
 {% set model = this.identifier.split("_") [-1] %}

models/bronze/core/bronze__streamline_txcount.sql

@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['core']
 ) }}
 
 {% set model = this.identifier.split("_") [-1] %}

models/silver/core/silver__asset_metadata.sql

@@ -1,5 +1,6 @@
 {{ config(
-  materialized = 'table'
+  materialized = 'table',
+  tags = ['daily']
 ) }}
 
 WITH base AS (

models/silver/core/silver__blockchain.sql

@@ -3,6 +3,7 @@
     unique_key = "block_id",
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH base AS (

models/silver/core/silver__blocks.sql

@@ -2,7 +2,8 @@
   materialized = 'incremental',
   unique_key = ['chain_id','block_id'],
   incremental_strategy = 'merge',
-  cluster_by = ['block_timestamp::DATE']
+  cluster_by = ['block_timestamp::DATE'],
+  tags = ['core']
 ) }}
 -- depends_on: {{ ref('bronze__streamline_blocks') }}
 

models/silver/core/silver__msg_attributes.sql

@@ -3,7 +3,8 @@
   unique_key = "_unique_key",
   incremental_strategy = 'merge',
   cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
-  post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
+  post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
+  tags = ['core']
 ) }}
 
 SELECT

models/silver/core/silver__msgs.sql

@@ -3,7 +3,8 @@
   unique_key = "_unique_key",
   incremental_strategy = 'merge',
   cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
-  post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
+  post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
+  tags = ['core']
 ) }}
 
 WITH b AS (

models/silver/core/silver__transactions.sql

@@ -3,7 +3,8 @@
   unique_key = "tx_id",
   incremental_strategy = 'merge',
   cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
-  post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_id)"
+  post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_id)",
+  tags = ['core']
 ) }}
 -- depends_on: {{ ref('bronze__streamline_transactions') }}
 WITH sl AS (

models/silver/core/silver__transfers.sql

@@ -3,7 +3,8 @@
     materialized = 'incremental',
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE']
+    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
+    tags = ['core']
 ) }}
 
 {% if execute %}

models/silver/core/silver__tx_body_msgs.sql

@@ -2,7 +2,8 @@
   materialized = 'incremental',
   unique_key = "_unique_key",
   incremental_strategy = 'merge',
-  cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE']
+  cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
+  tags = ['core']
 ) }}
 
 WITH b AS (

models/silver/crossdb/silver__croschain_icns.sql

@@ -1,5 +1,6 @@
 {{ config(
-    materialized = 'table'
+    materialized = 'table',
+    tags = ['daily']
 ) }}
 
 SELECT

models/silver/crossdb/silver__croschain_labels.sql

@@ -1,5 +1,6 @@
 {{ config(
-    materialized = 'table'
+    materialized = 'table',
+    tags = ['daily']
 ) }}
 
 SELECT

models/silver/defi/silver___early_swaps.sql

@@ -2,7 +2,7 @@
     materialized = 'incremental',
     unique_key = "tx_id",
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp::DATE'],
+    cluster_by = ['block_timestamp::DATE']
 ) }}
 
 WITH

models/silver/defi/silver__liquidity_provider_actions.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp::DATE']
+    cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH

models/silver/defi/silver__liquidity_provider_actions_unpool.sql

@@ -3,6 +3,7 @@
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH

models/silver/defi/silver__locked_liquidity_actions.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE']
+    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH

models/silver/defi/silver__locked_liquidity_actions_begin_unlock.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE']
+    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH

models/silver/defi/silver__locked_liquidity_actions_convert.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE']
+    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH

models/silver/defi/silver__locked_liquidity_balances.sql

@@ -3,6 +3,7 @@
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH all_staked AS (

models/silver/defi/silver__locked_liquidity_transactions.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
-    cluster_by = ['_inserted_timestamp::DATE']
+    cluster_by = ['_inserted_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH count_ids AS (

models/silver/defi/silver__pool_balances.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = ["pool_id","block_id"],
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp']
+    cluster_by = ['block_timestamp'],
+    tags = ['noncore']
 ) }}
 
 SELECT

models/silver/defi/silver__pool_fee_summary_day.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = ["block_date","pool_id","currency"],
     incremental_strategy = 'merge',
-    cluster_by = ['block_date']
+    cluster_by = ['block_date'],
+    tags = ['noncore']
 ) }}
 
 WITH last_block_of_day AS (

models/silver/defi/silver__pool_metadata.sql

@@ -2,6 +2,7 @@
     materialized = 'incremental',
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
+    tags = ['noncore']
 ) }}
 
 WITH base_atts AS (

models/silver/defi/silver__pool_summary_hour.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = ["pool_id","block_hour"],
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp']
+    cluster_by = ['block_timestamp'],
+    tags = ['noncore']
 ) }}
 
 WITH pool_token_prices AS (

models/silver/defi/silver__swaps.sql

@@ -3,6 +3,7 @@
     unique_key = "CONCAT_WS('-', tx_id, _body_index)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH

models/silver/defi/silver__token_summary_day.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = ["currency","block_date"],
     incremental_strategy = 'merge',
-    cluster_by = ['block_date']
+    cluster_by = ['block_date'],
+    tags = ['noncore']
 ) }}
 
 WITH last_block_of_day AS (

models/silver/defi/silver__token_swapped.sql

@@ -3,6 +3,7 @@
     unique_key = ["tx_id","msg_index"],
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH bhour AS (

models/silver/gov/silver__governance_proposal_deposits.sql

@@ -3,6 +3,7 @@
     unique_key = "tx_id",
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH

models/silver/gov/silver__governance_submit_proposal.sql

@@ -3,6 +3,7 @@
     unique_key = "tx_id",
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH

models/silver/gov/silver__governance_votes.sql

@@ -3,6 +3,7 @@
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH base_tx AS (

models/silver/gov/silver__staked_balances.sql

@@ -2,6 +2,7 @@
     materialized = 'incremental',
     incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp'],
+    tags = ['noncore']
 ) }}
 
 WITH all_staked AS (

models/silver/gov/silver__staking.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp::DATE']
+    cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH

models/silver/gov/silver__staking_rewards.sql

@@ -3,6 +3,7 @@
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH

models/silver/gov/silver__superfluid_actions.sql

@@ -3,6 +3,7 @@
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
     cluster_by = ['_inserted_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 SELECT

models/silver/gov/silver__superfluid_staked_balances.sql

@@ -3,6 +3,7 @@
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH super AS (

models/silver/gov/silver__superfluid_staking.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp::DATE']
+    cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH

models/silver/gov/silver__validator_commission.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = ["tx_id","msg_group","msg_sub_group"],
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp::DATE']
+    cluster_by = ['block_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH txs AS (

models/silver/gov/silver__validator_memos.sql

@@ -1,7 +1,8 @@
 {{ config(
     materialized = 'incremental',
     unique_key = 'proposal_id',
-    incremental_strategy = 'delete+insert'
+    incremental_strategy = 'delete+insert',
+    tags = ['noncore']
 ) }}
 
 WITH base AS (

models/silver/gov/silver__validator_metadata.sql

@@ -1,5 +1,6 @@
 {{ config(
-  materialized = 'table'
+  materialized = 'table',
+  tags = ['daily']
 ) }}
 
 SELECT
@@ -18,20 +19,19 @@ SELECT
   VALUE :min_self_delegation :: NUMBER AS min_self_delegation,
   VALUE :rank :: NUMBER AS RANK,
   VALUE :uptime :missed_blocks :: NUMBER AS missed_blocks,
-  VALUE AS raw_metadata, 
+  VALUE AS raw_metadata,
   concat_ws(
-        '-',
-        address, 
-        creator, 
-        blockchain
-    ) AS _unique_key, 
+    '-',
+    address,
+    creator,
+    blockchain
+  ) AS _unique_key,
   VALUE :update_time :: TIMESTAMP AS _inserted_timestamp
 FROM
   {{ source(
     'bronze_streamline',
     'validator_metadata_api'
   ) }}
-
   qualify(ROW_NUMBER() over(PARTITION BY blockchain, creator, address
 ORDER BY
   _inserted_timestamp DESC)) = 1

models/silver/mars/silver__red_bank_actions.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = ["tx_id","msg_index"],
     incremental_strategy = 'merge',
-    cluster_by = ['block_timestamp::DATE','action']
+    cluster_by = ['block_timestamp::DATE','action'],
+    tags = ['noncore']
 ) }}
 
 WITH mars_contracts AS (

models/silver/prices/silver__pool_token_prices.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = ["token_address","pool_id","block_id"],
     incremental_strategy = 'merge',
-    cluster_by = ['_inserted_timestamp::DATE']
+    cluster_by = ['_inserted_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH top_pools AS (

models/silver/prices/silver__pool_token_prices_usd.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = ["token_address","block_id"],
     incremental_strategy = 'merge',
-    cluster_by = ['_inserted_timestamp::DATE']
+    cluster_by = ['_inserted_timestamp::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH osmo_price AS (

models/silver/prices/silver__prices_swaps.sql

@@ -3,6 +3,7 @@
     unique_key = ["block_hour", "currency"],
     incremental_strategy = 'merge',
     cluster_by = ['block_hour::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH swaps AS (

models/silver/prices/silver__token_prices_coin_gecko_hourly.sql

@@ -3,6 +3,7 @@
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
     cluster_by = ['recorded_hour::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH date_hours AS (

models/silver/prices/silver__token_prices_coin_market_cap_hourly.sql

@@ -3,6 +3,7 @@
     unique_key = "_unique_key",
     incremental_strategy = 'merge',
     cluster_by = ['recorded_hour::DATE'],
+    tags = ['noncore']
 ) }}
 
 WITH date_hours AS (