diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index 2134636..9d1a23f 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -29,7 +29,6 @@ jobs: dbt_command: | dbt run-operation stage_external_sources --vars "ext_full_refresh: true" dbt run -s "osmosis_models,tag:balances" "osmosis_models,tag:daily" - dbt run-operation stage_external_sources --vars "ext_full_refresh: true" environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} secrets: inherit diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental_core.yml similarity index 76% rename from .github/workflows/dbt_run_incremental.yml rename to .github/workflows/dbt_run_incremental_core.yml index f4786ff..19591a2 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental_core.yml @@ -1,5 +1,5 @@ -name: dbt_run_incremental -run-name: dbt_run_incremental +name: dbt_run_incremental_core +run-name: dbt_run_incremental_core on: workflow_dispatch: @@ -28,8 +28,7 @@ jobs: with: dbt_command: | dbt run-operation stage_external_sources --vars "ext_full_refresh: true" - dbt run -s "osmosis_models,./models" --exclude models/silver/_observability models/streamline tag:balances tag:daily - dbt run-operation stage_external_sources --vars "ext_full_refresh: true" + dbt run -s "osmosis_models,tag:core" environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} secrets: inherit diff --git a/.github/workflows/dbt_run_incremental_non_core.yml b/.github/workflows/dbt_run_incremental_non_core.yml new file mode 100644 index 0000000..d6b098a --- /dev/null +++ b/.github/workflows/dbt_run_incremental_non_core.yml @@ -0,0 +1,34 @@ +name: dbt_run_incremental_non_core +run-name: dbt_run_incremental_non_core + +on: + workflow_dispatch: + # schedule: + # - cron: '25,55 * * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: | + dbt run -s "osmosis_models,tag:noncore" + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit + diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 15f323d..63513c7 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,5 +1,6 @@ workflow_name,workflow_schedule -dbt_run_incremental,"25,55 * * * *" +dbt_run_incremental_core,"25,55 * * * *" +dbt_run_incremental_non_core,"33 * * * *" dbt_run_streamline_blocks_txcount_realtime,"0,30 * * * *" dbt_run_streamline_transactions_realtime,"8,38 * * * *" dbt_test_tasks,"0,30 * * * *" diff --git a/models/bronze/bronze__transactions_2.sql b/models/bronze/bronze__transactions_2.sql index fb7814a..dd5d559 100644 --- a/models/bronze/bronze__transactions_2.sql +++ b/models/bronze/bronze__transactions_2.sql @@ -3,7 +3,8 @@ incremental_strategy = 'delete+insert', unique_key = 'id', cluster_by = ['_inserted_timestamp::date','block_timestamp::date'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION" + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core'] ) }} -- depends_on: {{ ref('bronze__streamline_transactions') }} diff --git a/models/bronze/core/bronze__streamline_FR_blocks.sql b/models/bronze/core/bronze__streamline_FR_blocks.sql index a430319..ee09608 100644 --- a/models/bronze/core/bronze__streamline_FR_blocks.sql +++ b/models/bronze/core/bronze__streamline_FR_blocks.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['core'] ) }} {% set model = this.identifier.split("_") [-1] %} diff --git a/models/bronze/core/bronze__streamline_FR_transactions.sql b/models/bronze/core/bronze__streamline_FR_transactions.sql index a430319..ee09608 100644 --- a/models/bronze/core/bronze__streamline_FR_transactions.sql +++ b/models/bronze/core/bronze__streamline_FR_transactions.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['core'] ) }} {% set model = this.identifier.split("_") [-1] %} diff --git a/models/bronze/core/bronze__streamline_FR_txcount.sql b/models/bronze/core/bronze__streamline_FR_txcount.sql index 796b55c..1329611 100644 --- a/models/bronze/core/bronze__streamline_FR_txcount.sql +++ b/models/bronze/core/bronze__streamline_FR_txcount.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['core'] ) }} {% set model = this.identifier.split("_") [-1] %} diff --git a/models/bronze/core/bronze__streamline_blocks.sql b/models/bronze/core/bronze__streamline_blocks.sql index 22b0c51..ce42165 100644 --- a/models/bronze/core/bronze__streamline_blocks.sql +++ b/models/bronze/core/bronze__streamline_blocks.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['core'] ) }} {% set model = this.identifier.split("_") [-1] %} diff --git a/models/bronze/core/bronze__streamline_transactions.sql b/models/bronze/core/bronze__streamline_transactions.sql index 22b0c51..ce42165 100644 --- a/models/bronze/core/bronze__streamline_transactions.sql +++ b/models/bronze/core/bronze__streamline_transactions.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['core'] ) }} {% set model = this.identifier.split("_") [-1] %} diff --git a/models/bronze/core/bronze__streamline_txcount.sql b/models/bronze/core/bronze__streamline_txcount.sql index da8e375..cf9325d 100644 --- a/models/bronze/core/bronze__streamline_txcount.sql +++ b/models/bronze/core/bronze__streamline_txcount.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + tags = ['core'] ) }} {% set model = this.identifier.split("_") [-1] %} diff --git a/models/silver/core/silver__asset_metadata.sql b/models/silver/core/silver__asset_metadata.sql index fe1bf75..839825d 100644 --- a/models/silver/core/silver__asset_metadata.sql +++ b/models/silver/core/silver__asset_metadata.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'table' + materialized = 'table', + tags = ['daily'] ) }} WITH base AS ( diff --git a/models/silver/core/silver__blockchain.sql b/models/silver/core/silver__blockchain.sql index d89fd03..019bdc3 100644 --- a/models/silver/core/silver__blockchain.sql +++ b/models/silver/core/silver__blockchain.sql @@ -3,6 +3,7 @@ unique_key = "block_id", incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH base AS ( diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index 99de194..4e9a727 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = ['chain_id','block_id'], incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE'], + tags = ['core'] ) }} -- depends_on: {{ ref('bronze__streamline_blocks') }} diff --git a/models/silver/core/silver__msg_attributes.sql b/models/silver/core/silver__msg_attributes.sql index 7514676..930d9c5 100644 --- a/models/silver/core/silver__msg_attributes.sql +++ b/models/silver/core/silver__msg_attributes.sql @@ -3,7 +3,8 @@ unique_key = "_unique_key", incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION" + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core'] ) }} SELECT diff --git a/models/silver/core/silver__msgs.sql b/models/silver/core/silver__msgs.sql index 55ac342..ba9b088 100644 --- a/models/silver/core/silver__msgs.sql +++ b/models/silver/core/silver__msgs.sql @@ -3,7 +3,8 @@ unique_key = "_unique_key", incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION" + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core'] ) }} WITH b AS ( diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index 409f544..8b13071 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -3,7 +3,8 @@ unique_key = "tx_id", incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_id)" + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_id)", + tags = ['core'] ) }} -- depends_on: {{ ref('bronze__streamline_transactions') }} WITH sl AS ( diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index 2d0214c..2dd8461 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -3,7 +3,8 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['core'] ) }} {% if execute %} diff --git a/models/silver/core/silver__tx_body_msgs.sql b/models/silver/core/silver__tx_body_msgs.sql index a801368..5c0a72e 100644 --- a/models/silver/core/silver__tx_body_msgs.sql +++ b/models/silver/core/silver__tx_body_msgs.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['core'] ) }} WITH b AS ( diff --git a/models/silver/crossdb/silver__croschain_icns.sql b/models/silver/crossdb/silver__croschain_icns.sql index aa6d1ce..4b2ff1d 100644 --- a/models/silver/crossdb/silver__croschain_icns.sql +++ b/models/silver/crossdb/silver__croschain_icns.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'table' + materialized = 'table', + tags = ['daily'] ) }} SELECT diff --git a/models/silver/crossdb/silver__croschain_labels.sql b/models/silver/crossdb/silver__croschain_labels.sql index 495df21..770cd4a 100644 --- a/models/silver/crossdb/silver__croschain_labels.sql +++ b/models/silver/crossdb/silver__croschain_labels.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'table' + materialized = 'table', + tags = ['daily'] ) }} SELECT diff --git a/models/silver/defi/silver___early_swaps.sql b/models/silver/defi/silver___early_swaps.sql index 54315c5..12b4a5d 100644 --- a/models/silver/defi/silver___early_swaps.sql +++ b/models/silver/defi/silver___early_swaps.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = "tx_id", incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE'], + cluster_by = ['block_timestamp::DATE'] ) }} WITH diff --git a/models/silver/defi/silver__liquidity_provider_actions.sql b/models/silver/defi/silver__liquidity_provider_actions.sql index cf1523c..3e388ee 100644 --- a/models/silver/defi/silver__liquidity_provider_actions.sql +++ b/models/silver/defi/silver__liquidity_provider_actions.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH diff --git a/models/silver/defi/silver__liquidity_provider_actions_unpool.sql b/models/silver/defi/silver__liquidity_provider_actions_unpool.sql index 4637c64..6800d6b 100644 --- a/models/silver/defi/silver__liquidity_provider_actions_unpool.sql +++ b/models/silver/defi/silver__liquidity_provider_actions_unpool.sql @@ -3,6 +3,7 @@ unique_key = "_unique_key", incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH diff --git a/models/silver/defi/silver__locked_liquidity_actions.sql b/models/silver/defi/silver__locked_liquidity_actions.sql index adf7569..92814d5 100644 --- a/models/silver/defi/silver__locked_liquidity_actions.sql +++ b/models/silver/defi/silver__locked_liquidity_actions.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['noncore'] ) }} WITH diff --git a/models/silver/defi/silver__locked_liquidity_actions_begin_unlock.sql b/models/silver/defi/silver__locked_liquidity_actions_begin_unlock.sql index 8f7ef9f..82b5056 100644 --- a/models/silver/defi/silver__locked_liquidity_actions_begin_unlock.sql +++ b/models/silver/defi/silver__locked_liquidity_actions_begin_unlock.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['noncore'] ) }} WITH diff --git a/models/silver/defi/silver__locked_liquidity_actions_convert.sql b/models/silver/defi/silver__locked_liquidity_actions_convert.sql index 2a8b4c1..0b8e317 100644 --- a/models/silver/defi/silver__locked_liquidity_actions_convert.sql +++ b/models/silver/defi/silver__locked_liquidity_actions_convert.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['noncore'] ) }} WITH diff --git a/models/silver/defi/silver__locked_liquidity_balances.sql b/models/silver/defi/silver__locked_liquidity_balances.sql index c3e4d8a..a8710d4 100644 --- a/models/silver/defi/silver__locked_liquidity_balances.sql +++ b/models/silver/defi/silver__locked_liquidity_balances.sql @@ -3,6 +3,7 @@ unique_key = "_unique_key", incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH all_staked AS ( diff --git a/models/silver/defi/silver__locked_liquidity_transactions.sql b/models/silver/defi/silver__locked_liquidity_transactions.sql index e0b2f40..f8aee17 100644 --- a/models/silver/defi/silver__locked_liquidity_transactions.sql +++ b/models/silver/defi/silver__locked_liquidity_transactions.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', - cluster_by = ['_inserted_timestamp::DATE'] + cluster_by = ['_inserted_timestamp::DATE'], + tags = ['noncore'] ) }} WITH count_ids AS ( diff --git a/models/silver/defi/silver__pool_balances.sql b/models/silver/defi/silver__pool_balances.sql index e94988d..42e12b6 100644 --- a/models/silver/defi/silver__pool_balances.sql +++ b/models/silver/defi/silver__pool_balances.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = ["pool_id","block_id"], incremental_strategy = 'merge', - cluster_by = ['block_timestamp'] + cluster_by = ['block_timestamp'], + tags = ['noncore'] ) }} SELECT diff --git a/models/silver/defi/silver__pool_fee_summary_day.sql b/models/silver/defi/silver__pool_fee_summary_day.sql index ae9fc25..a10e843 100644 --- a/models/silver/defi/silver__pool_fee_summary_day.sql +++ b/models/silver/defi/silver__pool_fee_summary_day.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = ["block_date","pool_id","currency"], incremental_strategy = 'merge', - cluster_by = ['block_date'] + cluster_by = ['block_date'], + tags = ['noncore'] ) }} WITH last_block_of_day AS ( diff --git a/models/silver/defi/silver__pool_metadata.sql b/models/silver/defi/silver__pool_metadata.sql index 03b9a89..e76e443 100644 --- a/models/silver/defi/silver__pool_metadata.sql +++ b/models/silver/defi/silver__pool_metadata.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', + tags = ['noncore'] ) }} WITH base_atts AS ( diff --git a/models/silver/defi/silver__pool_summary_hour.sql b/models/silver/defi/silver__pool_summary_hour.sql index 1edff04..5621050 100644 --- a/models/silver/defi/silver__pool_summary_hour.sql +++ b/models/silver/defi/silver__pool_summary_hour.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = ["pool_id","block_hour"], incremental_strategy = 'merge', - cluster_by = ['block_timestamp'] + cluster_by = ['block_timestamp'], + tags = ['noncore'] ) }} WITH pool_token_prices AS ( diff --git a/models/silver/defi/silver__swaps.sql b/models/silver/defi/silver__swaps.sql index 695a2c8..32c61ee 100644 --- a/models/silver/defi/silver__swaps.sql +++ b/models/silver/defi/silver__swaps.sql @@ -3,6 +3,7 @@ unique_key = "CONCAT_WS('-', tx_id, _body_index)", incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH diff --git a/models/silver/defi/silver__token_summary_day.sql b/models/silver/defi/silver__token_summary_day.sql index fb3c0a3..d8b9069 100644 --- a/models/silver/defi/silver__token_summary_day.sql +++ b/models/silver/defi/silver__token_summary_day.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = ["currency","block_date"], incremental_strategy = 'merge', - cluster_by = ['block_date'] + cluster_by = ['block_date'], + tags = ['noncore'] ) }} WITH last_block_of_day AS ( diff --git a/models/silver/defi/silver__token_swapped.sql b/models/silver/defi/silver__token_swapped.sql index 6cff944..140f90b 100644 --- a/models/silver/defi/silver__token_swapped.sql +++ b/models/silver/defi/silver__token_swapped.sql @@ -3,6 +3,7 @@ unique_key = ["tx_id","msg_index"], incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH bhour AS ( diff --git a/models/silver/gov/silver__governance_proposal_deposits.sql b/models/silver/gov/silver__governance_proposal_deposits.sql index 0012dae..a7a222f 100644 --- a/models/silver/gov/silver__governance_proposal_deposits.sql +++ b/models/silver/gov/silver__governance_proposal_deposits.sql @@ -3,6 +3,7 @@ unique_key = "tx_id", incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH diff --git a/models/silver/gov/silver__governance_submit_proposal.sql b/models/silver/gov/silver__governance_submit_proposal.sql index f900319..b4edd98 100644 --- a/models/silver/gov/silver__governance_submit_proposal.sql +++ b/models/silver/gov/silver__governance_submit_proposal.sql @@ -3,6 +3,7 @@ unique_key = "tx_id", incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH diff --git a/models/silver/gov/silver__governance_votes.sql b/models/silver/gov/silver__governance_votes.sql index 08cded5..48155b0 100644 --- a/models/silver/gov/silver__governance_votes.sql +++ b/models/silver/gov/silver__governance_votes.sql @@ -3,6 +3,7 @@ unique_key = "_unique_key", incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH base_tx AS ( diff --git a/models/silver/gov/silver__staked_balances.sql b/models/silver/gov/silver__staked_balances.sql index 6b2e7a5..a797b49 100644 --- a/models/silver/gov/silver__staked_balances.sql +++ b/models/silver/gov/silver__staked_balances.sql @@ -2,6 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp'], + tags = ['noncore'] ) }} WITH all_staked AS ( diff --git a/models/silver/gov/silver__staking.sql b/models/silver/gov/silver__staking.sql index 7990f6b..75f9546 100644 --- a/models/silver/gov/silver__staking.sql +++ b/models/silver/gov/silver__staking.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH diff --git a/models/silver/gov/silver__staking_rewards.sql b/models/silver/gov/silver__staking_rewards.sql index 8b67828..be553d0 100644 --- a/models/silver/gov/silver__staking_rewards.sql +++ b/models/silver/gov/silver__staking_rewards.sql @@ -3,6 +3,7 @@ unique_key = "_unique_key", incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH diff --git a/models/silver/gov/silver__superfluid_actions.sql b/models/silver/gov/silver__superfluid_actions.sql index e3b8ffc..86579a5 100644 --- a/models/silver/gov/silver__superfluid_actions.sql +++ b/models/silver/gov/silver__superfluid_actions.sql @@ -3,6 +3,7 @@ unique_key = "_unique_key", incremental_strategy = 'merge', cluster_by = ['_inserted_timestamp::DATE'], + tags = ['noncore'] ) }} SELECT diff --git a/models/silver/gov/silver__superfluid_staked_balances.sql b/models/silver/gov/silver__superfluid_staked_balances.sql index ca5f9c4..b51115c 100644 --- a/models/silver/gov/silver__superfluid_staked_balances.sql +++ b/models/silver/gov/silver__superfluid_staked_balances.sql @@ -3,6 +3,7 @@ unique_key = "_unique_key", incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH super AS ( diff --git a/models/silver/gov/silver__superfluid_staking.sql b/models/silver/gov/silver__superfluid_staking.sql index 48d8bc8..8d0738f 100644 --- a/models/silver/gov/silver__superfluid_staking.sql +++ b/models/silver/gov/silver__superfluid_staking.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = "_unique_key", incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH diff --git a/models/silver/gov/silver__validator_commission.sql b/models/silver/gov/silver__validator_commission.sql index bd7899e..9176681 100644 --- a/models/silver/gov/silver__validator_commission.sql +++ b/models/silver/gov/silver__validator_commission.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = ["tx_id","msg_group","msg_sub_group"], incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE'], + tags = ['noncore'] ) }} WITH txs AS ( diff --git a/models/silver/gov/silver__validator_memos.sql b/models/silver/gov/silver__validator_memos.sql index ebf8059..ef6a812 100644 --- a/models/silver/gov/silver__validator_memos.sql +++ b/models/silver/gov/silver__validator_memos.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', unique_key = 'proposal_id', - incremental_strategy = 'delete+insert' + incremental_strategy = 'delete+insert', + tags = ['noncore'] ) }} WITH base AS ( diff --git a/models/silver/gov/silver__validator_metadata.sql b/models/silver/gov/silver__validator_metadata.sql index d674fa5..2f587f1 100644 --- a/models/silver/gov/silver__validator_metadata.sql +++ b/models/silver/gov/silver__validator_metadata.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'table' + materialized = 'table', + tags = ['daily'] ) }} SELECT @@ -18,20 +19,19 @@ SELECT VALUE :min_self_delegation :: NUMBER AS min_self_delegation, VALUE :rank :: NUMBER AS RANK, VALUE :uptime :missed_blocks :: NUMBER AS missed_blocks, - VALUE AS raw_metadata, + VALUE AS raw_metadata, concat_ws( - '-', - address, - creator, - blockchain - ) AS _unique_key, + '-', + address, + creator, + blockchain + ) AS _unique_key, VALUE :update_time :: TIMESTAMP AS _inserted_timestamp FROM {{ source( 'bronze_streamline', 'validator_metadata_api' ) }} - qualify(ROW_NUMBER() over(PARTITION BY blockchain, creator, address ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/silver/mars/silver__red_bank_actions.sql b/models/silver/mars/silver__red_bank_actions.sql index 5945a66..bfb4514 100644 --- a/models/silver/mars/silver__red_bank_actions.sql +++ b/models/silver/mars/silver__red_bank_actions.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = ["tx_id","msg_index"], incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE','action'] + cluster_by = ['block_timestamp::DATE','action'], + tags = ['noncore'] ) }} WITH mars_contracts AS ( diff --git a/models/silver/prices/silver__pool_token_prices.sql b/models/silver/prices/silver__pool_token_prices.sql index dba4749..c25a6b3 100644 --- a/models/silver/prices/silver__pool_token_prices.sql +++ b/models/silver/prices/silver__pool_token_prices.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = ["token_address","pool_id","block_id"], incremental_strategy = 'merge', - cluster_by = ['_inserted_timestamp::DATE'] + cluster_by = ['_inserted_timestamp::DATE'], + tags = ['noncore'] ) }} WITH top_pools AS ( diff --git a/models/silver/prices/silver__pool_token_prices_usd.sql b/models/silver/prices/silver__pool_token_prices_usd.sql index ec9cc58..5142f6f 100644 --- a/models/silver/prices/silver__pool_token_prices_usd.sql +++ b/models/silver/prices/silver__pool_token_prices_usd.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = ["token_address","block_id"], incremental_strategy = 'merge', - cluster_by = ['_inserted_timestamp::DATE'] + cluster_by = ['_inserted_timestamp::DATE'], + tags = ['noncore'] ) }} WITH osmo_price AS ( diff --git a/models/silver/prices/silver__prices_swaps.sql b/models/silver/prices/silver__prices_swaps.sql index fca397b..72af4fa 100644 --- a/models/silver/prices/silver__prices_swaps.sql +++ b/models/silver/prices/silver__prices_swaps.sql @@ -3,6 +3,7 @@ unique_key = ["block_hour", "currency"], incremental_strategy = 'merge', cluster_by = ['block_hour::DATE'], + tags = ['noncore'] ) }} WITH swaps AS ( diff --git a/models/silver/prices/silver__token_prices_coin_gecko_hourly.sql b/models/silver/prices/silver__token_prices_coin_gecko_hourly.sql index a8076d7..1b5f0b2 100644 --- a/models/silver/prices/silver__token_prices_coin_gecko_hourly.sql +++ b/models/silver/prices/silver__token_prices_coin_gecko_hourly.sql @@ -3,6 +3,7 @@ unique_key = "_unique_key", incremental_strategy = 'merge', cluster_by = ['recorded_hour::DATE'], + tags = ['noncore'] ) }} WITH date_hours AS ( diff --git a/models/silver/prices/silver__token_prices_coin_market_cap_hourly.sql b/models/silver/prices/silver__token_prices_coin_market_cap_hourly.sql index 71b5a45..4025b57 100644 --- a/models/silver/prices/silver__token_prices_coin_market_cap_hourly.sql +++ b/models/silver/prices/silver__token_prices_coin_market_cap_hourly.sql @@ -3,6 +3,7 @@ unique_key = "_unique_key", incremental_strategy = 'merge', cluster_by = ['recorded_hour::DATE'], + tags = ['noncore'] ) }} WITH date_hours AS (