diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml index 14b565d6..1ecab024 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -41,7 +41,6 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run-operation run_sp_refresh_external_tables_full dbt run -s "solana_models,tag:scheduled_core" notify-failure: diff --git a/.github/workflows/dbt_run_incremental_intermediate.yml b/.github/workflows/dbt_run_incremental_intermediate.yml index 00175a21..6118afd8 100644 --- a/.github/workflows/dbt_run_incremental_intermediate.yml +++ b/.github/workflows/dbt_run_incremental_intermediate.yml @@ -41,7 +41,6 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run-operation run_sp_refresh_external_tables_full dbt run -s "solana_models,tag:scheduled_core" notify-failure: diff --git a/.github/workflows/dbt_run_incremental_non_core.yml b/.github/workflows/dbt_run_incremental_non_core.yml index 7dc1ed3b..66b549c0 100644 --- a/.github/workflows/dbt_run_incremental_non_core.yml +++ b/.github/workflows/dbt_run_incremental_non_core.yml @@ -42,7 +42,6 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run-operation run_sp_refresh_external_tables_full dbt run -s "solana_models,tag:scheduled_core" "solana_models,tag:scheduled_non_core" notify-failure: diff --git a/models/gold/gov/gov__fact_block_production.sql b/models/gold/gov/gov__fact_block_production.sql index 8d8e7934..7b00284f 100644 --- a/models/gold/gov/gov__fact_block_production.sql +++ b/models/gold/gov/gov__fact_block_production.sql @@ -5,7 +5,8 @@ cluster_by = ['epoch'], merge_exclude_columns = ["inserted_timestamp"], post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(node_pubkey)'), - tags = ['scheduled_non_core'] + tags = ['scheduled_non_core'], + full_refresh = false ) }} {% if execute %} @@ -53,6 +54,8 @@ FROM LEFT JOIN {{ ref('silver__epoch') }} AS e ON bp.epoch = e.epoch + -- historical data -- tables static and disabled, and manual change needed in rare case where fr is needed +{# {% if not is_incremental() %} UNION ALL SELECT @@ -70,3 +73,4 @@ SELECT FROM {{ ref('silver__historical_block_production') }} {% endif %} +#} diff --git a/models/gold/gov/gov__fact_stake_accounts.sql b/models/gold/gov/gov__fact_stake_accounts.sql index 0b247df6..0c4ffded 100644 --- a/models/gold/gov/gov__fact_stake_accounts.sql +++ b/models/gold/gov/gov__fact_stake_accounts.sql @@ -6,7 +6,8 @@ cluster_by = ['epoch', 'activation_epoch', 'deactivation_epoch'], merge_exclude_columns = ["inserted_timestamp"], post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(stake_pubkey, vote_pubkey)'), - tags = ['scheduled_non_core_hourly'] + tags = ['scheduled_non_core_hourly'], + full_refresh = false ) }} @@ -51,6 +52,8 @@ WHERE {% if is_incremental() %} AND modified_timestamp >= '{{ max_modified_timestamp }}' {% endif %} +-- historical data -- tables static and disabled, and manual change needed in rare case where fr is needed +{# UNION ALL SELECT epoch_recorded::INT AS epoch, @@ -104,3 +107,4 @@ SELECT FROM {{ ref('silver__historical_stake_account') }} {% endif %} +#} diff --git a/models/gold/gov/gov__fact_validators.sql b/models/gold/gov/gov__fact_validators.sql index dab6a345..b2fe352a 100644 --- a/models/gold/gov/gov__fact_validators.sql +++ b/models/gold/gov/gov__fact_validators.sql @@ -5,7 +5,8 @@ cluster_by = ['epoch','epoch_active'], merge_exclude_columns = ["inserted_timestamp"], post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(node_pubkey, vote_pubkey)'), - tags = ['scheduled_non_core'] + tags = ['scheduled_non_core'], + full_refresh = false ) }} {% if execute %} @@ -61,6 +62,8 @@ FROM WHERE modified_timestamp >= '{{ max_modified_timestamp }}' {% endif %} +-- historical data -- tables static and disabled, and manual change needed in rare case where fr is needed +{# {% if not is_incremental() %} UNION ALL SELECT @@ -93,3 +96,4 @@ SELECT FROM {{ ref('silver__historical_validator_app_data') }} {% endif %} +#} diff --git a/models/gold/gov/gov__fact_vote_accounts.sql b/models/gold/gov/gov__fact_vote_accounts.sql index de652db0..439d51e3 100644 --- a/models/gold/gov/gov__fact_vote_accounts.sql +++ b/models/gold/gov/gov__fact_vote_accounts.sql @@ -5,7 +5,8 @@ cluster_by = ['epoch','last_epoch_active'], merge_exclude_columns = ["inserted_timestamp"], post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(vote_pubkey, node_pubkey, owner)'), - tags = ['scheduled_non_core'] + tags = ['scheduled_non_core'], + full_refresh = false ) }} {% if execute %} @@ -56,6 +57,8 @@ FROM WHERE modified_timestamp >= '{{ max_modified_timestamp }}' {% endif %} +-- historical data -- tables static and disabled, and manual change needed in rare case where fr is needed +{# {% if not is_incremental() %} UNION ALL SELECT @@ -83,3 +86,4 @@ SELECT FROM {{ ref('silver__historical_vote_account') }} {% endif %} +#} diff --git a/models/silver/validator/silver__historical_block_production.sql b/models/silver/validator/silver__historical_block_production.sql index e1b8262c..d5ab27c4 100644 --- a/models/silver/validator/silver__historical_block_production.sql +++ b/models/silver/validator/silver__historical_block_production.sql @@ -1,6 +1,8 @@ {{ config( - materialized = 'view', - tags = ['validator_historical'] + materialized = 'table', + tags = ['validator_historical'], + full_refresh = false, + enabled = false ) }} SELECT diff --git a/models/silver/validator/silver__historical_stake_account.sql b/models/silver/validator/silver__historical_stake_account.sql index 37d3933c..043aea7f 100644 --- a/models/silver/validator/silver__historical_stake_account.sql +++ b/models/silver/validator/silver__historical_stake_account.sql @@ -1,6 +1,8 @@ {{ config ( - materialized = 'view', - tags = ['validator_historical'] + materialized = 'table', + tags = ['validator_historical'], + full_refresh = false, + enabled = false ) }} SELECT diff --git a/models/silver/validator/silver__historical_validator_app_data.sql b/models/silver/validator/silver__historical_validator_app_data.sql index 9d18ec1f..a37321d8 100644 --- a/models/silver/validator/silver__historical_validator_app_data.sql +++ b/models/silver/validator/silver__historical_validator_app_data.sql @@ -1,6 +1,8 @@ {{ config ( - materialized = 'view', - tags = ['validator_historical'] + materialized = 'table', + tags = ['validator_historical'], + full_refresh = false, + enabled = false ) }} WITH base AS ( diff --git a/models/silver/validator/silver__historical_vote_account.sql b/models/silver/validator/silver__historical_vote_account.sql index 040f9b00..1b9ce44d 100644 --- a/models/silver/validator/silver__historical_vote_account.sql +++ b/models/silver/validator/silver__historical_vote_account.sql @@ -1,6 +1,8 @@ {{ config ( - materialized = 'view', - tags = ['validator_historical'] + materialized = 'table', + tags = ['validator_historical'], + full_refresh = false, + enabled = false ) }} SELECT diff --git a/models/silver/validator/silver__snapshot_stake_accounts.sql b/models/silver/validator/silver__snapshot_stake_accounts.sql index 96783960..1a998ec4 100644 --- a/models/silver/validator/silver__snapshot_stake_accounts.sql +++ b/models/silver/validator/silver__snapshot_stake_accounts.sql @@ -3,7 +3,9 @@ unique_key = "CONCAT_WS('-', epoch_recorded, stake_pubkey)", incremental_strategy = 'delete+insert', cluster_by = ['modified_timestamp::DATE'], - tags = ['validator','exclude_test_daily'] + tags = ['validator','exclude_test_daily'], + full_refresh = false, + enabled = false ) }} WITH base AS ( diff --git a/models/silver/validator/silver__snapshot_validators_app_data.sql b/models/silver/validator/silver__snapshot_validators_app_data.sql index 09e300dc..38d9ea2f 100644 --- a/models/silver/validator/silver__snapshot_validators_app_data.sql +++ b/models/silver/validator/silver__snapshot_validators_app_data.sql @@ -3,13 +3,15 @@ unique_key = "CONCAT_WS('-', epoch_recorded, node_pubkey)", incremental_strategy = 'delete+insert', cluster_by = ['modified_timestamp::DATE'], - tags = ['validator'] + tags = ['validator'], + full_refresh = false ) }} {% set cutoff_date = "2024-11-04" %} WITH base AS ( - +-- historical data +{# SELECT json_data :account :: STRING AS node_pubkey, json_data :active_stake :: NUMBER AS active_stake, @@ -53,6 +55,7 @@ WITH base AS ( AND _inserted_timestamp > (SELECT max(_inserted_timestamp) FROM {{ this }}) {% endif %} UNION ALL +#} SELECT d.value:account::STRING AS node_pubkey, d.value:active_stake::NUMBER AS active_stake, diff --git a/models/silver/validator/silver__snapshot_vote_accounts.sql b/models/silver/validator/silver__snapshot_vote_accounts.sql index 38c58a63..ce06d933 100644 --- a/models/silver/validator/silver__snapshot_vote_accounts.sql +++ b/models/silver/validator/silver__snapshot_vote_accounts.sql @@ -3,12 +3,15 @@ unique_key = "CONCAT_WS('-', epoch_recorded, vote_pubkey)", incremental_strategy = 'delete+insert', cluster_by = ['modified_timestamp::DATE'], - tags = ['validator'] + tags = ['validator'], + full_refresh = false ) }} {% set cutoff_date = '2024-10-30' %} WITH base AS ( +-- historical data +{# SELECT _inserted_timestamp, json_data :account :data :parsed :info :authorizedVoters [0] :authorizedVoter :: STRING AS authorized_voter, @@ -42,6 +45,7 @@ WITH base AS ( ) {% endif %} UNION ALL +#} SELECT _inserted_timestamp, data :account :data :parsed :info :authorizedVoters [0] :authorizedVoter :: STRING AS authorized_voter, diff --git a/models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql b/models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql index ca0b4534..5c9f33d5 100644 --- a/models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql +++ b/models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql @@ -3,12 +3,15 @@ unique_key = "CONCAT_WS('-', epoch_recorded, vote_pubkey)", incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'], - tags = ['validator'] + tags = ['validator'], + full_refresh = false ) }} {% set cutoff_date = '2024-10-30' %} WITH base AS ( +-- historical data + {# SELECT json_data :status :: STRING AS epoch_status, json_data :data :activatedStake :: INT / pow( @@ -37,6 +40,7 @@ WITH base AS ( ) {% endif %} UNION ALL +#} SELECT status AS epoch_status, data :activatedStake :: INT / pow(