Merge branch 'main' of github.com:FlipsideCrypto/solana-models

.github/workflows/dbt_run_incremental.yml

@@ -41,7 +41,6 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run-operation run_sp_refresh_external_tables_full
           dbt run -s "solana_models,tag:scheduled_core"
 
   notify-failure:

.github/workflows/dbt_run_incremental_intermediate.yml

@@ -41,7 +41,6 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run-operation run_sp_refresh_external_tables_full
           dbt run -s "solana_models,tag:scheduled_core"
 
   notify-failure:

.github/workflows/dbt_run_incremental_non_core.yml

@@ -42,7 +42,6 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run-operation run_sp_refresh_external_tables_full
           dbt run -s "solana_models,tag:scheduled_core" "solana_models,tag:scheduled_non_core" 
 
   notify-failure:

models/gold/gov/gov__fact_block_production.sql

@@ -5,7 +5,8 @@
     cluster_by = ['epoch'],
     merge_exclude_columns = ["inserted_timestamp"],
     post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(node_pubkey)'),
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core'],
+    full_refresh = false
 ) }}
 
 {% if execute %}
@@ -53,6 +54,8 @@ FROM
 LEFT JOIN
   {{ ref('silver__epoch') }} AS e
   ON bp.epoch = e.epoch
+  -- historical data -- tables static and disabled, and manual change needed in rare case where fr is needed
+{# 
 {% if not is_incremental() %}
 UNION ALL
 SELECT
@@ -70,3 +73,4 @@ SELECT
 FROM
   {{ ref('silver__historical_block_production') }}
 {% endif %}
+#}

models/gold/gov/gov__fact_stake_accounts.sql

@@ -6,7 +6,8 @@
     cluster_by = ['epoch', 'activation_epoch', 'deactivation_epoch'],
     merge_exclude_columns = ["inserted_timestamp"],
     post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(stake_pubkey, vote_pubkey)'),
-    tags = ['scheduled_non_core_hourly']
+    tags = ['scheduled_non_core_hourly'],
+    full_refresh = false
   ) 
 }}
 
@@ -51,6 +52,8 @@ WHERE
     {% if is_incremental() %}
     AND modified_timestamp >= '{{ max_modified_timestamp }}'
     {% endif %}
+-- historical data -- tables static and disabled, and manual change needed in rare case where fr is needed
+{# 
 UNION ALL
 SELECT
     epoch_recorded::INT AS epoch,
@@ -104,3 +107,4 @@ SELECT
 FROM
     {{ ref('silver__historical_stake_account') }}
 {% endif %}
+#}

models/gold/gov/gov__fact_validators.sql

@@ -5,7 +5,8 @@
     cluster_by = ['epoch','epoch_active'],
     merge_exclude_columns = ["inserted_timestamp"],
     post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(node_pubkey, vote_pubkey)'),
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core'],
+    full_refresh = false
 ) }}
 
 {% if execute %}
@@ -61,6 +62,8 @@ FROM
 WHERE
     modified_timestamp >= '{{ max_modified_timestamp }}'
 {% endif %}
+-- historical data -- tables static and disabled, and manual change needed in rare case where fr is needed
+{# 
 {% if not is_incremental() %}
 UNION ALL
 SELECT
@@ -93,3 +96,4 @@ SELECT
 FROM
   {{ ref('silver__historical_validator_app_data') }}
 {% endif %}
+#}

models/gold/gov/gov__fact_vote_accounts.sql

@@ -5,7 +5,8 @@
     cluster_by = ['epoch','last_epoch_active'],
     merge_exclude_columns = ["inserted_timestamp"],
     post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(vote_pubkey, node_pubkey, owner)'),
-    tags = ['scheduled_non_core']
+    tags = ['scheduled_non_core'],
+    full_refresh = false
 ) }}
 
 {% if execute %}
@@ -56,6 +57,8 @@ FROM
 WHERE
     modified_timestamp >= '{{ max_modified_timestamp }}'
 {% endif %}
+-- historical data -- tables static and disabled, and manual change needed in rare case where fr is needed
+{# 
 {% if not is_incremental() %}
 UNION ALL
 SELECT
@@ -83,3 +86,4 @@ SELECT
 FROM
   {{ ref('silver__historical_vote_account') }}
 {% endif %}
+#}

models/silver/validator/silver__historical_block_production.sql

@@ -1,6 +1,8 @@
 {{ config(
-    materialized = 'view',
-    tags = ['validator_historical']
+    materialized = 'table',
+    tags = ['validator_historical'],
+    full_refresh = false,
+    enabled = false
 ) }}
 
 SELECT

models/silver/validator/silver__historical_stake_account.sql

@@ -1,6 +1,8 @@
 {{ config (
-    materialized = 'view',
-    tags = ['validator_historical']
+    materialized = 'table',
+    tags = ['validator_historical'],
+    full_refresh = false,
+    enabled = false
 ) }}
 
 SELECT

models/silver/validator/silver__historical_validator_app_data.sql

@@ -1,6 +1,8 @@
 {{ config (
-    materialized = 'view',
-    tags = ['validator_historical']
+    materialized = 'table',
+    tags = ['validator_historical'],
+    full_refresh = false,
+    enabled = false
 ) }}
 
 WITH base AS (

models/silver/validator/silver__historical_vote_account.sql

@@ -1,6 +1,8 @@
 {{ config (
-    materialized = 'view',
-    tags = ['validator_historical']
+    materialized = 'table',
+    tags = ['validator_historical'],
+    full_refresh = false,
+    enabled = false
 ) }}
 
 SELECT

models/silver/validator/silver__snapshot_stake_accounts.sql

@@ -3,7 +3,9 @@
     unique_key = "CONCAT_WS('-', epoch_recorded, stake_pubkey)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['modified_timestamp::DATE'],
-    tags = ['validator','exclude_test_daily']
+    tags = ['validator','exclude_test_daily'],
+    full_refresh = false,
+    enabled = false
 ) }}
 
 WITH base AS (

models/silver/validator/silver__snapshot_validators_app_data.sql

@@ -3,13 +3,15 @@
     unique_key = "CONCAT_WS('-', epoch_recorded, node_pubkey)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['modified_timestamp::DATE'],
-    tags = ['validator']
+    tags = ['validator'],
+    full_refresh = false
 ) }}
 
 {% set cutoff_date = "2024-11-04" %}
 
 WITH base AS (
-
+-- historical data
+{# 
     SELECT
         json_data :account :: STRING AS node_pubkey,
         json_data :active_stake :: NUMBER AS active_stake,
@@ -53,6 +55,7 @@ WITH base AS (
         AND _inserted_timestamp > (SELECT max(_inserted_timestamp) FROM {{ this }})
         {% endif %}
     UNION ALL
+#}
     SELECT
         d.value:account::STRING AS node_pubkey,
         d.value:active_stake::NUMBER AS active_stake,

models/silver/validator/silver__snapshot_vote_accounts.sql

@@ -3,12 +3,15 @@
     unique_key = "CONCAT_WS('-', epoch_recorded, vote_pubkey)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['modified_timestamp::DATE'],
-    tags = ['validator']
+    tags = ['validator'],
+    full_refresh = false
 ) }}
 
 {% set cutoff_date = '2024-10-30' %}
 
 WITH base AS (
+-- historical data
+{#
     SELECT
         _inserted_timestamp,
         json_data :account :data :parsed :info :authorizedVoters [0] :authorizedVoter :: STRING AS authorized_voter,
@@ -42,6 +45,7 @@ WITH base AS (
         )
         {% endif %}
     UNION ALL
+#}
     SELECT
         _inserted_timestamp,
         data :account :data :parsed :info :authorizedVoters [0] :authorizedVoter :: STRING AS authorized_voter,

models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql

@@ -3,12 +3,15 @@
     unique_key = "CONCAT_WS('-', epoch_recorded, vote_pubkey)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['_inserted_timestamp::DATE'],
-    tags = ['validator']
+    tags = ['validator'],
+    full_refresh = false
 ) }}
 
 {% set cutoff_date = '2024-10-30' %}
 
 WITH base AS (
+-- historical data
+ {#
     SELECT
         json_data :status :: STRING AS epoch_status,
         json_data :data :activatedStake :: INT / pow(
@@ -37,6 +40,7 @@ WITH base AS (
         )
         {% endif %}
     UNION ALL
+#}
     SELECT
         status AS epoch_status,
         data :activatedStake :: INT / pow(