From 8aacff420f66a0cf83bb3a69c159be98831b85ca Mon Sep 17 00:00:00 2001 From: desmond-hui <97470747+desmond-hui@users.noreply.github.com> Date: Fri, 27 Sep 2024 06:46:29 -0700 Subject: [PATCH] deprecate silver__snapshot_stake_accounts (#659) --- .github/workflows/dbt_run_daily.yml | 2 +- models/gold/gov/gov__fact_stake_accounts.sql | 149 +++++++++++-------- models/gold/gov/gov__fact_stake_accounts.yml | 2 +- 3 files changed, 87 insertions(+), 66 deletions(-) diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index d664e02b..c45031c0 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -48,7 +48,7 @@ jobs: dbt run-operation run_sp_snapshot_get_stake_accounts dbt run-operation run_sp_snapshot_get_vote_program_accounts dbt run -s models/bronze/bronze__validators_app_api.sql models/bronze/bronze__vote_accounts.sql models/bronze/bronze__stake_program_accounts.sql models/bronze/bronze__vote_accounts_extended_stats.sql - dbt run -s models/silver/validator/silver__snapshot_stake_accounts.sql models/silver/validator/silver__snapshot_validators_app_data.sql models/silver/validator/silver__snapshot_vote_accounts.sql models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql + dbt run -s models/silver/validator/silver__snapshot_validators_app_data.sql models/silver/validator/silver__snapshot_vote_accounts.sql models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql dbt run -s "solana_models,tag:nft_api" dbt run -s "solana_models,tag:daily" diff --git a/models/gold/gov/gov__fact_stake_accounts.sql b/models/gold/gov/gov__fact_stake_accounts.sql index eb04bfd0..494394c2 100644 --- a/models/gold/gov/gov__fact_stake_accounts.sql +++ b/models/gold/gov/gov__fact_stake_accounts.sql @@ -1,85 +1,106 @@ -{{ config( +{{ + config( materialized = 'incremental', - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'VALIDATOR' }}}, + meta = { 'database_tags': { 'table': { 'PURPOSE': 'VALIDATOR' }}}, unique_key = ['fact_stake_accounts_id'], - cluster_by = ['epoch','activation_epoch','deactivation_epoch'], + cluster_by = ['epoch', 'activation_epoch', 'deactivation_epoch'], merge_exclude_columns = ["inserted_timestamp"], - post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(stake_pubkey, vote_pubkey)'), + post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(stake_pubkey, vote_pubkey)'), tags = ['scheduled_non_core'] -) }} + ) +}} + +{% set v1_cutoff_epoch = 669 %} {% if execute %} {% if is_incremental() %} {% set query %} - SELECT MAX(modified_timestamp) AS max_modified_timestamp - FROM {{ this }} + SELECT + max(modified_timestamp) AS max_modified_timestamp + FROM + {{ this }} {% endset %} - {% set max_modified_timestamp = run_query(query).columns[0].values()[0] %} {% endif %} {% endif %} SELECT - epoch_recorded :: INT AS epoch, - stake_pubkey, - vote_pubkey, - authorized_staker, - authorized_withdrawer, - lockup, - rent_exempt_reserve, - credits_observed, - activation_epoch, - deactivation_epoch, - active_stake, - warmup_cooldown_rate, - type_stake, - program, - account_sol, - rent_epoch, - COALESCE ( - snapshot_stake_accounts_id, - {{ dbt_utils.generate_surrogate_key( - ['epoch', 'stake_pubkey'] - ) }} - ) AS fact_stake_accounts_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp + epoch_recorded::INT AS epoch, + stake_pubkey, + vote_pubkey, + authorized_staker, + authorized_withdrawer, + lockup, + rent_exempt_reserve, + credits_observed, + activation_epoch, + deactivation_epoch, + active_stake, + warmup_cooldown_rate, + type_stake, + program, + account_sol, + rent_epoch, + snapshot_stake_accounts_2_id AS fact_stake_accounts_id, + inserted_timestamp, + modified_timestamp FROM - {{ ref('silver__snapshot_stake_accounts') }} -{% if is_incremental() %} + {{ ref('silver__snapshot_stake_accounts_2') }} WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% endif %} + epoch > {{ v1_cutoff_epoch }} + {% if is_incremental() %} + AND modified_timestamp >= '{{ max_modified_timestamp }}' + {% endif %} +UNION ALL +SELECT + epoch_recorded::INT AS epoch, + stake_pubkey, + vote_pubkey, + authorized_staker, + authorized_withdrawer, + lockup, + rent_exempt_reserve, + credits_observed, + activation_epoch, + deactivation_epoch, + active_stake, + warmup_cooldown_rate, + type_stake, + program, + account_sol, + rent_epoch, + coalesce(snapshot_stake_accounts_id, {{ dbt_utils.generate_surrogate_key(['epoch', 'stake_pubkey']) }}) AS fact_stake_accounts_id, + coalesce(inserted_timestamp, '2000-01-01') AS inserted_timestamp, + coalesce(modified_timestamp, '2000-01-01') AS modified_timestamp +FROM + {{ ref('silver__snapshot_stake_accounts') }} +WHERE + epoch <= {{ v1_cutoff_epoch }} + {% if is_incremental() %} + AND modified_timestamp >= '{{ max_modified_timestamp }}' + {% endif %} {% if not is_incremental() %} UNION ALL SELECT - epoch_ingested_at :: INT AS epoch, - stake_pubkey, - vote_pubkey, - authorized_staker, - authorized_withdrawer, - lockup, - rent_exempt_reserve, - credits_observed, - activation_epoch, - deactivation_epoch, - active_stake, - warmup_cooldown_rate, - type_stake, - program, - account_sol, - rent_epoch, - {{ dbt_utils.generate_surrogate_key( - ['epoch', 'stake_pubkey'] - ) }} AS fact_stake_accounts_id, - '2000-01-01' as inserted_timestamp, - '2000-01-01' AS modified_timestamp + epoch_ingested_at::INT AS epoch, + stake_pubkey, + vote_pubkey, + authorized_staker, + authorized_withdrawer, + lockup, + rent_exempt_reserve, + credits_observed, + activation_epoch, + deactivation_epoch, + active_stake, + warmup_cooldown_rate, + type_stake, + program, + account_sol, + rent_epoch, + {{ dbt_utils.generate_surrogate_key(['epoch', 'stake_pubkey']) }} AS fact_stake_accounts_id, + '2000-01-01' AS inserted_timestamp, + '2000-01-01' AS modified_timestamp FROM - {{ ref('silver__historical_stake_account') }} + {{ ref('silver__historical_stake_account') }} {% endif %} diff --git a/models/gold/gov/gov__fact_stake_accounts.yml b/models/gold/gov/gov__fact_stake_accounts.yml index 69d9b330..642f045c 100644 --- a/models/gold/gov/gov__fact_stake_accounts.yml +++ b/models/gold/gov/gov__fact_stake_accounts.yml @@ -8,7 +8,7 @@ models: tests: - reference_tx_missing: reference_tables: - - 'silver__snapshot_stake_accounts' + - 'silver__snapshot_stake_accounts_2' id_column: 'stake_pubkey' columns: - name: epoch