Deprecate legacy streamline validator vote program accounts (#688)

* use streamline 2.0 data after cutoff

* remove legacy streamline assets

.github/workflows/dbt_run_daily.yml

@@ -45,8 +45,7 @@ jobs:
           dbt run -s models/silver/nfts/silver__nft_mints.sql --full-refresh
           dbt run-operation run_sp_snapshot_get_validators_app_data
           dbt run-operation run_sp_snapshot_get_stake_accounts
-          dbt run-operation run_sp_snapshot_get_vote_program_accounts
-          dbt run -s models/bronze/bronze__validators_app_api.sql models/bronze/bronze__vote_accounts.sql models/bronze/bronze__stake_program_accounts.sql
+          dbt run -s models/bronze/bronze__validators_app_api.sql models/bronze/bronze__stake_program_accounts.sql
           dbt run -s models/silver/validator/silver__snapshot_validators_app_data.sql models/silver/validator/silver__snapshot_vote_accounts.sql models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql
           dbt run -s "solana_models,tag:nft_api"
           dbt run -s "solana_models,tag:daily"

dbt_project.yml

@@ -36,7 +36,6 @@ on-run-start:
   - '{{ sp_create_bulk_get_validator_metadata() }}'
   - '{{ sp_create_snapshot_get_stake_accounts() }}'
   - '{{ sp_create_snapshot_get_validators_app_data() }}'
-  - '{{ sp_create_snapshot_get_vote_program_accounts() }}'
 
 on-run-end:
   - '{{ apply_meta_as_tags(results) }}'

macros/create_udfs.sql

@@ -8,7 +8,6 @@
             {{ udf_bulk_get_block_txs() }};
             {{ udf_snapshot_get_validators_app_data() }};
             {{ udf_snapshot_get_stake_accounts() }};
-            {{ udf_snapshot_get_vote_program_accounts() }};
             {{ udf_bulk_program_parser() }};
             {{ udf_decode_instructions() }};
             {{ udf_bulk_parse_compressed_nft_mints() }};

macros/streamline/snapshot_get_vote_program_accounts/run_sp_snapshot_get_vote_program_accounts.sql

@@ -1,7 +0,0 @@
-{% macro run_sp_snapshot_get_vote_program_accounts() %}
-{% set sql %}
-call silver.sp_snapshot_get_vote_program_accounts();
-{% endset %}
-
-{% do run_query(sql) %}
-{% endmacro %}

macros/streamline/snapshot_get_vote_program_accounts/sp_create_snapshot_get_vote_program_accounts.sql

@@ -1,21 +0,0 @@
-{% macro sp_create_snapshot_get_vote_program_accounts() %}
-  {% if var("UPDATE_UDFS_AND_SPS") %}
-    {% set sql %}
-    CREATE OR REPLACE PROCEDURE silver.sp_snapshot_get_vote_program_accounts() 
-    RETURNS variant 
-    LANGUAGE SQL 
-    AS 
-    $$
-      DECLARE
-        RESULT VARCHAR;
-      BEGIN
-        RESULT:= (
-            SELECT
-              silver.udf_snapshot_get_vote_program_accounts()
-          );
-        RETURN RESULT;
-      END;
-    $${% endset %}
-    {% do run_query(sql) %}
-  {% endif %}
-{% endmacro %}

macros/streamline/snapshot_get_vote_program_accounts/udf_snapshot_get_vote_program_accounts.sql

@@ -1,8 +0,0 @@
-{% macro udf_snapshot_get_vote_program_accounts() %}
-    CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_snapshot_get_vote_program_accounts() returns text api_integration = aws_solana_api_dev AS {% if target.database == 'SOLANA' -%}
-        'https://pj4rqb8z96.execute-api.us-east-1.amazonaws.com/prod/snapshot_get_vote_program_accounts'
-    {% else %}
-        'https://11zlwk4fm3.execute-api.us-east-1.amazonaws.com/dev/snapshot_get_vote_program_accounts'
-    {%- endif %}
-{% endmacro %}

models/silver/validator/silver__snapshot_vote_accounts.sql

@@ -6,8 +6,9 @@
     tags = ['validator']
 ) }}
 
-WITH base AS (
+{% set cutoff_date = '2024-10-30' %}
 
+WITH base AS (
     SELECT
         _inserted_timestamp,
         json_data :account :data :parsed :info :authorizedVoters [0] :authorizedVoter :: STRING AS authorized_voter,
@@ -30,16 +31,49 @@ WITH base AS (
         json_data :pubkey :: STRING AS vote_pubkey
     FROM
         {{ ref('bronze__vote_accounts') }}
-
-{% if is_incremental() %}
-WHERE
-    _inserted_timestamp > (
-        SELECT
-            MAX(_inserted_timestamp)
-        FROM
-            {{ this }}
-    )
-{% endif %}
+    WHERE
+        _inserted_timestamp::DATE <= '{{ cutoff_date }}'
+        {% if is_incremental() %}
+        AND _inserted_timestamp > (
+            SELECT
+                MAX(_inserted_timestamp)
+            FROM
+                {{ this }}
+        )
+        {% endif %}
+    UNION ALL
+    SELECT
+        _inserted_timestamp,
+        data :account :data :parsed :info :authorizedVoters [0] :authorizedVoter :: STRING AS authorized_voter,
+        data :account :data :parsed :info :authorizedVoters [0] :epoch :: NUMBER AS last_epoch_active,
+        data :account :data :parsed :info :authorizedWithdrawer :: STRING AS authorized_withdrawer,
+        data :account :data :parsed :info :commission :: NUMBER AS commission,
+        data :account :data :parsed :info :epochCredits :: ARRAY AS epoch_credits,
+        data :account :data :parsed :info :lastTimestamp :slot :: NUMBER AS last_timestamp_slot,
+        data :account :data :parsed :info :lastTimestamp :timestamp :: timestamp_tz AS last_timestamp,
+        data :account :data :parsed :info :nodePubkey :: STRING AS node_pubkey,
+        data :account :data :parsed :info :priorVoters :: ARRAY AS prior_voters,
+        data :account :data :parsed :info :rootSlot :: NUMBER AS root_slot,
+        data :account :data :parsed :info :votes :: ARRAY AS votes,
+        data :account :lamports / pow(
+            10,
+            9
+        ) AS account_sol,
+        data :account :owner :: STRING AS owner,
+        data :account :rentEpoch :: NUMBER AS rent_epoch,
+        data :pubkey :: STRING AS vote_pubkey
+    FROM
+        {{ ref('bronze__streamline_validator_vote_program_accounts_2')}}
+    WHERE
+        _inserted_timestamp::DATE > '{{ cutoff_date }}'
+        {% if is_incremental() %}
+        AND _inserted_timestamp > (
+            SELECT
+                MAX(_inserted_timestamp)
+            FROM
+                {{ this }}
+        )
+        {% endif %}
 ),
 vote_accounts_epoch_recorded AS (
     SELECT