Deprecate legacy streamline validator vote accounts (#687)

* update model with 2.0 data

* remove legacy pipeline assets

* remove sp create call

* remove bronze refresh

.github/workflows/dbt_run_daily.yml

@@ -44,10 +44,9 @@ jobs:
           dbt run -s models/silver/silver__daily_signers.sql+
           dbt run -s models/silver/nfts/silver__nft_mints.sql --full-refresh
           dbt run-operation run_sp_snapshot_get_validators_app_data
-          dbt run-operation run_sp_snapshot_get_vote_accounts
           dbt run-operation run_sp_snapshot_get_stake_accounts
           dbt run-operation run_sp_snapshot_get_vote_program_accounts
-          dbt run -s models/bronze/bronze__validators_app_api.sql models/bronze/bronze__vote_accounts.sql models/bronze/bronze__stake_program_accounts.sql models/bronze/bronze__vote_accounts_extended_stats.sql
+          dbt run -s models/bronze/bronze__validators_app_api.sql models/bronze/bronze__vote_accounts.sql models/bronze/bronze__stake_program_accounts.sql
           dbt run -s models/silver/validator/silver__snapshot_validators_app_data.sql models/silver/validator/silver__snapshot_vote_accounts.sql models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql
           dbt run -s "solana_models,tag:nft_api"
           dbt run -s "solana_models,tag:daily"

dbt_project.yml

@@ -36,7 +36,6 @@ on-run-start:
   - '{{ sp_create_bulk_get_validator_metadata() }}'
   - '{{ sp_create_snapshot_get_stake_accounts() }}'
   - '{{ sp_create_snapshot_get_validators_app_data() }}'
-  - '{{ sp_create_snapshot_get_vote_accounts() }}'
   - '{{ sp_create_snapshot_get_vote_program_accounts() }}'
 
 on-run-end:

macros/create_udfs.sql

@@ -6,7 +6,6 @@
             {{ udf_bulk_get_validator_metadata() }};
             {{ udf_bulk_get_blocks() }};
             {{ udf_bulk_get_block_txs() }};
-            {{ udf_snapshot_get_vote_accounts() }};
             {{ udf_snapshot_get_validators_app_data() }};
             {{ udf_snapshot_get_stake_accounts() }};
             {{ udf_snapshot_get_vote_program_accounts() }};

macros/streamline/snapshot_get_vote_accounts/run_sp_snapshot_get_vote_accounts.sql

@@ -1,7 +0,0 @@
-{% macro run_sp_snapshot_get_vote_accounts() %}
-{% set sql %}
-call silver.sp_snapshot_get_vote_accounts();
-{% endset %}
-
-{% do run_query(sql) %}
-{% endmacro %}

macros/streamline/snapshot_get_vote_accounts/sp_create_snapshot_get_vote_accounts.sql

@@ -1,21 +0,0 @@
-{% macro sp_create_snapshot_get_vote_accounts() %}
-  {% if var("UPDATE_UDFS_AND_SPS") %}
-    {% set sql %}
-    CREATE OR REPLACE PROCEDURE silver.sp_snapshot_get_vote_accounts() 
-    RETURNS variant 
-    LANGUAGE SQL 
-    AS 
-    $$
-      DECLARE
-        RESULT VARCHAR;
-      BEGIN
-        RESULT:= (
-            SELECT
-              silver.udf_snapshot_get_vote_accounts()
-          );
-        RETURN RESULT;
-      END;
-    $${% endset %}
-    {% do run_query(sql) %}
-  {% endif %}
-{% endmacro %}

macros/streamline/snapshot_get_vote_accounts/udf_snapshot_get_vote_accounts.sql

@@ -1,8 +0,0 @@
-{% macro udf_snapshot_get_vote_accounts() %}
-    CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_snapshot_get_vote_accounts() returns text api_integration = aws_solana_api_dev AS {% if target.database == 'SOLANA' -%}
-        'https://pj4rqb8z96.execute-api.us-east-1.amazonaws.com/prod/snapshot_get_vote_accounts'
-    {% else %}
-        'https://11zlwk4fm3.execute-api.us-east-1.amazonaws.com/dev/snapshot_get_vote_accounts'
-    {%- endif %}
-{% endmacro %}

models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql

@@ -6,8 +6,9 @@
     tags = ['validator']
 ) }}
 
-WITH base AS (
+{% set cutoff_date = '2024-10-30' %}
 
+WITH base AS (
     SELECT
         json_data :status :: STRING AS epoch_status,
         json_data :data :activatedStake :: INT / pow(
@@ -25,16 +26,44 @@ WITH base AS (
         _inserted_timestamp
     FROM
         {{ ref('bronze__vote_accounts_extended_stats') }}
-
-{% if is_incremental() %}
-WHERE
-    _inserted_timestamp > (
-        SELECT
-            MAX(_inserted_timestamp)
-        FROM
-            {{ this }}
-    )
-{% endif %}
+    WHERE
+        _inserted_timestamp::DATE <= '{{ cutoff_date }}'
+        {% if is_incremental() %}
+        AND _inserted_timestamp > (
+            SELECT
+                MAX(_inserted_timestamp)
+            FROM
+                {{ this }}
+        )
+        {% endif %}
+    UNION ALL
+    SELECT
+        status AS epoch_status,
+        data :activatedStake :: INT / pow(
+            10,
+            9
+        ) AS activatedStake,
+        data :commission :: NUMBER AS commission,
+        data :epochCredits [4] [0] :: NUMBER AS latest_epoch,
+        data :epochCredits AS epochCredits,
+        data :epochVoteAccount :: BOOLEAN AS epochVoteAccount,
+        data :lastVote :: NUMBER AS lastVote,
+        data :nodePubkey :: STRING AS nodePubkey,
+        data :rootSlot :: NUMBER AS rootSlot,
+        data :votePubkey :: STRING AS votePubkey,
+        _inserted_timestamp
+    FROM
+        {{ ref('bronze__streamline_validator_vote_accounts_2')}}
+    WHERE
+        _inserted_timestamp::DATE > '{{ cutoff_date }}'
+        {% if is_incremental() %}
+        AND _inserted_timestamp > (
+            SELECT
+                MAX(_inserted_timestamp)
+            FROM
+                {{ this }}
+        )
+        {% endif %}
 ),
 votes_accounts_epoch_recorded AS (
     SELECT