diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index e024becb..1b3892d3 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -45,8 +45,7 @@ jobs: dbt run -s models/silver/nfts/silver__nft_mints.sql --full-refresh dbt run-operation run_sp_snapshot_get_validators_app_data dbt run-operation run_sp_snapshot_get_stake_accounts - dbt run-operation run_sp_snapshot_get_vote_program_accounts - dbt run -s models/bronze/bronze__validators_app_api.sql models/bronze/bronze__vote_accounts.sql models/bronze/bronze__stake_program_accounts.sql + dbt run -s models/bronze/bronze__validators_app_api.sql models/bronze/bronze__stake_program_accounts.sql dbt run -s models/silver/validator/silver__snapshot_validators_app_data.sql models/silver/validator/silver__snapshot_vote_accounts.sql models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql dbt run -s "solana_models,tag:nft_api" dbt run -s "solana_models,tag:daily" diff --git a/dbt_project.yml b/dbt_project.yml index 85c6fd11..3afed3c8 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -36,7 +36,6 @@ on-run-start: - '{{ sp_create_bulk_get_validator_metadata() }}' - '{{ sp_create_snapshot_get_stake_accounts() }}' - '{{ sp_create_snapshot_get_validators_app_data() }}' - - '{{ sp_create_snapshot_get_vote_program_accounts() }}' on-run-end: - '{{ apply_meta_as_tags(results) }}' diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 6871bd7d..f13c116a 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -8,7 +8,6 @@ {{ udf_bulk_get_block_txs() }}; {{ udf_snapshot_get_validators_app_data() }}; {{ udf_snapshot_get_stake_accounts() }}; - {{ udf_snapshot_get_vote_program_accounts() }}; {{ udf_bulk_program_parser() }}; {{ udf_decode_instructions() }}; {{ udf_bulk_parse_compressed_nft_mints() }}; diff --git a/macros/streamline/snapshot_get_vote_program_accounts/run_sp_snapshot_get_vote_program_accounts.sql b/macros/streamline/snapshot_get_vote_program_accounts/run_sp_snapshot_get_vote_program_accounts.sql deleted file mode 100644 index a3839ddb..00000000 --- a/macros/streamline/snapshot_get_vote_program_accounts/run_sp_snapshot_get_vote_program_accounts.sql +++ /dev/null @@ -1,7 +0,0 @@ -{% macro run_sp_snapshot_get_vote_program_accounts() %} -{% set sql %} -call silver.sp_snapshot_get_vote_program_accounts(); -{% endset %} - -{% do run_query(sql) %} -{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/snapshot_get_vote_program_accounts/sp_create_snapshot_get_vote_program_accounts.sql b/macros/streamline/snapshot_get_vote_program_accounts/sp_create_snapshot_get_vote_program_accounts.sql deleted file mode 100644 index 5f151732..00000000 --- a/macros/streamline/snapshot_get_vote_program_accounts/sp_create_snapshot_get_vote_program_accounts.sql +++ /dev/null @@ -1,21 +0,0 @@ -{% macro sp_create_snapshot_get_vote_program_accounts() %} - {% if var("UPDATE_UDFS_AND_SPS") %} - {% set sql %} - CREATE OR REPLACE PROCEDURE silver.sp_snapshot_get_vote_program_accounts() - RETURNS variant - LANGUAGE SQL - AS - $$ - DECLARE - RESULT VARCHAR; - BEGIN - RESULT:= ( - SELECT - silver.udf_snapshot_get_vote_program_accounts() - ); - RETURN RESULT; - END; - $${% endset %} - {% do run_query(sql) %} - {% endif %} -{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/snapshot_get_vote_program_accounts/udf_snapshot_get_vote_program_accounts.sql b/macros/streamline/snapshot_get_vote_program_accounts/udf_snapshot_get_vote_program_accounts.sql deleted file mode 100644 index 22a5e7d8..00000000 --- a/macros/streamline/snapshot_get_vote_program_accounts/udf_snapshot_get_vote_program_accounts.sql +++ /dev/null @@ -1,8 +0,0 @@ -{% macro udf_snapshot_get_vote_program_accounts() %} - CREATE - OR REPLACE EXTERNAL FUNCTION silver.udf_snapshot_get_vote_program_accounts() returns text api_integration = aws_solana_api_dev AS {% if target.database == 'SOLANA' -%} - 'https://pj4rqb8z96.execute-api.us-east-1.amazonaws.com/prod/snapshot_get_vote_program_accounts' - {% else %} - 'https://11zlwk4fm3.execute-api.us-east-1.amazonaws.com/dev/snapshot_get_vote_program_accounts' - {%- endif %} -{% endmacro %} \ No newline at end of file diff --git a/models/silver/validator/silver__snapshot_vote_accounts.sql b/models/silver/validator/silver__snapshot_vote_accounts.sql index c3783599..ccd54dc1 100644 --- a/models/silver/validator/silver__snapshot_vote_accounts.sql +++ b/models/silver/validator/silver__snapshot_vote_accounts.sql @@ -6,8 +6,9 @@ tags = ['validator'] ) }} -WITH base AS ( +{% set cutoff_date = '2024-10-30' %} +WITH base AS ( SELECT _inserted_timestamp, json_data :account :data :parsed :info :authorizedVoters [0] :authorizedVoter :: STRING AS authorized_voter, @@ -30,16 +31,49 @@ WITH base AS ( json_data :pubkey :: STRING AS vote_pubkey FROM {{ ref('bronze__vote_accounts') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp > ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} + WHERE + _inserted_timestamp::DATE <= '{{ cutoff_date }}' + {% if is_incremental() %} + AND _inserted_timestamp > ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) + {% endif %} + UNION ALL + SELECT + _inserted_timestamp, + data :account :data :parsed :info :authorizedVoters [0] :authorizedVoter :: STRING AS authorized_voter, + data :account :data :parsed :info :authorizedVoters [0] :epoch :: NUMBER AS last_epoch_active, + data :account :data :parsed :info :authorizedWithdrawer :: STRING AS authorized_withdrawer, + data :account :data :parsed :info :commission :: NUMBER AS commission, + data :account :data :parsed :info :epochCredits :: ARRAY AS epoch_credits, + data :account :data :parsed :info :lastTimestamp :slot :: NUMBER AS last_timestamp_slot, + data :account :data :parsed :info :lastTimestamp :timestamp :: timestamp_tz AS last_timestamp, + data :account :data :parsed :info :nodePubkey :: STRING AS node_pubkey, + data :account :data :parsed :info :priorVoters :: ARRAY AS prior_voters, + data :account :data :parsed :info :rootSlot :: NUMBER AS root_slot, + data :account :data :parsed :info :votes :: ARRAY AS votes, + data :account :lamports / pow( + 10, + 9 + ) AS account_sol, + data :account :owner :: STRING AS owner, + data :account :rentEpoch :: NUMBER AS rent_epoch, + data :pubkey :: STRING AS vote_pubkey + FROM + {{ ref('bronze__streamline_validator_vote_program_accounts_2')}} + WHERE + _inserted_timestamp::DATE > '{{ cutoff_date }}' + {% if is_incremental() %} + AND _inserted_timestamp > ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) + {% endif %} ), vote_accounts_epoch_recorded AS ( SELECT