diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index c45031c0..e024becb 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -44,10 +44,9 @@ jobs: dbt run -s models/silver/silver__daily_signers.sql+ dbt run -s models/silver/nfts/silver__nft_mints.sql --full-refresh dbt run-operation run_sp_snapshot_get_validators_app_data - dbt run-operation run_sp_snapshot_get_vote_accounts dbt run-operation run_sp_snapshot_get_stake_accounts dbt run-operation run_sp_snapshot_get_vote_program_accounts - dbt run -s models/bronze/bronze__validators_app_api.sql models/bronze/bronze__vote_accounts.sql models/bronze/bronze__stake_program_accounts.sql models/bronze/bronze__vote_accounts_extended_stats.sql + dbt run -s models/bronze/bronze__validators_app_api.sql models/bronze/bronze__vote_accounts.sql models/bronze/bronze__stake_program_accounts.sql dbt run -s models/silver/validator/silver__snapshot_validators_app_data.sql models/silver/validator/silver__snapshot_vote_accounts.sql models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql dbt run -s "solana_models,tag:nft_api" dbt run -s "solana_models,tag:daily" diff --git a/dbt_project.yml b/dbt_project.yml index a0651aaa..85c6fd11 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -36,7 +36,6 @@ on-run-start: - '{{ sp_create_bulk_get_validator_metadata() }}' - '{{ sp_create_snapshot_get_stake_accounts() }}' - '{{ sp_create_snapshot_get_validators_app_data() }}' - - '{{ sp_create_snapshot_get_vote_accounts() }}' - '{{ sp_create_snapshot_get_vote_program_accounts() }}' on-run-end: diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 28f8d2b4..6871bd7d 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -6,7 +6,6 @@ {{ udf_bulk_get_validator_metadata() }}; {{ udf_bulk_get_blocks() }}; {{ udf_bulk_get_block_txs() }}; - {{ udf_snapshot_get_vote_accounts() }}; {{ udf_snapshot_get_validators_app_data() }}; {{ udf_snapshot_get_stake_accounts() }}; {{ udf_snapshot_get_vote_program_accounts() }}; diff --git a/macros/streamline/snapshot_get_vote_accounts/run_sp_snapshot_get_vote_accounts.sql b/macros/streamline/snapshot_get_vote_accounts/run_sp_snapshot_get_vote_accounts.sql deleted file mode 100644 index 30101fed..00000000 --- a/macros/streamline/snapshot_get_vote_accounts/run_sp_snapshot_get_vote_accounts.sql +++ /dev/null @@ -1,7 +0,0 @@ -{% macro run_sp_snapshot_get_vote_accounts() %} -{% set sql %} -call silver.sp_snapshot_get_vote_accounts(); -{% endset %} - -{% do run_query(sql) %} -{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/snapshot_get_vote_accounts/sp_create_snapshot_get_vote_accounts.sql b/macros/streamline/snapshot_get_vote_accounts/sp_create_snapshot_get_vote_accounts.sql deleted file mode 100644 index c026e110..00000000 --- a/macros/streamline/snapshot_get_vote_accounts/sp_create_snapshot_get_vote_accounts.sql +++ /dev/null @@ -1,21 +0,0 @@ -{% macro sp_create_snapshot_get_vote_accounts() %} - {% if var("UPDATE_UDFS_AND_SPS") %} - {% set sql %} - CREATE OR REPLACE PROCEDURE silver.sp_snapshot_get_vote_accounts() - RETURNS variant - LANGUAGE SQL - AS - $$ - DECLARE - RESULT VARCHAR; - BEGIN - RESULT:= ( - SELECT - silver.udf_snapshot_get_vote_accounts() - ); - RETURN RESULT; - END; - $${% endset %} - {% do run_query(sql) %} - {% endif %} -{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/snapshot_get_vote_accounts/udf_snapshot_get_vote_accounts.sql b/macros/streamline/snapshot_get_vote_accounts/udf_snapshot_get_vote_accounts.sql deleted file mode 100644 index 059f4881..00000000 --- a/macros/streamline/snapshot_get_vote_accounts/udf_snapshot_get_vote_accounts.sql +++ /dev/null @@ -1,8 +0,0 @@ -{% macro udf_snapshot_get_vote_accounts() %} - CREATE - OR REPLACE EXTERNAL FUNCTION silver.udf_snapshot_get_vote_accounts() returns text api_integration = aws_solana_api_dev AS {% if target.database == 'SOLANA' -%} - 'https://pj4rqb8z96.execute-api.us-east-1.amazonaws.com/prod/snapshot_get_vote_accounts' - {% else %} - 'https://11zlwk4fm3.execute-api.us-east-1.amazonaws.com/dev/snapshot_get_vote_accounts' - {%- endif %} -{% endmacro %} \ No newline at end of file diff --git a/models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql b/models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql index 2da0160d..ca0b4534 100644 --- a/models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql +++ b/models/silver/validator/silver__snapshot_vote_accounts_extended_stats.sql @@ -6,8 +6,9 @@ tags = ['validator'] ) }} -WITH base AS ( +{% set cutoff_date = '2024-10-30' %} +WITH base AS ( SELECT json_data :status :: STRING AS epoch_status, json_data :data :activatedStake :: INT / pow( @@ -25,16 +26,44 @@ WITH base AS ( _inserted_timestamp FROM {{ ref('bronze__vote_accounts_extended_stats') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp > ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} + WHERE + _inserted_timestamp::DATE <= '{{ cutoff_date }}' + {% if is_incremental() %} + AND _inserted_timestamp > ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) + {% endif %} + UNION ALL + SELECT + status AS epoch_status, + data :activatedStake :: INT / pow( + 10, + 9 + ) AS activatedStake, + data :commission :: NUMBER AS commission, + data :epochCredits [4] [0] :: NUMBER AS latest_epoch, + data :epochCredits AS epochCredits, + data :epochVoteAccount :: BOOLEAN AS epochVoteAccount, + data :lastVote :: NUMBER AS lastVote, + data :nodePubkey :: STRING AS nodePubkey, + data :rootSlot :: NUMBER AS rootSlot, + data :votePubkey :: STRING AS votePubkey, + _inserted_timestamp + FROM + {{ ref('bronze__streamline_validator_vote_accounts_2')}} + WHERE + _inserted_timestamp::DATE > '{{ cutoff_date }}' + {% if is_incremental() %} + AND _inserted_timestamp > ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) + {% endif %} ), votes_accounts_epoch_recorded AS ( SELECT