From 8a7e450bbaa8721c4c2fd5a6b38e8040908db836 Mon Sep 17 00:00:00 2001 From: tarikceric <46071768+tarikceric@users.noreply.github.com> Date: Tue, 9 May 2023 16:28:34 -0500 Subject: [PATCH] An 3162/daily validator update (#293) * rename folder * stake account udfs * add validator processes to daily run * add validator functions + tables * update selection to > _inserted_timestamp --- .github/workflows/dbt_run_daily.yml | 6 ++++++ macros/create_udfs.sql | 2 ++ .../run_sp_snapshot_get_stake_accounts.sql | 7 +++++++ .../sp_create_snapshot_get_stake_accounts.sql | 21 +++++++++++++++++++ .../udf_snapshot_get_stake_accounts.sql | 8 +++++++ ...un_sp_snapshot_get_validators_app_data.sql | 0 ...reate_snapshot_get_validators_app_data.sql | 0 .../udf_snapshot_get_validators_app_data.sql | 0 .../sp_refresh_external_tables_full.sql | 2 ++ .../silver__snapshot_stake_accounts.sql | 2 +- .../silver__snapshot_validators_app_data.sql | 2 +- .../silver__snapshot_vote_accounts.sql | 2 +- models/sources.yml | 3 ++- 13 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 macros/streamline/snapshot_get_stake_accounts/run_sp_snapshot_get_stake_accounts.sql create mode 100644 macros/streamline/snapshot_get_stake_accounts/sp_create_snapshot_get_stake_accounts.sql create mode 100644 macros/streamline/snapshot_get_stake_accounts/udf_snapshot_get_stake_accounts.sql rename macros/streamline/{snapshot_get_validator_app_data => snapshot_get_validators_app_data}/run_sp_snapshot_get_validators_app_data.sql (100%) rename macros/streamline/{snapshot_get_validator_app_data => snapshot_get_validators_app_data}/sp_create_snapshot_get_validators_app_data.sql (100%) rename macros/streamline/{snapshot_get_validator_app_data => snapshot_get_validators_app_data}/udf_snapshot_get_validators_app_data.sql (100%) diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index 90a45a79..4d0b11d5 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -42,3 +42,9 @@ jobs: run: | dbt run -s models/silver/silver__daily_signers.sql+ dbt run -s models/silver/nfts/silver__nft_mints.sql --full-refresh + dbt run-operation run_sp_snapshot_get_validators_app_data + dbt run-operation run_sp_snapshot_get_vote_accounts + dbt run-operation run_sp_snapshot_get_stake_accounts + dbt run -s models/bronze/bronze__validators_app_api.sql models/bronze/bronze__vote_accounts.sql models/bronze/bronze__stake_program_accounts.sql + dbt run -s models/silver/validator/silver__snapshot_stake_accounts.sql models/silver/validator/silver__snapshot_validators_app_data.sql models/silver/validator/silver__snapshot_vote_accounts.sql + diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 7e127e39..a31b565c 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -8,6 +8,8 @@ {{ udf_bulk_get_block_txs() }}; {{ udf_bulk_get_block_rewards() }}; {{ udf_snapshot_get_vote_accounts() }}; + {{ udf_snapshot_get_validators_app_data() }}; + {{ udf_snapshot_get_stake_accounts() }}; {% endif %} {{ create_udf_ordered_signers( diff --git a/macros/streamline/snapshot_get_stake_accounts/run_sp_snapshot_get_stake_accounts.sql b/macros/streamline/snapshot_get_stake_accounts/run_sp_snapshot_get_stake_accounts.sql new file mode 100644 index 00000000..7e82f655 --- /dev/null +++ b/macros/streamline/snapshot_get_stake_accounts/run_sp_snapshot_get_stake_accounts.sql @@ -0,0 +1,7 @@ +{% macro run_sp_snapshot_get_stake_accounts() %} +{% set sql %} +call silver.sp_snapshot_get_stake_accounts(); +{% endset %} + +{% do run_query(sql) %} +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/snapshot_get_stake_accounts/sp_create_snapshot_get_stake_accounts.sql b/macros/streamline/snapshot_get_stake_accounts/sp_create_snapshot_get_stake_accounts.sql new file mode 100644 index 00000000..403b5171 --- /dev/null +++ b/macros/streamline/snapshot_get_stake_accounts/sp_create_snapshot_get_stake_accounts.sql @@ -0,0 +1,21 @@ +{% macro sp_create_snapshot_get_stake_accounts() %} + {% if var("UPDATE_UDFS_AND_SPS") %} + {% set sql %} + CREATE OR REPLACE PROCEDURE silver.sp_snapshot_get_stake_accounts() + RETURNS variant + LANGUAGE SQL + AS + $$ + DECLARE + RESULT VARCHAR; + BEGIN + RESULT:= ( + SELECT + silver.udf_snapshot_get_stake_accounts() + ); + RETURN RESULT; + END; + $${% endset %} + {% do run_query(sql) %} + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/snapshot_get_stake_accounts/udf_snapshot_get_stake_accounts.sql b/macros/streamline/snapshot_get_stake_accounts/udf_snapshot_get_stake_accounts.sql new file mode 100644 index 00000000..63d22249 --- /dev/null +++ b/macros/streamline/snapshot_get_stake_accounts/udf_snapshot_get_stake_accounts.sql @@ -0,0 +1,8 @@ +{% macro udf_snapshot_get_stake_accounts() %} + CREATE + OR REPLACE EXTERNAL FUNCTION silver.udf_snapshot_get_stake_accounts() returns text api_integration = aws_solana_api_dev AS {% if target.database == 'SOLANA' -%} + 'https://pj4rqb8z96.execute-api.us-east-1.amazonaws.com/prod/bulk_snapshot_get_stake_program_accounts' + {% else %} + 'https://11zlwk4fm3.execute-api.us-east-1.amazonaws.com/dev/bulk_snapshot_get_stake_program_accounts' + {%- endif %} +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/snapshot_get_validator_app_data/run_sp_snapshot_get_validators_app_data.sql b/macros/streamline/snapshot_get_validators_app_data/run_sp_snapshot_get_validators_app_data.sql similarity index 100% rename from macros/streamline/snapshot_get_validator_app_data/run_sp_snapshot_get_validators_app_data.sql rename to macros/streamline/snapshot_get_validators_app_data/run_sp_snapshot_get_validators_app_data.sql diff --git a/macros/streamline/snapshot_get_validator_app_data/sp_create_snapshot_get_validators_app_data.sql b/macros/streamline/snapshot_get_validators_app_data/sp_create_snapshot_get_validators_app_data.sql similarity index 100% rename from macros/streamline/snapshot_get_validator_app_data/sp_create_snapshot_get_validators_app_data.sql rename to macros/streamline/snapshot_get_validators_app_data/sp_create_snapshot_get_validators_app_data.sql diff --git a/macros/streamline/snapshot_get_validator_app_data/udf_snapshot_get_validators_app_data.sql b/macros/streamline/snapshot_get_validators_app_data/udf_snapshot_get_validators_app_data.sql similarity index 100% rename from macros/streamline/snapshot_get_validator_app_data/udf_snapshot_get_validators_app_data.sql rename to macros/streamline/snapshot_get_validators_app_data/udf_snapshot_get_validators_app_data.sql diff --git a/macros/streamline/sp_refresh_external_tables_full.sql b/macros/streamline/sp_refresh_external_tables_full.sql index def8db0e..61603832 100644 --- a/macros/streamline/sp_refresh_external_tables_full.sql +++ b/macros/streamline/sp_refresh_external_tables_full.sql @@ -10,6 +10,8 @@ $$ alter external table streamline.{{ target.database }}.decoded_instructions_data_api refresh; alter external table streamline.{{ target.database }}.validator_metadata_api refresh; alter external table streamline.{{ target.database }}.validator_vote_accounts refresh; + alter external table streamline.{{ target.database }}.validators_app_list_api refresh; + alter external table streamline.{{ target.database }}.stake_program_accounts refresh; return TRUE; end; $${% endset %} diff --git a/models/silver/validator/silver__snapshot_stake_accounts.sql b/models/silver/validator/silver__snapshot_stake_accounts.sql index 75e4c359..1c99ea59 100644 --- a/models/silver/validator/silver__snapshot_stake_accounts.sql +++ b/models/silver/validator/silver__snapshot_stake_accounts.sql @@ -33,7 +33,7 @@ SELECT {{ ref('bronze__stake_program_accounts') }} {% if is_incremental() %} -WHERE _inserted_timestamp >= ( +WHERE _inserted_timestamp > ( SELECT MAX(_inserted_timestamp) FROM diff --git a/models/silver/validator/silver__snapshot_validators_app_data.sql b/models/silver/validator/silver__snapshot_validators_app_data.sql index 9d384b54..3a21ee71 100644 --- a/models/silver/validator/silver__snapshot_validators_app_data.sql +++ b/models/silver/validator/silver__snapshot_validators_app_data.sql @@ -45,7 +45,7 @@ SELECT {{ ref('bronze__validators_app_api') }} {% if is_incremental() %} -WHERE _inserted_timestamp >= ( +WHERE _inserted_timestamp > ( SELECT MAX(_inserted_timestamp) FROM diff --git a/models/silver/validator/silver__snapshot_vote_accounts.sql b/models/silver/validator/silver__snapshot_vote_accounts.sql index 5e395ec7..b09c6fcb 100644 --- a/models/silver/validator/silver__snapshot_vote_accounts.sql +++ b/models/silver/validator/silver__snapshot_vote_accounts.sql @@ -26,7 +26,7 @@ WITH base AS ( {{ ref('bronze__vote_accounts') }} {% if is_incremental() %} -WHERE _inserted_timestamp >= ( +WHERE _inserted_timestamp > ( SELECT MAX(_inserted_timestamp) FROM diff --git a/models/sources.yml b/models/sources.yml index f8b22c91..49265cc6 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -45,7 +45,8 @@ sources: - name: block_rewards_api - name: block_txs_api - name: validator_vote_accounts - - name: validator_app_list_api + - name: validators_app_list_api + - name: stake_program_accounts - name: bronze_api schema: bronze_api tables: