An 3162/daily validator update (#293)

* rename folder

* stake account udfs

* add validator processes to daily run

* add validator functions + tables

* update selection to > _inserted_timestamp

.github/workflows/dbt_run_daily.yml

@@ -42,3 +42,9 @@ jobs:
         run: |
           dbt run -s models/silver/silver__daily_signers.sql+
           dbt run -s models/silver/nfts/silver__nft_mints.sql --full-refresh
+          dbt run-operation run_sp_snapshot_get_validators_app_data
+          dbt run-operation run_sp_snapshot_get_vote_accounts
+          dbt run-operation run_sp_snapshot_get_stake_accounts 
+          dbt run -s models/bronze/bronze__validators_app_api.sql models/bronze/bronze__vote_accounts.sql models/bronze/bronze__stake_program_accounts.sql 
+          dbt run -s models/silver/validator/silver__snapshot_stake_accounts.sql models/silver/validator/silver__snapshot_validators_app_data.sql models/silver/validator/silver__snapshot_vote_accounts.sql
+

macros/create_udfs.sql

@@ -8,6 +8,8 @@
             {{ udf_bulk_get_block_txs() }};
             {{ udf_bulk_get_block_rewards() }};
             {{ udf_snapshot_get_vote_accounts() }};
+            {{ udf_snapshot_get_validators_app_data() }};
+            {{ udf_snapshot_get_stake_accounts() }};
         {% endif %}
 
         {{ create_udf_ordered_signers(

macros/streamline/snapshot_get_stake_accounts/run_sp_snapshot_get_stake_accounts.sql

@@ -0,0 +1,7 @@
+{% macro run_sp_snapshot_get_stake_accounts() %}
+{% set sql %}
+call silver.sp_snapshot_get_stake_accounts();
+{% endset %}
+
+{% do run_query(sql) %}
+{% endmacro %}

macros/streamline/snapshot_get_stake_accounts/sp_create_snapshot_get_stake_accounts.sql

@@ -0,0 +1,21 @@
+{% macro sp_create_snapshot_get_stake_accounts() %}
+  {% if var("UPDATE_UDFS_AND_SPS") %}
+    {% set sql %}
+    CREATE OR REPLACE PROCEDURE silver.sp_snapshot_get_stake_accounts() 
+    RETURNS variant 
+    LANGUAGE SQL 
+    AS 
+    $$
+      DECLARE
+        RESULT VARCHAR;
+      BEGIN
+        RESULT:= (
+            SELECT
+              silver.udf_snapshot_get_stake_accounts()
+          );
+        RETURN RESULT;
+      END;
+    $${% endset %}
+    {% do run_query(sql) %}
+  {% endif %}
+{% endmacro %}

macros/streamline/snapshot_get_stake_accounts/udf_snapshot_get_stake_accounts.sql

@@ -0,0 +1,8 @@
+{% macro udf_snapshot_get_stake_accounts() %}
+    CREATE
+    OR REPLACE EXTERNAL FUNCTION silver.udf_snapshot_get_stake_accounts() returns text api_integration = aws_solana_api_dev AS {% if target.database == 'SOLANA' -%}
+        'https://pj4rqb8z96.execute-api.us-east-1.amazonaws.com/prod/bulk_snapshot_get_stake_program_accounts'
+    {% else %}
+        'https://11zlwk4fm3.execute-api.us-east-1.amazonaws.com/dev/bulk_snapshot_get_stake_program_accounts'
+    {%- endif %}
+{% endmacro %}

macros/streamline/sp_refresh_external_tables_full.sql

@@ -10,6 +10,8 @@ $$
         alter external table streamline.{{ target.database }}.decoded_instructions_data_api refresh;
         alter external table streamline.{{ target.database }}.validator_metadata_api refresh;
         alter external table streamline.{{ target.database }}.validator_vote_accounts refresh;
+        alter external table streamline.{{ target.database }}.validators_app_list_api refresh;
+        alter external table streamline.{{ target.database }}.stake_program_accounts refresh;
         return TRUE;
     end;
 $${% endset %}

models/silver/validator/silver__snapshot_stake_accounts.sql

@@ -33,7 +33,7 @@ SELECT
     {{ ref('bronze__stake_program_accounts') }}
 
 {% if is_incremental() %}
-WHERE _inserted_timestamp >= (
+WHERE _inserted_timestamp > (
   SELECT
     MAX(_inserted_timestamp)
   FROM

models/silver/validator/silver__snapshot_validators_app_data.sql

@@ -45,7 +45,7 @@ SELECT
     {{ ref('bronze__validators_app_api') }}
 
 {% if is_incremental() %}
-WHERE _inserted_timestamp >= (
+WHERE _inserted_timestamp > (
   SELECT
     MAX(_inserted_timestamp)
   FROM

models/silver/validator/silver__snapshot_vote_accounts.sql

@@ -26,7 +26,7 @@ WITH base AS (
     {{ ref('bronze__vote_accounts') }}
 
 {% if is_incremental() %}
-WHERE _inserted_timestamp >= (
+WHERE _inserted_timestamp > (
   SELECT
     MAX(_inserted_timestamp)
   FROM

models/sources.yml

@@ -45,7 +45,8 @@ sources:
       - name: block_rewards_api
       - name: block_txs_api
       - name: validator_vote_accounts
-      - name: validator_app_list_api
+      - name: validators_app_list_api
+      - name: stake_program_accounts
   - name: bronze_api
     schema: bronze_api
     tables: