actions, test cleanup

.github/workflows/dbt_run_daily.yml

@@ -4,10 +4,11 @@ run-name: dbt_run_scheduled
 on:
   workflow_dispatch:
   schedule:
-    # Runs "every hour" (see https://crontab.guru)
-    - cron: '0 */1 * * *'
+    # Runs 0700 daily (see https://crontab.guru) 
+    - cron: '0 7 * * *'
 
 env:
+  USE_VARS: "${{ vars.USE_VARS }}"
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
   DBT_VERSION: "${{ vars.DBT_VERSION }}"
   ACCOUNT: "${{ vars.ACCOUNT }}"

.github/workflows/dbt_run_daily_dev_refresh.yml

@@ -8,6 +8,7 @@ on:
     - cron: '0 8 * * *'
 
 env:
+  USE_VARS: "${{ vars.USE_VARS }}"
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
   DBT_VERSION: "${{ vars.DBT_VERSION }}"
   ACCOUNT: "${{ vars.ACCOUNT }}"

.github/workflows/dbt_test.yml

@@ -4,9 +4,10 @@ run-name: dbt_test_scheduled
 on:
   workflow_dispatch:
   schedule:
-    # Run at 1600 UTC daily (see https://crontab.guru)
-    - cron: '0 16 * * *'
+    # Run at 0900 UTC daily (see https://crontab.guru)
+    - cron: '0 9 * * *'
 env:
+  USE_VARS: "${{ vars.USE_VARS }}"
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
   DBT_VERSION: "${{ vars.DBT_VERSION }}"
   ACCOUNT: "${{ vars.ACCOUNT }}"

.github/workflows/dbt_test_workflow.yml

@@ -1,30 +0,0 @@
-name: dbt_test_workflow
-run-name: dbt_test_workflow
-
-on:
-  workflow_dispatch:
-
-env:
-  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
-  DBT_VERSION: "${{ vars.DBT_VERSION }}"
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
-concurrency:
-  group: ${{ github.workflow }}
-
-jobs:
-  called_workflow_template:
-    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@alter_workflow_var
-    with:
-      dbt_command: >
-        dbt seed; dbt run
-      environment: workflow_prod
-      warehouse: ${{ vars.WAREHOUSE }}
-    secrets: inherit 

macros/run_sp_create_prod_clone.sql

@@ -3,7 +3,7 @@
     call terra._internal.create_prod_clone(
         'terra',
         'terra_dev',
-        'dbt_cloud'
+        'internal_dev'
     );
 {% endset %}
     {% do run_query(clone_query) %}

models/core/core__ez_staking.yml

@@ -9,6 +9,7 @@ models:
             - ACTION
             - MESSAGE_INDEX
             - DELEGATOR_ADDRESS
+            - VALIDATOR_ADDRESS
     columns:
       - name: ACTION
         description: "{{ doc('action') }}"

models/core/core__fact_blocks.yml

@@ -10,7 +10,7 @@ models:
             - block_id
       - sequence_gaps:
           column_name: block_id
-          where: BLOCK_TIMESTAMP < CURRENT_DATE
+          where: BLOCK_TIMESTAMP < CURRENT_DATE and BLOCK_ID > 4380273
 
     columns:
       - name: BLOCK_ID
@@ -161,7 +161,8 @@ models:
       - name: VALIDATOR_ADDRESS_ARRAY
         description: "{{ doc('validator_address_array') }}"
         tests:
-          - not_null
+          - not_null:
+              where: BLOCK_TIMESTAMP < CURRENT_DATE and BLOCK_ID > 4380273
           - dbt_expectations.expect_column_values_to_be_in_type_list:
               column_type_list:
                 - ARRAY

models/silver/silver__blocks.sql

@@ -21,40 +21,33 @@ WITH base_blocks AS (
     FROM
         {{ ref('bronze__blocks') }}
     WHERE
-        {{ incremental_load_filter('_inserted_timestamp') }}
+        {{ incremental_last_x_days(
+            '_inserted_timestamp',
+            3
+        ) }}
         qualify ROW_NUMBER() over (
             PARTITION BY block_id
             ORDER BY
                 _inserted_timestamp DESC
         ) = 1
 ),
-validator_signatures AS (
-    SELECT
-        header :last_commit :height AS block_id,
-        header :last_commit :signatures AS signatures
-    FROM
-        base_blocks
-),
-validator_addresses AS (
-    SELECT
-        validator_signatures.block_id AS block_id,
-        s0.value :validator_address AS validator_address
-    FROM
-        validator_signatures,
-        LATERAL FLATTEN(
-            input => validator_signatures.signatures
-        ) AS s0
-),
 validators_address_array AS (
     SELECT
-        CAST(validator_addresses.block_id AS NUMBER(38, 0)) AS block_id,
-        ARRAY_AGG(
-            DISTINCT validator_addresses.validator_address
-        ) AS address_array
+        block_id,
+        validator_address_array,
+        _inserted_timestamp
     FROM
-        validator_addresses
-    GROUP BY
-        validator_addresses.block_id
+        {{ ref('silver__blocks_val_array') }}
+    WHERE
+        {{ incremental_last_x_days(
+            '_inserted_timestamp',
+            3
+        ) }}
+        qualify ROW_NUMBER() over (
+            PARTITION BY block_id
+            ORDER BY
+                _inserted_timestamp DESC
+        ) = 1
 ),
 FINAL AS (
     SELECT
@@ -77,7 +70,7 @@ FINAL AS (
         base_blocks.header :validators_hash :: STRING AS validators_hash,
         base_blocks._ingested_at AS _ingested_at,
         base_blocks._inserted_timestamp AS _inserted_timestamp,
-        validators_address_array.address_array :: ARRAY AS validator_address_array
+        validators_address_array.validator_address_array :: ARRAY AS validator_address_array
     FROM
         base_blocks
         LEFT JOIN validators_address_array

models/silver/silver__blocks_val_array.sql

@@ -0,0 +1,61 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = 'block_id',
+    incremental_strategy = 'delete+insert',
+    cluster_by = ['_inserted_timestamp::DATE'],
+) }}
+
+WITH base_blocks AS (
+
+    SELECT
+        header,
+        _inserted_timestamp
+    FROM
+        {{ ref('bronze__blocks') }}
+    WHERE
+        {{ incremental_last_x_days(
+            '_inserted_timestamp',
+            3
+        ) }}
+        qualify ROW_NUMBER() over (
+            PARTITION BY block_id
+            ORDER BY
+                _inserted_timestamp DESC
+        ) = 1
+),
+validator_signatures AS (
+    SELECT
+        header :last_commit :height AS block_id,
+        header :last_commit :signatures AS signatures,
+        _inserted_timestamp
+    FROM
+        base_blocks
+),
+validator_addresses AS (
+    SELECT
+        validator_signatures.block_id AS block_id,
+        s0.value :validator_address AS validator_address,
+        _inserted_timestamp
+    FROM
+        validator_signatures,
+        LATERAL FLATTEN(
+            input => validator_signatures.signatures
+        ) AS s0
+),
+validators_address_array AS (
+    SELECT
+        CAST(validator_addresses.block_id AS NUMBER(38, 0)) AS block_id,
+        ARRAY_AGG(
+            DISTINCT validator_addresses.validator_address
+        ) AS validator_address_array,
+        _inserted_timestamp
+    FROM
+        validator_addresses
+    GROUP BY
+        validator_addresses.block_id,
+        _inserted_timestamp
+)
+SELECT
+    *
+FROM
+    validators_address_array

models/silver/silver__blocks_val_array.yml

@@ -0,0 +1,38 @@
+version: 2
+
+models:
+  - name: silver__blocks_val_array
+    description: |-
+      This table records all the blocks of Terra2 blockchain.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - block_id
+    columns:
+      - name: BLOCK_ID
+        description: "{{ doc('block_id')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+
+      
+      - name: VALIDATOR_ADDRESS_ARRAY
+        description: "{{ doc('validator_address_array') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - ARRAY
+                - VARIANT
+                - OBJECT
+
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('_inserted_timestamp')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ