AN-1201/tests (#18)

* tests

* docs

* date filter

* and

* events

macros/tests/sequence_gaps.sql

@@ -0,0 +1,34 @@
+{% test sequence_gaps(
+    model,
+    partition_by,
+    column_name
+) %}
+{%- set partition_sql = partition_by | join(", ") -%}
+{%- set previous_column = "prev_" ~ column_name -%}
+WITH source AS (
+    SELECT
+        {{ partition_sql + "," if partition_sql }}
+        {{ column_name }},
+        LAG(
+            {{ column_name }},
+            1
+        ) over (
+            {{ "PARTITION BY " ~ partition_sql if partition_sql }}
+            ORDER BY
+                {{ column_name }} ASC
+        ) AS {{ previous_column }}
+    FROM
+        {{ model }}
+)
+SELECT
+    {{ partition_sql + "," if partition_sql }}
+    {{ previous_column }},
+    {{ column_name }},
+    {{ column_name }} - {{ previous_column }}
+    - 1 AS gap
+FROM
+    source
+WHERE
+    {{ column_name }} - {{ previous_column }} <> 1
+ORDER BY
+    gap DESC {% endtest %}

macros/tests/tx_gaps.sql

@@ -0,0 +1,35 @@
+{% test tx_gaps(
+    model,
+    column_name,
+    column_block,
+    column_tx_count
+) %}
+WITH block_base AS (
+    SELECT
+        {{ column_block }},
+        {{ column_tx_count }}
+    FROM
+        {{ ref('silver__blocks') }}
+),
+model_name AS (
+    SELECT
+        {{ column_block }},
+        COUNT(
+            DISTINCT {{ column_name }}
+        ) AS model_tx_count
+    FROM
+        {{ model }}
+    GROUP BY
+        {{ column_block }}
+)
+SELECT
+    block_base.{{ column_block }},
+    {{ column_tx_count }},
+    model_name.{{ column_block }},
+    model_tx_count
+FROM
+    block_base
+    LEFT JOIN model_name
+    ON block_base.{{ column_block }} = model_name.{{ column_block }}
+WHERE
+    {{ column_tx_count }} <> model_tx_count {% endtest %}

models/gold/gold__blocks.sql

@@ -11,10 +11,11 @@ WITH silver_blocks AS (
         *
     FROM
         {{ ref('silver__blocks') }}
+    WHERE
+        block_timestamp >= '2022-05-09'
 
 {% if is_incremental() %}
-WHERE
-    _ingested_at :: DATE >= CURRENT_DATE - 2
+AND _ingested_at :: DATE >= CURRENT_DATE - 2
 {% endif %}
 ),
 gold_blocks AS (

models/gold/gold__blocks.yml

@@ -4,6 +4,10 @@ version: 2
 models:
   - name: gold__blocks
     description: Information about blocks on the FLOW network and corresponding metadata.
+    tests:
+      - sequence_gaps:
+          column_name: block_height
+          where: BLOCK_TIMESTAMP < CURRENT_DATE
 
     columns:
       - name: block_height

models/gold/gold__events.sql

@@ -11,10 +11,11 @@ WITH silver_events AS (
         *
     FROM
         {{ ref('silver__events') }}
+    WHERE
+        block_timestamp >= '2022-05-09'
 
 {% if is_incremental() %}
-WHERE
-    _ingested_at :: DATE >= CURRENT_DATE -2
+AND _ingested_at :: DATE >= CURRENT_DATE -2
 {% endif %}
 ),
 silver_event_attributes AS (
@@ -22,10 +23,11 @@ silver_event_attributes AS (
         *
     FROM
         {{ ref('silver__event_attributes') }}
+    WHERE
+        block_timestamp >= '2022-05-09'
 
 {% if is_incremental() %}
-WHERE
-    _ingested_at :: DATE >= CURRENT_DATE -2
+AND _ingested_at :: DATE >= CURRENT_DATE -2
 {% endif %}
 ),
 objs AS (

models/gold/gold__transactions.sql

@@ -11,10 +11,11 @@ WITH silver_txs AS (
         *
     FROM
         {{ ref('silver__transactions') }}
+    WHERE
+        block_timestamp >= '2022-05-09'
 
 {% if is_incremental() %}
-WHERE
-    _ingested_at :: DATE > CURRENT_DATE - 2
+AND _ingested_at :: DATE > CURRENT_DATE - 2
 {% endif %}
 ),
 gold_txs AS (

models/gold/gold__transactions.yml

@@ -17,6 +17,10 @@ models:
         tests:
           - not_null
           - unique
+          - tx_gaps:
+              column_block: block_height
+              column_tx_count: tx_count
+              where: BLOCK_TIMESTAMP < CURRENT_DATE
 
       - name: block_timestamp
         description: "{{ doc('block_timestamp') }}"

models/silver/silver__blocks.yml

@@ -8,7 +8,7 @@ models:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - block_height
-
+          
     columns:
       - name: block_height
         description: "{{ doc('block_height') }}"