incremental logic and bespoke tests

macros/tests/sequence_distinct_gaps_dim_block_id.sql

@@ -0,0 +1,42 @@
+{% macro sequence_distinct_gaps_dim_block_id(
+        table,
+        column
+    ) %}
+    {%- set partition_sql = partition_by | join(", ") -%}
+    {%- set previous_column = "prev_" ~ column -%}
+    WITH source AS (
+        SELECT
+            {{ partition_sql + "," if partition_sql }}
+            {{ column }},
+            LAG(
+                {{ column }},
+                1
+            ) over (
+                ORDER BY
+                    {{ column }} ASC
+            ) AS {{ previous_column }}
+        FROM
+            (
+                SELECT
+                    DISTINCT {{ column }}
+                FROM
+                    {{ table }} A
+                    JOIN {{ ref('core__dim_block') }}
+                    b
+                    ON A.dim_block_id = b.dim_block_id
+                WHERE
+                    A.dim_block_id <> '-1'
+            )
+    )
+SELECT
+    {{ previous_column }},
+    {{ column }},
+    {{ column }} - {{ previous_column }}
+    - 1 AS gap
+FROM
+    source
+WHERE
+    {{ column }} - {{ previous_column }} <> 1
+ORDER BY
+    gap DESC
+{% endmacro %}

models/core/core__dim_block.sql

@@ -37,7 +37,7 @@ WHERE
             )
         FROM
             {{ this }}
-    ) - INTERVAL '4 HOURS'
+    ) - INTERVAL '48 HOURS'
 {% endif %}
 UNION ALL
 SELECT

models/core/core__fact_active_vault_events.sql

@@ -25,6 +25,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR add_asgard_addr IN (
+    SELECT
+      add_asgard_addr
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_add_events.sql

@@ -34,6 +34,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR tx_id IN (
+    SELECT
+      tx_id
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_asgard_fund_yggdrasil_events.sql

@@ -28,6 +28,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR tx_id IN (
+    SELECT
+      tx_id
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_block_pool_depths.sql

@@ -27,6 +27,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR pool_name IN (
+    SELECT
+      pool_name
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_bond_actions.sql

@@ -42,6 +42,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR tx_id IN (
+    SELECT
+      tx_id
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_bond_events.sql

@@ -32,6 +32,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR tx_id IN (
+    SELECT
+      tx_id
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_errata_events.sql

@@ -28,6 +28,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR asset IN (
+    SELECT
+      asset
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_fee_events.sql

@@ -28,6 +28,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR tx_id IN (
+    SELECT
+      tx_id
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_gas_events.sql

@@ -28,6 +28,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR asset IN (
+    SELECT
+      asset
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_inactive_vault_events.sql

@@ -25,6 +25,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR add_asgard_address IN (
+    SELECT
+      add_asgard_address
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_liquidity_actions.sql

@@ -40,7 +40,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_new_node_events.sql

@@ -25,6 +25,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR node_address IN (
+    SELECT
+      node_address
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_outbound_events.sql

@@ -31,7 +31,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_pending_liquidity_events.sql

@@ -32,7 +32,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_pool_balance_change_events.sql

@@ -30,6 +30,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR asset IN (
+    SELECT
+      asset
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_pool_block_balances.sql

@@ -30,7 +30,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_pool_block_fees.sql

@@ -29,7 +29,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_pool_events.sql

@@ -26,6 +26,14 @@ WHERE
     FROM
       {{ this }}
   ) - INTERVAL '4 HOURS'
+  OR asset IN (
+    SELECT
+      asset
+    FROM
+      {{ this }}
+    WHERE
+      dim_block_id = '-1'
+  )
 {% endif %}
 )
 SELECT

models/core/core__fact_refund_events.sql

@@ -34,7 +34,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_reserve_events.sql

@@ -32,7 +32,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_rewards_event_entries.sql

@@ -25,7 +25,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_rewards_events.sql

@@ -24,7 +24,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_set_ip_address_events.sql

@@ -25,7 +25,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_set_mimir_events.sql

@@ -25,7 +25,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_set_node_keys_events.sql

@@ -27,7 +27,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_set_version_events.sql

@@ -25,7 +25,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_slash_amounts.sql

@@ -26,7 +26,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_slash_points.sql

@@ -26,7 +26,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_stake_events.sql

@@ -33,7 +33,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_swaps.sql

@@ -47,7 +47,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_switch_events.sql

@@ -29,7 +29,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_thorname_change_events.sql

@@ -34,7 +34,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_total_block_rewards.sql

@@ -26,7 +26,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_transfer_events.sql

@@ -27,7 +27,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_transfers.sql

@@ -28,7 +28,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_unstake_events.sql

@@ -38,7 +38,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_update_node_account_status_events.sql

@@ -25,7 +25,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_upgrades.sql

@@ -31,7 +31,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/core/core__fact_validator_request_leave_events.sql

@@ -26,7 +26,7 @@ WHERE
       )
     FROM
       {{ this }}
-  ) - INTERVAL '4 HOURS'
+  ) - INTERVAL '48 HOURS'
 {% endif %}
 )
 SELECT

models/silver/silver__block_rewards.yml

@@ -27,7 +27,4 @@ models:
       - name: LIQUIDITY_EARNINGS
         tests:
           - not_null
-      - name: AVG_NODE_COUNT
-        tests:
-          - not_null:
-              where: DAY < CURRENT_DATE - 2
+      - name: AVG_NODE_COUNT

tests/core__fact_block_pool_depths__block_id-assert_no_gaps.sql

@@ -0,0 +1 @@
+{{ sequence_distinct_gaps_dim_block_id (ref("core__fact_block_pool_depths"), "block_id") }}

tests/core__fact_block_rewards__day-assert_no_gap.sql

@@ -0,0 +1 @@
+{{ date_gaps(ref("core__fact_block_rewards"), [], "day", dict(start_date = "cast('2021-07-22' as date)", end_date = "cast('2021-08-11' as date)")) }}

tests/core__fact_daily_earnings__day-assert_no_gaps.sql

@@ -0,0 +1 @@
+{{ date_gaps(ref("core__fact_daily_earnings"), [], "day", dict(start_date = "cast('2021-07-22' as date)", end_date = "cast('2021-08-11' as date)")) }}

tests/core__fact_daily_pool_stats__day-assert_no_gaps.sql

@@ -0,0 +1 @@
+{{ date_gaps(ref("core__fact_daily_pool_stats"), [], "day", dict(start_date = "cast('2021-07-22' as date)", end_date = "cast('2021-08-11' as date)")) }}

tests/core__fact_daily_tvl__day-assert_no_gaps.sql

@@ -0,0 +1 @@
+{{ date_gaps(ref("core__fact_daily_tvl"), [], "day", dict(start_date = "cast('2021-07-22' as date)", end_date = "cast('2021-08-11' as date)")) }}

tests/core__fact_pool_block_balances__block_id-assert_no_gaps.sql

@@ -0,0 +1 @@
+{{ sequence_distinct_gaps_dim_block_id(ref("corte__fact_pool_block_balances"), "block_id") }}

tests/core__fact_pool_block_fees__day-assert_no_gap.sql

@@ -0,0 +1 @@
+{{ date_gaps(ref("core__fact_pool_block_fees"), [], "day", dict(start_date = "cast('2021-07-22' as date)", end_date = "cast('2021-08-11' as date)")) }}

tests/core__fact_pool_block_statistics__day-assert_no_gap.sql

@@ -0,0 +1 @@
+{{ date_gaps(ref("core__fact_pool_block_statistics"), [], "day", dict(start_date = "cast('2021-07-22' as date)", end_date = "cast('2021-08-11' as date)")) }}

tests/core__fact_prices__block_id-assert_no_gaps.sql

@@ -0,0 +1 @@
+{{ sequence_distinct_gaps_dim_block_id(ref("core__fact_prices"), "block_id") }}

tests/core__fact_swaps_assert_all_ids_exist_swap_events.sql

@@ -0,0 +1 @@
+{{ all_ids_exist(ref("core__fact_swap_events"), "tx_id", ref("core__fact_swaps"), "tx_id") }}

tests/core__fact_total_block_rewards__block_id-assert_no_gaps.sql

@@ -0,0 +1,5 @@
+{{ config(
+    warn_if = "> 10",
+    severity = "warn"
+) }}
+{{ sequence_distinct_gaps_dim_block_id(ref("core__fact_total_block_rewards"), "block_id") }}

tests/core__fact_total_value_locked__day-assert_no_gap.sql

@@ -0,0 +1 @@
+{{ date_gaps(ref("core__fact_total_value_locked"), [], "day", dict(start_date = "cast('2021-07-22' as date)", end_date = "cast('2021-08-11' as date)"), dict(start_date = "cast('2021-11-12' as date)", end_date = "cast('2021-11-18' as date)")) }}