From 2a4846b9729b42e2ef451d9bc9b7f73b40716266 Mon Sep 17 00:00:00 2001
From: Jack Forgash <58153492+forgxyz@users.noreply.github.com>
Date: Thu, 20 Feb 2025 12:38:16 -0700
Subject: [PATCH] Quickfix rewards balances (#412)

* chg test threshold and decrease concurrency

* rm created_at null tests

* elevate response json, fix qualify loc

* fix unique key

* chg gha silver to 2utc
---
 .../dbt_run_scheduled_reward_points_silver.yml       |  4 ++--
 .../gold/rewards/rewards__dim_storefront_items.sql   |  1 +
 .../gold/rewards/rewards__dim_storefront_items.yml   |  8 ++------
 .../gold/rewards/rewards__fact_points_balances.yml   |  2 +-
 .../balances/streamline__reward_points_realtime.sql  |  6 +++---
 .../silver_api__storefront_items.sql                 | 12 +++++++-----
 .../points/tests__unique_address_count_threshold.sql |  2 +-
 7 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/dbt_run_scheduled_reward_points_silver.yml b/.github/workflows/dbt_run_scheduled_reward_points_silver.yml
index bc9bc20..f22a231 100644
--- a/.github/workflows/dbt_run_scheduled_reward_points_silver.yml
+++ b/.github/workflows/dbt_run_scheduled_reward_points_silver.yml
@@ -4,8 +4,8 @@ run-name: dbt_run_scheduled_reward_points_silver
 on:
   workflow_dispatch:
   schedule:
-    # Daily at 01:00 UTC
-    - cron: "0 1 * * *"
+    # Daily at 02:00 UTC
+    - cron: "0 2 * * *"
 
 env:
   SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
diff --git a/models/gold/rewards/rewards__dim_storefront_items.sql b/models/gold/rewards/rewards__dim_storefront_items.sql
index 68d6e72..acc532a 100644
--- a/models/gold/rewards/rewards__dim_storefront_items.sql
+++ b/models/gold/rewards/rewards__dim_storefront_items.sql
@@ -57,6 +57,7 @@ SELECT
     token_type,
     updated_at,
     website_id,
+    response_json,
     storefront_item_id AS dim_storefront_items_id,
     inserted_timestamp,
     modified_timestamp,
diff --git a/models/gold/rewards/rewards__dim_storefront_items.yml b/models/gold/rewards/rewards__dim_storefront_items.yml
index 027b47e..a0210f7 100644
--- a/models/gold/rewards/rewards__dim_storefront_items.yml
+++ b/models/gold/rewards/rewards__dim_storefront_items.yml
@@ -4,10 +4,6 @@ models:
   - name: rewards__dim_storefront_items
     description: "Response from the Minting Assets API"
     tests:
-      - dbt_utils.recency:
-          datepart: week
-          field: created_at
-          interval: 1
       - dbt_utils.recency:
           datepart: day
           field: request_timestamp
@@ -19,8 +15,6 @@ models:
           - not_null
 
       - name: CREATED_AT
-        tests:
-          - not_null
 
       - name: ADDITIONAL_MEDIA
 
@@ -122,6 +116,8 @@ models:
 
       - name: WEBSITE_ID  
 
+      - name: RESPONSE_JSON
+
       - name: DIM_STOREFRONT_ITEMS_ID
         tests:
           - not_null
diff --git a/models/gold/rewards/rewards__fact_points_balances.yml b/models/gold/rewards/rewards__fact_points_balances.yml
index 90dbe4a..1095f83 100644
--- a/models/gold/rewards/rewards__fact_points_balances.yml
+++ b/models/gold/rewards/rewards__fact_points_balances.yml
@@ -36,7 +36,7 @@ models:
         tests:
           - not_null:
               where:
-                request_date > '2025-02-07'
+                request_date > '2025-02-20'
           - dbt_expectations.expect_column_values_to_be_in_type_list:
               column_type_list:
                 - NUMBER
diff --git a/models/streamline/external/balances/streamline__reward_points_realtime.sql b/models/streamline/external/balances/streamline__reward_points_realtime.sql
index 0cf798f..73d12f4 100644
--- a/models/streamline/external/balances/streamline__reward_points_realtime.sql
+++ b/models/streamline/external/balances/streamline__reward_points_realtime.sql
@@ -5,9 +5,9 @@
         target = "{{this.schema}}.{{this.identifier}}",
         params = {
             "external_table": "reward_points",
-            "sql_limit": "32000",
-            "producer_batch_size": "4000",
-            "worker_batch_size": "1000",
+            "sql_limit": "36000",
+            "producer_batch_size": "3000",
+            "worker_batch_size": "1500",
             "sql_source": "{{this.identifier}}"
         }
     )
diff --git a/models/streamline/external/storefront_items/silver_api__storefront_items.sql b/models/streamline/external/storefront_items/silver_api__storefront_items.sql
index f3f76a2..0a6ca54 100644
--- a/models/streamline/external/storefront_items/silver_api__storefront_items.sql
+++ b/models/streamline/external/storefront_items/silver_api__storefront_items.sql
@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = "storefront_item_id",
+    unique_key = "item_id",
     incremental_strategy = 'merge',
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['_inserted_timestamp :: DATE'],
@@ -30,9 +30,6 @@ WHERE
     )
 {% endif %}
 
-qualify(ROW_NUMBER() over (PARTITION BY item_id
-ORDER BY
-    _inserted_timestamp DESC)) = 1
 )
 SELECT
     item_id,
@@ -88,7 +85,7 @@ SELECT
     DATA :tokenType :: STRING AS token_type,
     DATA :updatedAt :: timestamp_ntz AS updated_at,
     DATA :websiteId :: STRING AS website_id,
-
+    DATA AS response_json,
     partition_key,
     INDEX,
     _inserted_timestamp,
@@ -100,3 +97,8 @@ SELECT
     '{{ invocation_id }}' AS _invocation_id
 FROM
     silver_responses
+
+
+qualify(ROW_NUMBER() over (PARTITION BY item_id
+ORDER BY
+    _inserted_timestamp DESC)) = 1
diff --git a/tests/points/tests__unique_address_count_threshold.sql b/tests/points/tests__unique_address_count_threshold.sql
index 77663ef..c0c1d1b 100644
--- a/tests/points/tests__unique_address_count_threshold.sql
+++ b/tests/points/tests__unique_address_count_threshold.sql
@@ -18,4 +18,4 @@ SELECT
 FROM
     distinct_count
 WHERE
-    ct > 30000
+    ct > 36000