Quickfix rewards balances (#412)

* chg test threshold and decrease concurrency

* rm created_at null tests

* elevate response json, fix qualify loc

* fix unique key

* chg gha silver to 2utc

.github/workflows/dbt_run_scheduled_reward_points_silver.yml

@@ -4,8 +4,8 @@ run-name: dbt_run_scheduled_reward_points_silver
 on:
   workflow_dispatch:
   schedule:
-    # Daily at 01:00 UTC
-    - cron: "0 1 * * *"
+    # Daily at 02:00 UTC
+    - cron: "0 2 * * *"
 
 env:
   SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"

models/gold/rewards/rewards__dim_storefront_items.sql

@@ -57,6 +57,7 @@ SELECT
     token_type,
     updated_at,
     website_id,
+    response_json,
     storefront_item_id AS dim_storefront_items_id,
     inserted_timestamp,
     modified_timestamp,

models/gold/rewards/rewards__dim_storefront_items.yml

@@ -4,10 +4,6 @@ models:
   - name: rewards__dim_storefront_items
     description: "Response from the Minting Assets API"
     tests:
-      - dbt_utils.recency:
-          datepart: week
-          field: created_at
-          interval: 1
       - dbt_utils.recency:
           datepart: day
           field: request_timestamp
@@ -19,8 +15,6 @@ models:
           - not_null
 
       - name: CREATED_AT
-        tests:
-          - not_null
 
       - name: ADDITIONAL_MEDIA
 
@@ -122,6 +116,8 @@ models:
 
       - name: WEBSITE_ID  
 
+      - name: RESPONSE_JSON
+
       - name: DIM_STOREFRONT_ITEMS_ID
         tests:
           - not_null

models/gold/rewards/rewards__fact_points_balances.yml

@@ -36,7 +36,7 @@ models:
         tests:
           - not_null:
               where:
-                request_date > '2025-02-07'
+                request_date > '2025-02-20'
           - dbt_expectations.expect_column_values_to_be_in_type_list:
               column_type_list:
                 - NUMBER

models/streamline/external/balances/streamline__reward_points_realtime.sql

@@ -5,9 +5,9 @@
         target = "{{this.schema}}.{{this.identifier}}",
         params = {
             "external_table": "reward_points",
-            "sql_limit": "32000",
-            "producer_batch_size": "4000",
-            "worker_batch_size": "1000",
+            "sql_limit": "36000",
+            "producer_batch_size": "3000",
+            "worker_batch_size": "1500",
             "sql_source": "{{this.identifier}}"
         }
     )

models/streamline/external/storefront_items/silver_api__storefront_items.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = "storefront_item_id",
+    unique_key = "item_id",
     incremental_strategy = 'merge',
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['_inserted_timestamp :: DATE'],
@@ -30,9 +30,6 @@ WHERE
     )
 {% endif %}
 
-qualify(ROW_NUMBER() over (PARTITION BY item_id
-ORDER BY
-    _inserted_timestamp DESC)) = 1
 )
 SELECT
     item_id,
@@ -88,7 +85,7 @@ SELECT
     DATA :tokenType :: STRING AS token_type,
     DATA :updatedAt :: timestamp_ntz AS updated_at,
     DATA :websiteId :: STRING AS website_id,
-
+    DATA AS response_json,
     partition_key,
     INDEX,
     _inserted_timestamp,
@@ -100,3 +97,8 @@ SELECT
     '{{ invocation_id }}' AS _invocation_id
 FROM
     silver_responses
+
+
+qualify(ROW_NUMBER() over (PARTITION BY item_id
+ORDER BY
+    _inserted_timestamp DESC)) = 1

tests/points/tests__unique_address_count_threshold.sql

@@ -18,4 +18,4 @@ SELECT
 FROM
     distinct_count
 WHERE
-    ct > 30000
+    ct > 36000