From 2e7d617fe784e04fac2c0b4aaa96968482b24ed3 Mon Sep 17 00:00:00 2001
From: Jack Forgash <58153492+forgxyz@users.noreply.github.com>
Date: Tue, 22 Apr 2025 11:02:33 -0600
Subject: [PATCH] Hotfix/Points Transfers Model FR Override (#435)

* upd gitignore

* add fallback to FR table if bad data
---
 .gitignore                                    |  5 +++-
 .../silver_api__points_transfers_response.sql | 29 +++++++++++++++++--
 .../silver_api__points_transfers_response.yml |  7 -----
 3 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/.gitignore b/.gitignore
index a5fbc78..4e6a53f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,4 +19,7 @@ logs/
 .env
 
 .github/workflows/.dbt/.user.yml
-local*
\ No newline at end of file
+local*
+venv/
+.cursor/
+.cursorrules
diff --git a/models/streamline/external/transfers/silver_api__points_transfers_response.sql b/models/streamline/external/transfers/silver_api__points_transfers_response.sql
index 2d13a3b..b4c6377 100644
--- a/models/streamline/external/transfers/silver_api__points_transfers_response.sql
+++ b/models/streamline/external/transfers/silver_api__points_transfers_response.sql
@@ -14,15 +14,36 @@
         MAX(partition_key)
     FROM
         {{ ref('bronze_api__points_transfers') }}
+    WHERE
+        TYPEOF(data) != 'NULL_VALUE'
 
         {% endset %}
         {% set max_partition_key = run_query(query) [0] [0] %}
+        {% set must_fr = False %}
         {% do log(
             "max_partition_key: " ~ max_partition_key,
             info = True
         ) %}
         {% if max_partition_key == '' or max_partition_key is none %}
-            {% do exceptions.raise_compiler_error("max_partition_key is not set. Aborting model execution.") %}
+            {% do log("No recent data found. Using FR table.", info = True) %}
+            {% set query %}
+
+            SELECT
+                MAX(partition_key)
+            FROM
+                {{ ref('bronze_api__FR_points_transfers') }}
+            WHERE
+                TYPEOF(data) != 'NULL_VALUE'
+
+            {% endset%}
+
+            {% set max_partition_key = run_query(query) [0] [0] %}
+            {% set must_fr = True %}
+            {% do log(
+                "max_partition_key: " ~ max_partition_key,
+                info = True
+            ) %}
+            {# {% do exceptions.raise_compiler_error("max_partition_key is not set. Aborting model execution.") %} #}
         {% endif %}
     {% endif %}
 
@@ -44,6 +65,10 @@ SELECT
     SYSDATE() AS modified_timestamp,
     '{{ invocation_id }}' AS _invocation_id
 FROM
-    {{ ref('bronze_api__points_transfers') }}
+    {% if must_fr %}
+        {{ ref('bronze_api__FR_points_transfers') }}
+    {% else %}
+        {{ ref('bronze_api__points_transfers') }}
+    {% endif %}
 WHERE
     partition_key = {{ max_partition_key }}
diff --git a/models/streamline/external/transfers/silver_api__points_transfers_response.yml b/models/streamline/external/transfers/silver_api__points_transfers_response.yml
index 84eb1ea..cc0a0eb 100644
--- a/models/streamline/external/transfers/silver_api__points_transfers_response.yml
+++ b/models/streamline/external/transfers/silver_api__points_transfers_response.yml
@@ -25,13 +25,6 @@ models:
         tests:
           - not_null
 
-      - name: data_mb
-        tests:
-          - dbt_utils.accepted_range:
-              min_value: 0
-              max_value: 15.5
-              severity: error
-
       - name: points_transfers_id
         tests:
           - not_null