Pipeline metadata (#238)

* add: if condition

* removed row_list

* add:  gold view, fix macro, and silver models

* gold view extension

* add: test bronce

* g

* adding test

* add: test

* add: working test

* delete: tags

* delete: hardcode ref

macros/api_udf/check_table_exists.sql

@@ -10,9 +10,9 @@
   {% set results = run_query(query) %}
   {% if execute %}
     {% if results and results.rows[0][0] > 0 %}
-      True
+      "True"
     {% else %}
-      False
+      "False"
     {% endif %}
   {% endif %}
 {% endmacro %}

models/gold/nft/nft__dim_allday_metadata.sql

@@ -9,9 +9,7 @@
     },
     tag = ['scheduled']
 ) }}
-
 WITH allday AS (
-
     SELECT
         nft_id,
         nft_collection,
@@ -33,6 +31,28 @@ WITH allday AS (
         moment_stats_full
     FROM
         {{ ref('silver__nft_allday_metadata') }}
+    UNION
+    SELECT
+        nft_id,
+        nft_collection,
+        nflallday_id,
+        serial_number,
+        moment_tier,
+        total_circulation,
+        moment_description,
+        player,
+        team,
+        season,
+        week,
+        classification,
+        play_type,
+        moment_date,
+        series,
+        set_name,
+        video_urls,
+        moment_stats_full
+    FROM
+        {{ ref('silver__nft_allday_metadata_s') }}
     EXCEPT
     SELECT
         nft_id,

models/silver/nft/livequery/silver__allday_moments_metadata_needed_s.sql

@@ -1,10 +1,10 @@
+-- depends_on: {{ ref('silver__nft_allday_metadata_s') }}
 {{ config(
     materialized = 'view',
     tags = ['livequery', 'allday', 'moment_metadata'],
 ) }}
 
-
-{% set table_exists = check_table_exists('silver', 'nft_allday_metadata_s') %}
+{% set table_exists = check_table_exists('SILVER', 'NFT_ALLDAY_METADATA_S') %}
 
 WITH mints AS (
 
@@ -12,7 +12,7 @@ WITH mints AS (
         event_contract,
         event_data :id :: STRING AS moment_id
     FROM
-        {{ ref('silver__nft_moments') }}
+        {{ ref('silver__nft_moments_s') }}
     WHERE
         event_contract = 'A.e4cf4bdc1751c65d.AllDay'
         AND event_type = 'MomentNFTMinted'
@@ -22,7 +22,7 @@ sales AS (
         nft_collection AS event_contract,
         nft_id AS moment_id
     FROM
-        {{ ref('silver__nft_sales') }}
+        {{ ref('silver__nft_sales_s') }}
     WHERE
         nft_collection = 'A.e4cf4bdc1751c65d.AllDay'
 ),
@@ -47,9 +47,8 @@ SELECT
     nft_id AS moment_id
 FROM
     {{ ref('silver__nft_allday_metadata') }}
+{% if table_exists | trim == '"True"' %}
 EXCEPT
-{% if table_exists == True %}
-    EXCEPT
     SELECT
         nft_collection AS event_contract,
         nft_id AS moment_id

models/silver/nft/silver__nft_allday_metadata_s.sql

@@ -80,3 +80,8 @@ SELECT
     *
 FROM
     FINAL
+qualify ROW_NUMBER() over (
+        PARTITION BY nft_allday_metadata_s_id
+        ORDER BY
+            _inserted_timestamp DESC
+    ) = 1

tests/tests__bronze_api_allday_metadata_response_missing.sql

@@ -0,0 +1,62 @@
+{{ config(
+    severity = "warn",
+    error_if = ">100",
+) }}
+
+WITH latest_day AS (
+
+    SELECT
+        DATE_TRUNC('day', MAX(_inserted_timestamp)) AS last_ingestion_day
+    FROM
+        {{ source(
+            'bronze_api',
+            'allday_metadata'
+        ) }}
+),
+api_call AS (
+    SELECT
+        *
+    FROM
+        {{ source(
+            'bronze_api',
+            'allday_metadata'
+        ) }}
+    WHERE
+        _inserted_timestamp :: DATE = (
+            SELECT
+                last_ingestion_day
+            FROM
+                latest_day
+        )
+        AND contract = 'A.e4cf4bdc1751c65d.AllDay'
+),
+flatten_res AS (
+    SELECT
+        requested_ids,
+        ARRAY_SIZE(requested_ids) AS requested_ids_length,
+        ARRAY_SIZE(
+            res :data :data :searchMomentNFTsV2 :edges
+        ) AS res_length,
+        _INSERTED_TIMESTAMP
+    FROM
+        api_call
+),
+mismatch_calc AS (
+    SELECT
+        requested_ids,
+        CASE
+            WHEN requested_ids_length = 0 THEN 0
+            ELSE ABS(
+                requested_ids_length - res_length
+            ) / requested_ids_length * 100
+        END AS mismatch_percentage,
+        _INSERTED_TIMESTAMP
+    FROM
+        flatten_res
+)
+SELECT
+    *
+FROM
+    mismatch_calc
+WHERE
+    mismatch_percentage > 30