diff --git a/models/bronze/bronze__moments_metadata.sql b/models/bronze/bronze__moments_metadata.sql index 4f6519d..d00229e 100644 --- a/models/bronze/bronze__moments_metadata.sql +++ b/models/bronze/bronze__moments_metadata.sql @@ -7,6 +7,7 @@ SELECT contract, DATA, VALUE, + _inserted_date, TO_TIMESTAMP_NTZ(SUBSTR(SPLIT_PART(metadata$filename, '/', 4), 1, 10) :: NUMBER, 0) AS _inserted_timestamp FROM {{ source( diff --git a/models/streamline/streamline__all_topshot_moments_minted_metadata_needed.sql b/models/streamline/streamline__all_topshot_moments_minted_metadata_needed.sql index d0f2904..58a2382 100644 --- a/models/streamline/streamline__all_topshot_moments_minted_metadata_needed.sql +++ b/models/streamline/streamline__all_topshot_moments_minted_metadata_needed.sql @@ -34,14 +34,37 @@ all_topshots AS ( moment_id FROM sales +), +always_null AS ( + SELECT + id, + contract, + COUNT(*) AS num_times_null_resp + FROM + {{ ref('streamline__null_moments_metadata') }} + WHERE + contract = 'A.0b2a3299cc857e29.TopShot' + GROUP BY + 1, + 2 + HAVING + num_times_null_resp > 2 ) SELECT DISTINCT * FROM all_topshots EXCEPT -SELECT - nft_collection as event_contract, - nft_id AS moment_id -FROM - {{ ref('silver__nft_topshot_metadata') }} + ( + SELECT + nft_collection AS event_contract, + nft_id AS moment_id + FROM + {{ ref('silver__nft_topshot_metadata') }} + UNION + SELECT + contract, + id + FROM + always_null + ) diff --git a/models/streamline/streamline__null_moments_metadata.sql b/models/streamline/streamline__null_moments_metadata.sql new file mode 100644 index 0000000..8209ed6 --- /dev/null +++ b/models/streamline/streamline__null_moments_metadata.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = 'incremental', + unique_key = ["id","contract","_inserted_date"] +) }} + +SELECT + id, + contract, + _inserted_date, + _inserted_timestamp +FROM + {{ ref('bronze__moments_metadata') }} +WHERE + DATA :getMintedMoment :: STRING IS NULL + +{% if is_incremental() %} +AND _inserted_date >= ( + SELECT + MAX(_inserted_date) + FROM + {{ this }} +) +AND _inserted_timestamp > ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} +) +{% else %} +AND _inserted_date >= '2022-12-09' +{% endif %}