AN-1870/allday ids (#51)

* allday nft ids

* silver topshot view for compat

* limit allday to 3000

* str

* add nfl all day metadata integration and refactor streamline process

* no longer needed

Co-authored-by: Desmond Hui <desmond@flipsidecrypto.com>

dbt_project.yml

@@ -27,7 +27,6 @@ clean-targets: # directories to be removed by `dbt clean`
 on-run-start:
   - "{{create_sps()}}"
   - "{{create_udfs()}}"
-  - "{{sp_create_bulk_get_topshot_moments_minted_metadata()}}"
 
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models

macros/create_udfs.sql

@@ -1,6 +1,7 @@
 {% macro create_udfs() %}
     {% set sql %}
     {{ udf_bulk_get_topshot_moments_minted_metadata() }};
+    {{ udf_bulk_get_nfl_allday_moments_metadata() }};
     {% endset %}
     {% do run_query(sql) %}
 {% endmacro %}

macros/sp_create_bulk_get_topshot_moments_minted_metadata.sql

@@ -1,26 +0,0 @@
-{% macro sp_create_bulk_get_topshot_moments_minted_metadata() %}
-  {% set sql %}
-  CREATE
-  OR REPLACE PROCEDURE silver.sp_bulk_get_topshot_moments_minted_metadata() returns variant LANGUAGE SQL AS $$
-DECLARE
-  RESULT VARCHAR;
-row_cnt INTEGER;
-BEGIN
-  row_cnt:= (
-    SELECT
-      COUNT(1)
-    FROM
-      {{ ref('silver__all_topshot_moments_minted_metadata_needed') }}
-  );
-if (
-    row_cnt > 0
-  ) THEN RESULT:= (
-    SELECT
-      silver.udf_bulk_get_topshot_moments_minted_metadata()
-  );
-  ELSE RESULT:= NULL;
-END if;
-RETURN RESULT;
-END;$$ {% endset %}
-{% do run_query(sql) %}
-{% endmacro %}

macros/streamline/get_nfl_allday_moments_metadata/run_bulk_get_nfl_allday_moments_metadata.sql

@@ -0,0 +1,15 @@
+{% macro run_bulk_get_nfl_allday_moments_metadata() %}
+{% set sql %}
+
+
+select streamline.udf_bulk_get_nfl_allday_moments_metadata()
+where exists (
+    select 1
+    from streamline.allday_moments_metadata_needed
+    limit 1 
+)
+
+{% endset %}
+
+{% do run_query(sql) %}
+{% endmacro %}

macros/streamline/get_nfl_allday_moments_metadata/udf_bulk_get_nfl_allday_moments_metadata.sql

@@ -0,0 +1,8 @@
+{% macro udf_bulk_get_nfl_allday_moments_metadata() %}
+    CREATE
+    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_nfl_allday_moments_metadata() returns text api_integration = aws_flow_api_dev AS {% if target.database == "FLOW" -%}
+        'https://3ltti6kisi.execute-api.us-east-1.amazonaws.com/prod/bulk_get_nfl_allday_metadata'
+    {% else %}
+        'https://wn6lmi2rs4.execute-api.us-east-1.amazonaws.com/dev/bulk_get_nfl_allday_metadata'
+    {%- endif %}
+{% endmacro %}

macros/streamline/get_topshot_moments_metadata/run_bulk_get_topshot_moments_metadata.sql

@@ -0,0 +1,15 @@
+{% macro run_bulk_get_topshot_moments_metadata() %}
+{% set sql %}
+
+
+select streamline.udf_bulk_get_topshot_moments_minted_metadata()
+where exists (
+    select 1
+    from streamline.all_topshot_moments_minted_metadata_needed
+    limit 1 
+)
+
+{% endset %}
+
+{% do run_query(sql) %}
+{% endmacro %}

macros/streamline/get_topshot_moments_metadata/udf_bulk_get_topshot_moments_minted_metadata.sql

@@ -1,8 +1,8 @@
 {% macro udf_bulk_get_topshot_moments_minted_metadata() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_topshot_moments_minted_metadata() returns text api_integration = aws_flow_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_topshot_moments_minted_metadata() returns text api_integration = aws_flow_api_dev AS {% if target.database == "FLOW" -%}
         'https://3ltti6kisi.execute-api.us-east-1.amazonaws.com/prod/bulk_get_topshot_moments_minted_metadata'
     {% else %}
         'https://wn6lmi2rs4.execute-api.us-east-1.amazonaws.com/dev/bulk_get_topshot_moments_minted_metadata'
     {%- endif %}
-{% endmacro %}
+{% endmacro %}

models/bronze/bronze__topshot_metadata.sql

@@ -11,5 +11,5 @@ SELECT
 FROM
     {{ source(
         'flow_external',
-        'topshot_moments_minted_metadata_api'
+        'moments_metadata_api'
     ) }}

models/sources.yml

@@ -25,10 +25,10 @@ sources:
     schema: bronze
     loader: S3
     tables:
-      - name: topshot_moments_minted_metadata_api
-        description: "External table of topshot moments minted metadata"
+      - name: moments_metadata_api
+        description: "External table of dapper labs moments metadata"
         external:
-          location: "@{{target.database}}.bronze.analytics_external_tables/{{target.database}}/MOMENTS_MINTED_METADATA_API"
+          location: "@{{target.database}}.bronze.analytics_external_tables/{{target.database}}/MOMENTS_MINTED_METADATA_API/"
           file_format: "( type = json, strip_outer_array = TRUE )"
           auto_refresh: true
         columns:

models/streamline/streamline__all_topshot_moments_minted_metadata_needed.sql

@@ -1,6 +1,5 @@
 {{ config(
     materialized = 'view',
-    post_hook = 'call silver.sp_bulk_get_topshot_moments_minted_metadata()'
 ) }}
 
 WITH mints AS (
@@ -47,7 +46,7 @@ SELECT
 FROM
     {{ source(
         'flow_external',
-        'topshot_moments_minted_metadata_api'
+        'moments_metadata_api'
     ) }}
-LIMIT
-    3000
+WHERE
+    contract = 'A.0b2a3299cc857e29.TopShot'

models/streamline/streamline__allday_moments_metadata_needed.sql

@@ -0,0 +1,45 @@
+{{ config(
+    materialized = 'view',
+) }}
+
+WITH mints AS (
+
+    SELECT
+        event_contract,
+        event_data :id :: STRING AS moment_id
+    FROM
+        {{ ref('silver__events_final') }}
+    WHERE
+        event_contract = 'A.e4cf4bdc1751c65d.AllDay'
+        AND event_type = 'MomentNFTMinted'
+),
+sales AS (
+    SELECT
+        nft_collection AS event_contract,
+        nft_id as moment_id
+    FROM
+        {{ ref('silver__nft_sales') }}
+    WHERE
+        nft_collection = 'A.e4cf4bdc1751c65d.AllDay'
+),
+all_day_ids AS (
+    SELECT
+        *
+    FROM
+        mints
+    UNION
+    SELECT
+        *
+    FROM
+        sales
+)
+SELECT
+    *
+FROM
+    all_day_ids
+EXCEPT 
+select 
+    contract as event_contract,
+    id AS moment_id
+from {{ source('flow_external', 'moments_metadata_api')}}
+where contract = 'A.e4cf4bdc1751c65d.AllDay'