add silver and gold models (#682)

models/gold/nft/nft__dim_nft_metadata.sql

@@ -1,7 +1,7 @@
 {{ config(
   materialized = 'view',
-  meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'NFT' }} },
-  tags = ['scheduled_non_core','exclude_change_tracking']
+  meta = { 'database_tags':{ 'table':{ 'PURPOSE': 'NFT' }} },
+  tags = ['scheduled_non_core', 'exclude_change_tracking']
 ) }}
 
 SELECT
@@ -19,6 +19,22 @@ SELECT
   A.modified_timestamp
 FROM
   {{ ref('silver__helius_nft_metadata') }} A
-  LEFT JOIN {{ ref('silver__nft_collection') }}
-  b
+LEFT JOIN 
+  {{ ref('silver__nft_collection') }} b
   ON A.nft_collection_id = b.nft_collection_id
+UNION ALL
+SELECT
+  mint,
+  NULL AS nft_collection_name, -- collection data pipe is currently broken so these will be null until it's fixed
+  NULL AS collection_id,
+  creators,
+  authority,
+  metadata,
+  image_url,
+  metadata_uri,
+  nft_name,
+  helius_cnft_metadata_id AS dim_nft_metadata_id,
+  inserted_timestamp,
+  modified_timestamp
+FROM
+  {{ ref('silver__helius_cnft_metadata') }}

models/silver/metadata/helius/silver__helius_cnft_metadata.sql

@@ -0,0 +1,63 @@
+  -- depends_on: {{ ref('bronze__streamline_helius_cnft_metadata') }}
+  
+{{ config(
+    materialized = 'incremental',
+    unique_key = "mint",
+    cluster_by = ['_inserted_timestamp::DATE'],
+    post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(mint,nft_name,nft_collection_id)'),
+    tags = ['scheduled_non_core']
+) }}
+
+{% if execute and is_incremental() %}
+    {% set max_inserted_timestamp_query %} 
+    SELECT
+        max(_inserted_timestamp)
+    FROM
+        {{ this }}
+    {% endset %}
+    {% set max_inserted_timestamp = run_query(max_inserted_timestamp_query)[0][0] %}
+{% endif %}
+
+WITH pre_final AS (
+    SELECT
+        mint,
+        data:authorities[0]:address::STRING AS authority,
+        data:creators::ARRAY AS creators,
+        data:content:metadata:attributes::ARRAY AS metadata,
+        data:content:links:image::STRING AS image_url,
+        data:content:json_uri::STRING AS metadata_uri,
+        data:content:metadata:name::STRING AS nft_name,
+        data:grouping[0]:group_key::STRING AS group_key,
+        data:grouping[0]:group_value::STRING AS group_value,
+        _inserted_timestamp
+    FROM
+        {% if is_incremental() %}
+        {{ ref('bronze__streamline_helius_cnft_metadata') }}
+        {% else %}
+        {{ ref('bronze__streamline_FR_helius_cnft_metadata') }}
+        {% endif %}
+    WHERE
+        mint IS NOT NULL
+        {% if is_incremental() %}
+        AND _inserted_timestamp >= '{{ max_inserted_timestamp }}'
+        {% endif %}
+    QUALIFY
+        row_number() OVER (PARTITION BY mint ORDER BY _inserted_timestamp DESC) = 1
+)
+SELECT
+    mint,
+    authority,
+    creators,
+    metadata,
+    image_url,
+    metadata_uri,
+    nft_name,
+    iff(group_key = 'collection', group_value, NULL) AS helius_collection_id,
+    iff(helius_collection_id IS NOT NULL, {{ dbt_utils.generate_surrogate_key(['helius_collection_id']) }}, NULL) AS nft_collection_id,
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(['mint']) }} AS helius_cnft_metadata_id,
+    sysdate() AS inserted_timestamp,
+    sysdate() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    pre_final

models/silver/metadata/helius/silver__helius_cnft_metadata.yml

@@ -0,0 +1,60 @@
+version: 2
+models:
+  - name: silver__helius_cnft_metadata
+    description: Contains cNFT metadata provided by Helius DAS API
+    recent_date_filter: &recent_date_filter
+      config:
+        where: _inserted_timestamp::date >= current_date - 7
+    columns:
+      - name: MINT
+        description: "{{ doc('mint') }}"
+        tests:
+          - not_null: *recent_date_filter
+          - unique
+      - name: CREATORS
+        description: "Creators of the NFT and what percentage of royalties they receive"
+      - name: AUTHORITY
+        description: "Authority address for the mint. When editions are minted, the authority remains the one from the master NFT"
+      - name: METADATA
+        description: "Metadata attributes that describe the NFT"
+      - name: IMAGE_URL
+        description: "{{ doc('image_url') }}"
+      - name: METADATA_URI
+        description: "{{ doc('token_metadata_uri') }}"
+      - name: NFT_NAME
+        description: "The name of the NFT"
+      - name: HELIUS_COLLECTION_ID
+        description: >
+          The ID of the collection from Helius
+      - name: NFT_COLLECTION_ID
+        description: >
+          The surrogate key of the Helius collection. NULL if helius_collection_id is NULL
+        tests:
+          - not_null:
+              config:
+                where: >
+                  helius_collection_id is not null
+                  AND _inserted_timestamp::date >= current_date - 7
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('_inserted_timestamp') }}"
+        tests: 
+          - not_null
+      - name: HELIUS_CNFT_METADATA_ID
+        description: '{{ doc("pk") }}'
+        tests: 
+          - not_null: *recent_date_filter
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+        tests: 
+          - not_null: *recent_date_filter
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 
+        tests: 
+          - not_null
+      - name: _INVOCATION_ID
+        description: '{{ doc("_invocation_id") }}' 
+        tests: 
+          - not_null:
+              name: test_silver__not_null_silver_helius_cnft_metadata_invocation_id
+              <<: *recent_date_filter
+