tweaks, documented in pr comment

models/descriptions/id.md

@@ -0,0 +1,5 @@
+{% docs id %}
+
+A unique identifier for the record.
+
+{% enddocs %}

models/descriptions/inserted_timestamp.md

@@ -1,5 +1,5 @@
 {% docs inserted_timestamp %}
 
-The timestamp of when the data was inserted into the database.
+The timestamp at which the record was initially created and inserted into this table.
 
 {% enddocs %}

models/descriptions/invocation_id.md

@@ -1,5 +1,5 @@
 {% docs invocation_id %}
 
-The unique identifier.
+A job ID to identify the run that last modified a record.
 
 {% enddocs %}

models/descriptions/modified_timestamp.md

@@ -1,5 +1,5 @@
 {% docs modified_timestamp %}
 
-The timestamp of when the data was last modified.
+The timestamp at which this record was last modified by an internal process.
 
 {% enddocs %}

models/descriptions/owner.md

@@ -0,0 +1,5 @@
+{% docs owner %}
+
+The owner of the token, which is the signer_id for minting events and the receiver_id for transfers.
+
+{% enddocs %}

models/silver/atlas/silver__atlas_nft_30_trailing.sql

@@ -1,31 +1,49 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = 'day',
+    unique_key = 'id',
     incremental_strategy = "merge",
-    merge_exclude_columns = ["inserted_timestamp"],    
+    merge_exclude_columns = ["inserted_timestamp"],
     tags = ['atlas']
 ) }}
 
 WITH date_range AS (
-    SELECT
-        date_day as day
-    FROM {{ ref('silver__dates') }}
-    WHERE
-        {% if is_incremental() %}
-            date_day >= SYSDATE() - INTERVAL '3 DAY'
-        {% else %}
-            date_day >= '2021-01-01' -- first day of data
-        {% endif %}
-    AND date_day <= SYSDATE()::DATE 
-)
 
+    SELECT
+        date_day AS DAY
+    FROM
+        {{ ref('silver__dates') }}
+    WHERE
+
+{% if is_incremental() %}
+date_day >= SYSDATE() - INTERVAL '3 DAY'
+{% else %}
+    date_day >= '2021-01-01' -- first day of data
+{% endif %}
+AND date_day <= SYSDATE() :: DATE
+),
+FINAL AS (
+    SELECT
+        d.day AS DAY,
+        COUNT(
+            t.tx_hash
+        ) AS txns
+    FROM
+        date_range d
+        LEFT JOIN {{ ref('silver__atlas_nft_transactions') }}
+        t
+        ON t.day BETWEEN d.day - INTERVAL '29 day'
+        AND d.day
+    GROUP BY
+        d.day
+)
 SELECT
-    d.day as day,
-    COUNT(t.tx_hash) AS txns,
-    SYSDATE() as inserted_timestamp,
-    SYSDATE() as modified_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['day']
+    ) }} AS id,
+    DAY,
+    txns,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
     '{{ invocation_id }}' AS invocation_id
-FROM date_range d
-LEFT JOIN {{ ref('silver__atlas_nft_transactions') }} t
-    ON t.day BETWEEN d.day - INTERVAL '29 day' AND d.day
-GROUP BY d.day
+FROM
+    FINAL

models/silver/atlas/silver__atlas_nft_30_trailing.yml

@@ -3,9 +3,15 @@ version: 2
 models:
   - name: silver__atlas_nft_30_trailing
     description: |-
-      This incremental dbt model generates a summary of NFT transactions from the 'silver__atlas_nft_transactions' table. It provides a daily count of transactions, accounting for a 29-day lookback period for each day within the specified date range.
+      This incremental dbt model generates a summary of NFT transactions from the 'silver__atlas_nft_transactions' table. It provides a daily count of transactions, accounting for a 30-day lookback period for each day within the specified date range.
 
     columns:
+      - name: id
+        description: "{{ doc('id')}}"
+        tests:
+          - not_null
+          - unique
+
       - name: day
         description: "{{ doc('date')}}"
         tests:

models/silver/atlas/silver__atlas_nft_detailed.sql

@@ -1,41 +1,48 @@
-
-
 {{ config(
-    materialized = 'incremental',
-    unique_key = 'action_id',
-    incremental_strategy = 'merge',
-    merge_exclude_columns = ["inserted_timestamp"],    
+    materialized = 'table',
+    unique_key = 'id',
     tags = ['atlas']
 ) }}
 
 WITH nft_data AS (
+
     SELECT
         *
-    FROM {{ ref('silver__atlas_nft_transactions') }}
-    WHERE
-        {% if var("MANUAL_FIX") %}
-            {{ partition_load_manual('no_buffer') }}
-        {% else %}
-            {{ incremental_last_x_days('_inserted_timestamp', 3) }}
-        {% endif %}
+    FROM
+        {{ ref('silver__atlas_nft_transactions') }}
 )
-
-select
-    concat_ws(
-        '-',
-        day,
-        receiver_id
-    ) AS action_id,
-    day,
+SELECT
+    {{ dbt_utils.generate_surrogate_key(
+        ['DAY', 'receiver_id']
+    ) }} AS id,
+    DAY,
     receiver_id,
-    COUNT(DISTINCT token_id) AS tokens,
-    COUNT(CASE WHEN method_name = 'nft_transfer' then tx_hash end) AS all_transfers,
-    COUNT(DISTINCT owner) AS owners,
+    COUNT(
+        DISTINCT token_id
+    ) AS tokens,
+    COUNT(
+        CASE
+            WHEN method_name = 'nft_transfer' THEN tx_hash
+        END
+    ) AS all_transfers,
+    COUNT(
+        DISTINCT owner
+    ) AS owners,
     COUNT(*) AS transactions,
-    COUNT(CASE WHEN method_name != 'nft_transfer' then tx_hash end) AS mints,
-    SYSDATE() as inserted_timestamp,
-    SYSDATE() as modified_timestamp,
-    '{{ invocation_id }}' AS invocation_id
-FROM nft_data
-GROUP BY 1, 2, 3
-ORDER BY 4 DESC
+    COUNT(
+        CASE
+            WHEN method_name != 'nft_transfer' THEN tx_hash
+        END
+    ) AS mints,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS invocation_id,
+    MAX(_inserted_timestamp) AS _inserted_timestamp
+FROM
+    nft_data
+GROUP BY
+    1,
+    2,
+    3
+ORDER BY
+    4 DESC

models/silver/atlas/silver__atlas_nft_detailed.yml

@@ -6,8 +6,8 @@ models:
       This is an incremental dbt model that gives an overview of NFT transactions in NEAR.
 
 columns:
-  - name: action_id
-    description: "A unique identifier for the action, constructed as a concatenation of the day and receiver_id."
+  - name: id
+    description: "{{ doc('id')}}"
     tests:
       - not_null
       - unique

models/silver/atlas/silver__atlas_nft_table.sql

@@ -1,32 +1,55 @@
-
-
 {{ config(
-    unique_key = 'receiver_id',
-    incremental_strategy = 'merge',
-    merge_exclude_columns = ["inserted_timestamp"],   
-    tags = ['atlas']
+  materialized = 'table',
+  unique_key = 'id',
+  tags = ['atlas']
 ) }}
 
 WITH nft_data AS (
-    SELECT
-        *
-    FROM {{ ref('silver__atlas_nft_transactions') }}
-)
 
-select
+  SELECT
+    *
+  FROM
+    {{ ref('silver__atlas_nft_transactions') }}
+)
+SELECT
+  {{ dbt_utils.generate_surrogate_key(
+    ['receiver_id']
+  ) }} AS id,
   receiver_id,
-  count(distinct token_id) as tokens,
-  count(case when method_name = 'nft_transfer' and day >= (SYSDATE()::DATE - interval '1 day')
-    then tx_hash end) as transfers_24h,
-  count(case when method_name = 'nft_transfer' and day >= ( SYSDATE()::DATE - interval '3 day') 
-    then tx_hash end) as transfers_3d,
-  count(case when method_name = 'nft_transfer' then tx_hash end) as all_transfers,
-  count(distinct owner) as owners,
-  count(*) as transactions,
-  count(case when method_name != 'nft_transfer' then tx_hash end) as mints,
-  SYSDATE() as inserted_timestamp,
-  SYSDATE() as modified_timestamp,
-  '{{ invocation_id }}' AS invocation_id
-from nft_data
-group by 1
-order by 3 desc
+  COUNT(
+    DISTINCT token_id
+  ) AS tokens,
+  COUNT(
+    CASE
+      WHEN method_name = 'nft_transfer'
+      AND DAY >= (SYSDATE() :: DATE - INTERVAL '1 day') THEN tx_hash END
+    ) AS transfers_24h,
+    COUNT(
+      CASE
+        WHEN method_name = 'nft_transfer'
+        AND DAY >= (SYSDATE() :: DATE - INTERVAL '3 day') THEN tx_hash END
+      ) AS transfers_3d,
+      COUNT(
+        CASE
+          WHEN method_name = 'nft_transfer' THEN tx_hash
+        END
+      ) AS all_transfers,
+      COUNT(
+        DISTINCT owner
+      ) AS owners,
+      COUNT(*) AS transactions,
+      COUNT(
+        CASE
+          WHEN method_name != 'nft_transfer' THEN tx_hash
+        END
+      ) AS mints,
+      SYSDATE() AS inserted_timestamp,
+      SYSDATE() AS modified_timestamp,
+      '{{ invocation_id }}' AS invocation_id
+      FROM
+        nft_data
+      GROUP BY
+        1,
+        2
+      ORDER BY
+        3 DESC

models/silver/atlas/silver__atlas_nft_table.yml

@@ -4,8 +4,14 @@ models:
   - name: silver__atlas_nft_table
     description: |-
       This view model provides a breakdown of NFT transaction activities by receiver_id. It includes counts of unique tokens, transfers within the last 24 hours and 3 days, all transfers, unique owners, total transactions, and minting events.
-
+      
     columns:
+      - name: id
+        description: "{ { doc('id')}}"
+        tests:
+          - unique
+          - not_null
+
       - name: receiver_id
         description: "{ { doc('receiver_id')}}"
         tests:

models/silver/atlas/silver__atlas_nft_transactions.sql

@@ -1,8 +1,8 @@
 {{ config(
     materialized = "incremental",
     cluster_by = ["day"],
-    unique_key = "action_id",
-    merge_exclude_columns = ["inserted_timestamp"],    
+    unique_key = "id",
+    merge_exclude_columns = ["inserted_timestamp"],
     incremental_strategy = "merge",
     tags = ['atlas']
 ) }}
@@ -10,57 +10,70 @@
 WITH nft_mints AS (
 
     SELECT
-        block_timestamp::date as day,
+        block_timestamp :: DATE AS DAY,
+        receipt_object_id,
         tx_hash,
         method_name,
         receiver_id,
         signer_id,
-        owner_id as owner,
+        owner_id AS owner,
         token_id,
-        _inserted_timestamp
-    FROM {{ ref('silver__standard_nft_mint_s3') }}
+        COALESCE(
+            _inserted_timestamp,
+            _load_timestamp
+        ) AS _inserted_timestamp
+    FROM
+        {{ ref('silver__standard_nft_mint_s3') }}
     WHERE
-            {% if var("MANUAL_FIX") %}
-                {{ partition_load_manual('no_buffer') }}
-            {% else %}
-                {{ incremental_load_filter('_inserted_timestamp') }}
-            {% endif %}
-),
-
-nft_transfers AS (
-
-    SELECT
-        block_timestamp::date as day,
-        tx_hash,
-        method_name,
-        receiver_id,
-        signer_id,
-        args['receiver_id'] as owner,
-        args['token_id'] as token_id,
-        _inserted_timestamp
-    FROM {{ ref('silver__actions_events_function_call_s3') }}
-    WHERE method_name = 'nft_transfer'
-    AND 
         {% if var("MANUAL_FIX") %}
             {{ partition_load_manual('no_buffer') }}
         {% else %}
             {{ incremental_load_filter('_inserted_timestamp') }}
         {% endif %}
 ),
-
-unioned_nft_data AS (
-    SELECT * FROM nft_mints
-    UNION ALL
-    SELECT * FROM nft_transfers
-)
-
-SELECT
-    concat_ws(
-        '-',
+nft_transfers AS (
+    SELECT
+        block_timestamp :: DATE AS DAY,
+        SPLIT(
+            action_id,
+            '-'
+        ) [0] :: STRING AS receipt_object_id,
         tx_hash,
-        method_name
-    ) AS action_id,
-    day,
+        method_name,
+        receiver_id,
+        signer_id,
+        args ['receiver_id'] AS owner,
+        args ['token_id'] AS token_id,
+        COALESCE(
+            _inserted_timestamp,
+            _load_timestamp
+        ) AS _inserted_timestamp
+    FROM
+        {{ ref('silver__actions_events_function_call_s3') }}
+    WHERE
+        method_name = 'nft_transfer'
+        AND {% if var("MANUAL_FIX") %}
+            {{ partition_load_manual('no_buffer') }}
+        {% else %}
+            {{ incremental_load_filter('_inserted_timestamp') }}
+        {% endif %}
+),
+unioned_nft_data AS (
+    SELECT
+        *
+    FROM
+        nft_mints
+    UNION ALL
+    SELECT
+        *
+    FROM
+        nft_transfers
+)
+SELECT
+    {{ dbt_utils.generate_surrogate_key(
+        ['receipt_object_id', 'method_name', 'token_id', 'owner']
+    ) }} AS id,
+    DAY,
     tx_hash,
     method_name,
     receiver_id,
@@ -70,5 +83,15 @@ SELECT
     SYSDATE() AS inserted_timestamp,
     SYSDATE() AS modified_timestamp,
     _inserted_timestamp,
-    '{{ invocation_id }}' AS _invocation_id
-FROM unioned_nft_data
+    '{{ invocation_id }}' AS invocation_id
+FROM
+    unioned_nft_data
+WHERE
+    -- failed receipts may have unparsable base64 FunctionCall args
+    token_id IS NOT NULL
+    AND owner IS NOT NULL 
+qualify ROW_NUMBER() over (
+    PARTITION BY id
+    ORDER BY
+        _inserted_timestamp DESC
+) = 1

models/silver/atlas/silver__atlas_nft_transactions.yml

@@ -4,10 +4,15 @@ models:
   - name: silver__atlas_nft_transactions
     description: |-
       This incremental dbt model unifies NFT minting and transfer data into a single view, providing a comprehensive look at NFT activities. It captures daily activities by transaction hash, method name, receiver ID, signer ID, owner, and token ID.
+    tests:
+      - dbt_utils.recency:
+          datepart: day
+          field: _inserted_timestamp
+          interval: 1
 
     columns:
-      - name: action_id
-        description: "{{doc('action_id')}}"
+      - name: id
+        description: "{{doc('id')}}"
         tests:
           - unique
           - not_null
@@ -16,7 +21,6 @@ models:
         description: "{{doc('date')}}"
         tests:
           - not_null
-          - dbt_expectations.expect_column_values_to_be_date
 
       - name: tx_hash
         description: "{{doc('tx_hash')}}"
@@ -24,7 +28,7 @@ models:
           - not_null
 
       - name: method_name
-        description: "The name of the method called in the transaction, indicating the type of action ('nft_transfer' or mint related)."
+        description: "{{doc('method_name')}}"
         tests:
           - not_null
 
@@ -39,7 +43,7 @@ models:
           - not_null
 
       - name: owner
-        description: "The owner of the token, which is the signer_id for minting events and the receiver_id for transfers."
+        description: "{{doc('owner')}}"
         tests:
           - not_null