An 4638/cnft sales (#536)

* tensor swap models

* update code style

* fix

* update tensorswap cnft sales

* update per pr comments

* update logic for mint and transfers, add relationship test

* add relationship test

data/testing__nft_sales_tensorswap_cnft.csv

@@ -0,0 +1,4 @@
+tx_id,purchaser,seller,mint,sales_amount
+2T4LVBT6wUiW3VbTBoMg6ry8BmT7GzrhQeeaKUKjnn9XRzgGNrEzYviTUv1FQgW4FCoN2r9o7sNxQeyqLb52AjPj,FYz5ZqtKkRGAFaodArkcUwmtGDhdbowgFU4q7SXc3Mie,GxwR9kYzDJDmxqbsPhnGBwUvK4i29EcMbEm2tomNcDLj,2L2iNR4q4efKsPs58udXfvnNA1SHeWNvbroU1GMNvYLB,4.892198498
+3km8gnunyq2qTuPongzqwGdWGKGDEXTmbnusYbAAieUSwNpxBprQRM6XkzGhgbWkn6sBj3TLcBZHk8ihRfYgk57N,8uTruHG5hyhezmiNPVP4E4KRhZZPjadpSV7TLLqfMiGH,7fFCQtH44BazcoQCFGveQUBawfmu8ZsEJoacxFsBqYxy,2mFHWsqZSWfXUDPxDvCJdwm1NW2uH1uecc4FcSnnRkzC,7.55548
+3asER9bSgki7Xod7m6d7bsYg3h4h8QaLsCdWdV4eA4QznndjEULtfQDVSebcAgsCsCjJHpmn7e4sPWFdtRD33p9d,4LcLBr9q7SuVvEeFCRtEdetdcHVP47jWL9HafGo4sdMp,GxkZwiNkML9qQfHV44jug8T1sRrNw8iEPr979FZvThnu,4u4HDDr5A9JsaK6728s9k9cTx3g2iTHjqDSkrbBbS9eQ,5.42

models/descriptions/is_compressed.md

@@ -0,0 +1,5 @@
+{% docs is_compressed %}
+
+Identifies if the NFT is a compressed NFT
+
+{% enddocs %}

models/descriptions/leaf_index.md

@@ -0,0 +1,5 @@
+{% docs leaf_index %}
+
+The position of the leaf within the merkle tree used to locate and verify the leaf's data
+
+{% enddocs %}

models/descriptions/merkle_tree.md

@@ -0,0 +1,5 @@
+{% docs merkle_tree %}
+
+The address of the merkle tree which contains the cNFT
+
+{% enddocs %}

models/descriptions/tree_authority.md

@@ -0,0 +1,5 @@
+{% docs tree_authority %}
+
+The address that has authority to manage the merkle tree, inclding adding new leaves or updating the tree structure
+
+{% enddocs %}

models/gold/nft/nft__fact_nft_sales.sql

@@ -15,6 +15,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     {{ dbt_utils.generate_surrogate_key(
         ['tx_id', 'mint']
     ) }} AS fact_nft_sales_id,
@@ -37,6 +41,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     COALESCE (
         nft_sales_magic_eden_v2_id,
         {{ dbt_utils.generate_surrogate_key(
@@ -65,6 +73,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     COALESCE (
         nft_sales_solanart_id,
         {{ dbt_utils.generate_surrogate_key(
@@ -93,6 +105,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     {{ dbt_utils.generate_surrogate_key(
         ['tx_id']
     ) }} AS fact_nft_sales_id,
@@ -112,6 +128,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     {{ dbt_utils.generate_surrogate_key(
         ['tx_id', 'mint']
     ) }} AS fact_nft_sales_id,
@@ -133,6 +153,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     {{ dbt_utils.generate_surrogate_key(
         ['tx_id', 'mint']
     ) }} AS fact_nft_sales_id,
@@ -152,6 +176,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     {{ dbt_utils.generate_surrogate_key(
         ['tx_id']
     ) }} AS fact_nft_sales_id,
@@ -171,6 +199,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     COALESCE (
         nft_sales_hadeswap_id,
         {{ dbt_utils.generate_surrogate_key(
@@ -199,6 +231,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     COALESCE (
         nft_sales_hyperspace_id,
         {{ dbt_utils.generate_surrogate_key(
@@ -227,6 +263,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     {{ dbt_utils.generate_surrogate_key(
         ['tx_id','mint']
     ) }} AS fact_nft_sales_id,
@@ -246,6 +286,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     COALESCE (
         nft_sales_exchange_art_id,
         {{ dbt_utils.generate_surrogate_key(
@@ -274,6 +318,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     COALESCE (
         nft_sales_amm_sell_id,
         {{ dbt_utils.generate_surrogate_key(
@@ -302,6 +350,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     COALESCE (
         nft_sales_tensorswap_id,
         {{ dbt_utils.generate_surrogate_key(
@@ -330,6 +382,10 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     nft_sales_solsniper_id AS fact_nft_sales_id,
     inserted_timestamp,
      modified_timestamp
@@ -347,8 +403,33 @@ SELECT
     seller,
     mint,
     sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
     nft_sales_solsniper_id AS fact_nft_sales_id,
     inserted_timestamp,
     modified_timestamp,
 FROM
     {{ ref('silver__nft_sales_solsniper_v1_events_view') }}
+UNION
+SELECT
+    'tensorswap',
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    program_id,
+    purchaser,
+    seller,
+    mint,
+    sales_amount,
+    tree_authority,
+    merkle_tree,
+    leaf_index,
+    TRUE as is_compressed,
+    nft_sales_tensorswap_cnft_id AS fact_nft_sales_id,
+    inserted_timestamp,
+    modified_timestamp,
+FROM
+    {{ ref('silver__nft_sales_tensorswap_cnft') }}

models/gold/nft/nft__fact_nft_sales.yml

@@ -43,6 +43,22 @@ models:
         description: "{{ doc('marketplace') }}"
         tests:
           - dbt_expectations.expect_column_to_exist
+      - name: TREE_AUTHORITY
+        description: "{{ doc('tree_authority') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: MERKLE_TREE
+        description: "{{ doc('merkle_tree') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: LEAF_INDEX
+        description: "{{ doc('leaf_index') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: IS_COMPRESSED
+        description: "{{ doc('is_compressed') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
       - name: FACT_NFT_SALES_ID
         description: '{{ doc("pk") }}'   
       - name: INSERTED_TIMESTAMP

models/silver/nfts/silver__nft_sales_tensorswap_cnft.sql

@@ -0,0 +1,159 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
+    unique_key = ['nft_sales_tensorswap_cnft_id'],
+    cluster_by = ['block_timestamp::DATE'],
+    merge_exclude_columns = ["inserted_timestamp"],
+    tags = ['scheduled_non_core']
+) }}
+
+-- depends_on: {{ ref('silver__decoded_instructions_combined') }}
+/* run incremental timestamp value first then use it as a static value */
+{% if execute %}
+
+{% if is_incremental() %}
+{% set max_inserted_query %}
+
+SELECT
+    MAX(_inserted_timestamp) AS _inserted_timestamp
+FROM
+    {{ this }}
+
+    {% endset %}
+    {% set max_inserted_timestamp = run_query(max_inserted_query).columns [0].values() [0] %}
+{% endif %}
+
+{% set base_query %}
+CREATE OR REPLACE temporary TABLE silver.decoded_instructions_tensorswap__intermediate_tmp AS
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    INDEX,
+    inner_index,
+    program_id,
+    decoded_instruction,
+    event_type,
+    _inserted_timestamp
+FROM
+    {{ ref('silver__decoded_instructions_combined') }}
+WHERE
+    program_id = 'TCMPhJdwDryooaGtiocG1u3xcYbRpiJzb283XfCZsDp'
+    AND event_type in ('tcompNoop', 'buy')
+
+{% if is_incremental() %}
+AND _inserted_timestamp >= '{{ max_inserted_timestamp }}'
+{% else %}
+    AND block_timestamp :: DATE >= '2023-05-16'
+{% endif %}
+
+{% endset %}
+{% do run_query(
+    base_query
+) %}
+
+{% set between_stmts = fsc_utils.dynamic_range_predicate("silver.decoded_instructions_tensorswap__intermediate_tmp","block_timestamp::date") %}
+{% endif %}
+
+with decoded_mints AS (
+    SELECT
+        tx_id,
+        INDEX,
+        COALESCE(
+        decoded_instruction :args :event :taker :tupleData :"0" :assetId :: STRING,
+        decoded_instruction :args :event :taker :tupleData :"0" :targetId :: STRING) AS mint
+    FROM
+        silver.decoded_instructions_tensorswap__intermediate_tmp
+    WHERE
+        event_type = 'tcompNoop'
+),
+decoded AS (
+    SELECT
+        A.block_timestamp,
+        A.block_id,
+        A.tx_id,
+        A.index,
+        A.inner_index,
+        A.program_id,
+        silver.udf_get_account_pubkey_by_name('payer',A.decoded_instruction :accounts) AS purchaser,
+        silver.udf_get_account_pubkey_by_name('owner',A.decoded_instruction :accounts) AS seller,
+        silver.udf_get_account_pubkey_by_name('treeAuthority',a.decoded_instruction :accounts) AS tree_authority,
+        silver.udf_get_account_pubkey_by_name('merkleTree',a.decoded_instruction :accounts) AS merkle_tree,
+        decoded_instruction:args:index::int as leaf_index,
+        b.mint,
+        _inserted_timestamp
+    FROM
+        silver.decoded_instructions_tensorswap__intermediate_tmp A
+        INNER JOIN decoded_mints b
+        ON A.tx_id = b.tx_id
+        AND A.index = b.index
+    WHERE
+        A.event_type = 'buy'
+),
+transfers AS (
+    SELECT
+        A.block_timestamp,
+        A.tx_id,
+        A.tx_from,
+        A.succeeded,
+        COALESCE(SPLIT_PART(INDEX :: text, '.', 1) :: INT, INDEX :: INT) AS index_1,
+        SUM(A.amount) as sales_amount
+    FROM
+        {{ ref('silver__transfers') }} A
+        INNER JOIN (
+            SELECT
+                tx_id,
+                block_timestamp::date as dt
+            FROM
+                decoded
+        ) d
+        ON d.dt = a.block_timestamp::DATE
+            AND d.tx_id = A.tx_id
+    WHERE
+        A.succeeded
+        and {{ between_stmts }}
+    group by 1,2,3,4,5
+),
+pre_final as (
+SELECT
+    A.block_id,
+    A.block_timestamp,
+    A.program_id,
+    A.tx_id,
+    b.succeeded,
+    A.purchaser,
+    A.seller,
+    A.mint,
+    a.tree_authority,
+    a.merkle_tree,
+    a.leaf_index,
+    A._inserted_timestamp,
+    b.sales_amount
+FROM
+    decoded A
+    LEFT JOIN transfers b
+    ON A.tx_id = b.tx_id
+    AND A.purchaser = b.tx_from
+    AND A.index = b.index_1
+)
+
+SELECT 
+    block_id,
+    block_timestamp,
+    program_id,
+    tx_id,
+    succeeded,
+    purchaser,
+    seller,
+    mint,
+    tree_authority,
+    merkle_tree,
+    leaf_index,
+    _inserted_timestamp,
+    sales_amount,
+  {{ dbt_utils.generate_surrogate_key(['tx_id','mint','purchaser']) }} as nft_sales_tensorswap_cnft_id,
+    sysdate() as inserted_timestamp,
+    sysdate() as modified_timestamp,
+    '{{ invocation_id }}' AS invocation_id
+FROM
+    pre_final

models/silver/nfts/silver__nft_sales_tensorswap_cnft.yml

@@ -0,0 +1,76 @@
+version: 2
+models:
+  - name: silver__nft_sales_tensorswap_cnft
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TX_ID
+            - MINT
+      - compare_model_subset:
+          name: silver__nft_sales_tensorswap_cnft_business_logic_test
+          compare_model: ref('testing__nft_sales_tensorswap_cnft')
+          compare_columns: 
+            - tx_id
+            - purchaser
+            - seller
+            - mint
+            - round(sales_amount,8)
+          model_condition: "where tx_id in ('2T4LVBT6wUiW3VbTBoMg6ry8BmT7GzrhQeeaKUKjnn9XRzgGNrEzYviTUv1FQgW4FCoN2r9o7sNxQeyqLb52AjPj',
+            '3km8gnunyq2qTuPongzqwGdWGKGDEXTmbnusYbAAieUSwNpxBprQRM6XkzGhgbWkn6sBj3TLcBZHk8ihRfYgk57N',
+            '3asER9bSgki7Xod7m6d7bsYg3h4h8QaLsCdWdV4eA4QznndjEULtfQDVSebcAgsCsCjJHpmn7e4sPWFdtRD33p9d')"
+    columns:
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 2
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - not_null
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests:
+          - not_null
+      - name: SUCCEEDED
+        description: "{{ doc('tx_succeeded') }}"
+        tests:
+          - not_null
+      - name: PROGRAM_ID 
+        description: "{{ doc('program_id') }}"
+        tests: 
+          - not_null
+      - name: PURCHASER
+        description: "{{ doc('purchaser') }}"
+        tests: 
+          - not_null
+      - name: SELLER
+        description: "{{ doc('seller') }}"
+        tests: 
+          - not_null
+      - name: MINT
+        description: "{{ doc('mint') }}"
+        tests: 
+          - not_null
+      - name: SALES_AMOUNT
+        description: "{{ doc('sales_amount') }}"
+        tests: 
+          - not_null
+      - name: TREE_AUTHORITY
+        description: "{{ doc('tree_authority') }}"
+        tests: 
+          - not_null
+      - name: MERKLE_TREE
+        description: "{{ doc('merkle_tree') }}"
+        tests: 
+          - not_null
+      - name: LEAF_INDEX
+        description: "{{ doc('leaf_index') }}"
+        tests: 
+          - not_null
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('_inserted_timestamp') }}"
+        tests: 
+          - not_null

models/silver/parser/silver__decoded_instructions_combined.yml

@@ -18,10 +18,17 @@ models:
         tests:
           - not_null
           - dbt_utils.relationships_where:
+              name: dbt_utils_relationships_where_silver__decoded_instructions_combined_nft_sales_solsniper_tx_id
               to: ref('silver__nft_sales_solsniper')
               field: tx_id
               from_condition: "program_id = 'SNPRohhBurQwrpwAptw1QYtpFdfEKitr4WSJ125cN1g' and event_type = 'executeSolNftOrder' and _inserted_timestamp >= current_date - 7"
               to_condition: "_inserted_timestamp >= current_date - 7"
+          - dbt_utils.relationships_where:
+              name: dbt_utils_relationships_where_silver__decoded_instructions_combined_nft_sales_tensorswap_cnft_tx_id
+              to: ref('silver__nft_sales_tensorswap_cnft')
+              field: tx_id
+              from_condition: "program_id = 'TCMPhJdwDryooaGtiocG1u3xcYbRpiJzb283XfCZsDp' and event_type = 'buy' and _inserted_timestamp >= current_date - 7"
+              to_condition: "_inserted_timestamp >= current_date - 7"
       - name: SIGNERS
       - name: SUCCEEDED
       - name: INDEX