add solsniper to nft sales (#505)

* add solsniper

* include additional events

* clean up v1 views

* update incremental logic

* left join to transfers

* remove null test

* update code style

* add solsniper relation test

* updates per comments

* select view in core

data/testing__nft_sales_solsniper.csv

@@ -0,0 +1,7 @@
+tx_id,purchaser,seller,mint,sales_amount
+4PvnivhKKYjwXmQrTHo2E3xAnaajNkLpFvH4ZpbJvEE7RsmgGZ7pmPwoBoSwbyAFfrkMNHHUDiLbmkzoz3FUPv41,6e3LxX21vhhPx9rkBQnZ6VFnf15NoXPCUuJMP9XrzZQy,A42sG3sKAYCWQJokBq5A1ZYdHjmzpr9whHvQ9gXKP8nW,5MKTQzs5ey4ZivtPgniYBE3WNPuzXBWzaXXbjC1R5amG,0.013
+3r2HhjQJSC2BbGrxR3QvAxWiEz5ypnbGmxtMkEfDvv2ZYs6177uMeCS4DZqKN4VBs4Kn2p2AGBNdtSDw2vXbMYgH,9rPC8qp7vzTVBbUA9Khh3Hiig4oQChRraEJEYXQMwrQg,3hbssdZztFHNxGdNhqmhUkn8HkRXv8E6UBS6qmRDbnKz,AkGDLvhV2SMZ5qe1JVoYnp8RGfgdYS1vwPsuysvUXxP1,0.1111
+2xBQgZdJ97wtAta6Ru7J2zn65YQJMc36nh8bDqAcuKA9x7EscBm5QDfo6g4DPBbSGGN67gGmFkLQZtTVkSZWHDb,2eB7soMqQdSmUVGxP7hyzAZaT3P3F2fjqZSqtgT5Yy4w,FNBb12zch5XdYXy2ahYHWopbmDat9HmQYdJTiuB9qXLF,5QuiyuGr5qDLGXeggan9gybirnRBMswweygBFMtjmMjP,0.1468825
+2Ebi5XwpsUhcoFveoenc9SSkCDibsyAqR2ZqxwUGefZq7cCf6ouhDdBWMWLssj2t69R6hmLJXg8sUjqzT2opXURo,26wmwTQnyG9mKgfRowUYoYXwZujhFJ6YjE5VB2HweyqG,Gkngxrgn3UemyyTBXoxgqHLuQk5TRFrnrAQJDZkYvAUZ,4i2boE1zgGU3gVDZxdNt76MK3NUy9Ua7QD9DJkC8TV4P,0.0385
+3Ybf7TeVmSAvfXGJTuTa7oZpd5ohHW7YgZ1AtS1LGjMKAS68rNUmAAJtD4nNYqGy7V5QPpyB2ZGNvALsDpEmS5JR,D2wko4M68sNgKK82UJkT2eA6U6YMHjsrWPHwCpgDEufU,7mPs4ApQNvMKB5GczGAfUL3SvkbkKB6WRmpJ8bvgKwsb,45u5M3VNFKdsk1vn9hrTbxP1jF2oMjGxPgBZxyKRraaF,2.436075
+R61rsLJ5EXfXW5kkcukjAUKTMeMrr9qZhwtmpcQfx9eXJe5CRPPKoW7wh7HSUnW2r34JQobMKvoPSzKD2JY41D9,FzdLkwyjQDhzMo4FPnXXjVLoJHH7TgZLu3Syyrv3mekE,F4T7B6jjNgRNudhA57WaQKna2aam626dZjNDbe1mMAy3,Gygeawu4KAZAeZQUhMBFXb2BnMezoUi3eS4tvbsSdGwU,0.2335825

models/gold/nft/nft__fact_nft_sales.sql

@@ -318,3 +318,37 @@ SELECT
     ) AS modified_timestamp
 FROM
     {{ ref('silver__nft_sales_tensorswap') }}
+UNION
+SELECT
+    'solsniper',
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    program_id,
+    purchaser,
+    seller,
+    mint,
+    sales_amount,
+    nft_sales_solsniper_id AS fact_nft_sales_id,
+    inserted_timestamp,
+     modified_timestamp
+FROM
+    {{ ref('silver__nft_sales_solsniper') }}
+UNION
+SELECT
+    'solsniper',
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    program_id,
+    purchaser,
+    seller,
+    mint,
+    sales_amount,
+    nft_sales_solsniper_id AS fact_nft_sales_id,
+    inserted_timestamp,
+    modified_timestamp,
+FROM
+    {{ ref('silver__nft_sales_solsniper_v1_events_view') }}

models/gold/nft/nft__fact_nft_sales.yml

@@ -1,7 +1,7 @@
 version: 2
 models:
   - name: nft__fact_nft_sales
-    description: NFT sales on Solana that occur on Magic Eden, Yawww, Opensea, the SMB marketplace, Solanart, Solport, Coral Cube, Hyperspace, Hadeswap, Exchange Art and Tensorswap.  
+    description: NFT sales on Solana that occur on Magic Eden, Yawww, Opensea, the SMB marketplace, Solanart, Solport, Coral Cube, Hyperspace, Hadeswap, Exchange Art, Tensorswap and Solsniper.  
     columns:
       - name: BLOCK_TIMESTAMP
         description: "{{ doc('block_timestamp') }}"

models/silver/nfts/silver__nft_sales_solsniper.sql

@@ -0,0 +1,148 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
+    unique_key = ['nft_sales_solsniper_id'],
+    cluster_by = ['block_timestamp::DATE'],
+    merge_exclude_columns = ["inserted_timestamp"],
+    tags = ['scheduled_non_core']
+) }}
+-- depends_on: {{ ref('silver__decoded_instructions_combined') }}
+/* run incremental timestamp value first then use it as a static value */
+{% if execute %}
+
+{% if is_incremental() %}
+{% set max_inserted_query %}
+
+SELECT
+    MAX(_inserted_timestamp) AS _inserted_timestamp
+FROM
+    {{ this }}
+
+    {% endset %}
+    {% set max_inserted_timestamp = run_query(max_inserted_query).columns [0].values() [0] %}
+{% endif %}
+
+{% set base_query %}
+CREATE OR REPLACE temporary TABLE silver.decoded_instructions_solsniper__intermediate_tmp AS
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    INDEX,
+    inner_index,
+    program_id,
+    decoded_instruction,
+    event_type,
+    _inserted_timestamp
+FROM
+    {{ ref('silver__decoded_instructions_combined') }}
+WHERE
+    program_id = 'SNPRohhBurQwrpwAptw1QYtpFdfEKitr4WSJ125cN1g'
+    AND event_type = 'executeSolNftOrder'
+
+{% if is_incremental() %}
+AND _inserted_timestamp >= '{{ max_inserted_timestamp }}'
+{% else %}
+    AND block_timestamp :: DATE >= '2023-05-02'
+{% endif %}
+
+{% endset %}
+{% do run_query(
+    base_query
+) %}
+{% set between_stmts = fsc_utils.dynamic_range_predicate("silver.decoded_instructions_solsniper__intermediate_tmp","block_timestamp::date"
+) %}
+{% endif %}
+
+WITH decoded AS (
+    SELECT
+        block_timestamp,
+        block_id,
+        tx_id,
+        INDEX,
+        inner_index,
+        program_id,
+        silver.udf_get_account_pubkey_by_name('buyer',decoded_instruction :accounts) AS buyer,
+        silver.udf_get_account_pubkey_by_name('seller',decoded_instruction :accounts) AS seller, -- main payment sent here
+        silver.udf_get_account_pubkey_by_name('treasury',decoded_instruction :accounts) AS treasury_fee_account, --fees sent here
+        silver.udf_get_account_pubkey_by_name('buyerEscrowVault',decoded_instruction :accounts) AS buyer_escrow_vault, -- where payment comes from
+        silver.udf_get_account_pubkey_by_name('sellNftMint',decoded_instruction :accounts) AS mint,
+        _inserted_timestamp
+    FROM
+        silver.decoded_instructions_solsniper__intermediate_tmp
+),
+transfers AS (
+    SELECT
+        A.*,
+        COALESCE(SPLIT_PART(INDEX :: text, '.', 1) :: INT, INDEX :: INT) AS index_1,
+        NULLIF(SPLIT_PART(INDEX :: text, '.', 2), '') :: INT AS inner_index_1
+    FROM
+        {{ ref('silver__transfers') }} A
+        INNER JOIN (
+            SELECT
+                DISTINCT tx_id
+            FROM
+                decoded
+        ) d
+        ON d.tx_id = A.tx_id
+    WHERE
+        A.succeeded
+        AND {{ between_stmts }}
+),
+pre_final AS (
+    SELECT
+        A.block_id,
+        A.block_timestamp,
+        A.program_id,
+        A.tx_id,
+        b.succeeded,
+        A.buyer AS purchaser,
+        A.seller,
+        C.amount AS fee_amt,
+        A.mint,
+        A._inserted_timestamp,
+        SUM(
+            b.amount
+        ) AS sale_amt,
+    FROM
+        decoded A
+        LEFT JOIN transfers b
+        ON A.tx_id = b.tx_id
+        AND A.buyer_escrow_vault = b.tx_from
+        AND A.seller = b.tx_to
+        AND A.index = b.index_1
+        LEFT JOIN transfers C
+        ON A.tx_id = C.tx_id
+        AND A.buyer_escrow_vault = C.tx_from
+        AND A.treasury_fee_account = C.tx_to
+        AND A.index = C.index_1
+    GROUP BY
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7,
+        8,
+        9,
+        10
+)
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    program_id,
+    purchaser,
+    seller,
+    mint,
+    sale_amt + fee_amt AS sales_amount,
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(['tx_id','mint']) }} AS nft_sales_solsniper_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS invocation_id
+FROM
+    pre_final
+

models/silver/nfts/silver__nft_sales_solsniper.yml

@@ -0,0 +1,65 @@
+version: 2
+models:
+  - name: silver__nft_sales_solsniper
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TX_ID
+            - MINT
+      - compare_model_subset:
+          name: silver__nft_sales_solsniper_business_logic_test
+          compare_model: ref('testing__nft_sales_solsniper')
+          compare_columns: 
+            - tx_id
+            - purchaser
+            - seller
+            - mint
+            - round(sales_amount,8)
+          model_condition: "where tx_id in ('4PvnivhKKYjwXmQrTHo2E3xAnaajNkLpFvH4ZpbJvEE7RsmgGZ7pmPwoBoSwbyAFfrkMNHHUDiLbmkzoz3FUPv41',
+            '3r2HhjQJSC2BbGrxR3QvAxWiEz5ypnbGmxtMkEfDvv2ZYs6177uMeCS4DZqKN4VBs4Kn2p2AGBNdtSDw2vXbMYgH',
+            '2xBQgZdJ97wtAta6Ru7J2zn65YQJMc36nh8bDqAcuKA9x7EscBm5QDfo6g4DPBbSGGN67gGmFkLQZtTVkSZWHDb',
+            '2Ebi5XwpsUhcoFveoenc9SSkCDibsyAqR2ZqxwUGefZq7cCf6ouhDdBWMWLssj2t69R6hmLJXg8sUjqzT2opXURo',
+            '3Ybf7TeVmSAvfXGJTuTa7oZpd5ohHW7YgZ1AtS1LGjMKAS68rNUmAAJtD4nNYqGy7V5QPpyB2ZGNvALsDpEmS5JR',
+            'R61rsLJ5EXfXW5kkcukjAUKTMeMrr9qZhwtmpcQfx9eXJe5CRPPKoW7wh7HSUnW2r34JQobMKvoPSzKD2JY41D9')"
+    columns:
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 2
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - not_null
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests:
+          - not_null
+      - name: SUCCEEDED
+        description: "{{ doc('tx_succeeded') }}"
+        tests:
+          - not_null
+      - name: PROGRAM_ID 
+        description: "{{ doc('program_id') }}"
+        tests: 
+          - not_null
+      - name: PURCHASER
+        description: "{{ doc('purchaser') }}"
+        tests: 
+          - not_null
+      - name: SELLER
+        description: "{{ doc('seller') }}"
+        tests: 
+          - not_null
+      - name: MINT
+        description: "{{ doc('mint') }}"
+        tests: 
+          - not_null
+      - name: SALES_AMOUNT
+        description: "{{ doc('sales_amount') }}"
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('_inserted_timestamp') }}"
+        tests: 
+          - not_null

models/silver/nfts/silver__nft_sales_solsniper_v1_events.sql

@@ -0,0 +1,193 @@
+{{ config(
+    materialized = 'table',
+    unique_key = ['nft_sales_solsniper_id'],
+    cluster_by = ['block_timestamp::DATE'],
+) }}
+
+WITH base AS (
+
+    SELECT
+        *
+    FROM
+        {{ ref('silver__decoded_instructions_combined') }}
+    WHERE
+        program_id = 'SNPRohhBurQwrpwAptw1QYtpFdfEKitr4WSJ125cN1g'
+        AND event_type IN (
+            'withdrawAndCloseBuyOrderLock',
+            'buyMeListingV1',
+            'buyMeMip1ListingV1',
+            'buyTensorSingleListingV1',
+            'buyTensorPoolListingV1'
+        )
+        AND block_timestamp :: DATE BETWEEN '2023-05-26'
+        AND '2024-01-30'
+),
+decoded AS (
+    SELECT
+        A.block_timestamp,
+        A.block_id,
+        A.tx_id,
+        A.index,
+        A.inner_index,
+        A.program_id,
+        A.event_type AS event_type_temp,
+        silver.udf_get_account_pubkey_by_name(
+            'buyer',
+            b.decoded_instruction :accounts
+        ) AS buyer,
+        CASE
+            WHEN A.event_type = 'buyMeListingV1' THEN silver.udf_get_account_pubkey_by_name(
+                'Remaining 0',
+                A.decoded_instruction :accounts
+            )
+            WHEN A.event_type = 'buyMeMip1ListingV1' THEN silver.udf_get_account_pubkey_by_name(
+                'Remaining 4',
+                A.decoded_instruction :accounts
+            )
+            WHEN A.event_type = 'buyTensorPoolListingV1' THEN silver.udf_get_account_pubkey_by_name(
+                'tswapPoolOwner',
+                A.decoded_instruction :accounts
+            )
+            WHEN A.event_type = 'buyTensorSingleListingV1' THEN silver.udf_get_account_pubkey_by_name(
+                'seller',
+                A.decoded_instruction :accounts
+            )
+        END AS seller,
+        CASE
+            WHEN A.event_type = 'buyMeListingV1' THEN silver.udf_get_account_pubkey_by_name(
+                'Remaining 5',
+                A.decoded_instruction :accounts
+            )
+            WHEN A.event_type = 'buyMeMip1ListingV1' THEN silver.udf_get_account_pubkey_by_name(
+                'Remaining 10',
+                A.decoded_instruction :accounts
+            )
+        END AS payment_acct,  -- sent here to distribute to seller and fee for ME
+        CASE
+            WHEN A.event_type = 'buyTensorPoolListingV1' THEN silver.udf_get_account_pubkey_by_name(
+                'tswapSolEscrow',
+                A.decoded_instruction :accounts
+            )
+            ELSE NULL
+        END AS tensor_sol_escrow,
+        silver.udf_get_account_pubkey_by_name(
+            'buyerEscrowVault',
+            A.decoded_instruction :accounts
+        ) AS buyer_escrow_vault, -- where payment comes from
+        silver.udf_get_account_pubkey_by_name(
+            'nftMint',
+            A.decoded_instruction :accounts
+        ) AS mint,
+        A._inserted_timestamp
+    FROM
+        base A
+        LEFT JOIN base b
+        ON A.tx_id = b.tx_id
+    WHERE
+        A.event_type IN (
+            'buyMeListingV1',
+            'buyMeMip1ListingV1',
+            'buyTensorSingleListingV1',
+            'buyTensorPoolListingV1'
+        )
+        AND b.event_type = 'withdrawAndCloseBuyOrderLock'
+),
+transfers AS (
+    SELECT
+        A.*,
+        COALESCE(SPLIT_PART(INDEX :: text, '.', 1) :: INT, INDEX :: INT) AS index_1,
+        NULLIF(SPLIT_PART(INDEX :: text, '.', 2), '') :: INT AS inner_index_1
+    FROM
+        {{ ref('silver__transfers') }} A
+        INNER JOIN (
+            SELECT
+                DISTINCT tx_id
+            FROM
+                decoded
+        ) d
+        ON d.tx_id = A.tx_id
+    WHERE
+        A.succeeded
+        AND A.block_timestamp :: DATE BETWEEN '2023-05-26'
+        AND '2024-01-30'
+),
+pre_final AS (
+    SELECT
+        A.block_id,
+        A.block_timestamp,
+        A.program_id,
+        A.tx_id,
+        b.succeeded,
+        A.buyer AS purchaser,
+        A.seller,
+        A.mint,
+        A._inserted_timestamp,
+        b.amount AS sales_amount
+    FROM
+        decoded A
+        INNER JOIN transfers b
+        ON A.tx_id = b.tx_id
+        AND A.buyer_escrow_vault = b.tx_from
+        AND A.payment_acct = b.tx_to
+        AND A.index = b.index_1
+        AND A.event_type_temp IN (
+            'buyMeListingV1',
+            'buyMeMip1ListingV1'
+        )
+    UNION ALL
+    SELECT
+        A.block_id,
+        A.block_timestamp,
+        A.program_id,
+        A.tx_id,
+        b.succeeded,
+        A.buyer AS purchaser,
+        A.seller,
+        A.mint,
+        A._inserted_timestamp,
+        SUM(
+            b.amount
+        ) AS sales_amount,
+    FROM
+        decoded A
+        INNER JOIN transfers b
+        ON A.tx_id = b.tx_id
+        AND A.buyer_escrow_vault = b.tx_from
+        AND (
+            A.seller = b.tx_to
+            OR A.tensor_sol_escrow = b.tx_to
+        )
+        AND A.index = b.index_1
+    WHERE
+        A.event_type_temp IN (
+            'buyTensorSingleListingV1',
+            'buyTensorPoolListingV1'
+        )
+    GROUP BY
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7,
+        8,
+        9
+)
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    program_id,
+    purchaser,
+    seller,
+    mint,
+    sales_amount,
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(['tx_id','mint']) }} AS nft_sales_solsniper_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS invocation_id
+FROM
+    pre_final

models/silver/nfts/silver__nft_sales_solsniper_v1_events_view.sql

@@ -0,0 +1,24 @@
+{{ config(
+  materialized = 'view'
+) }}
+
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    program_id,
+    purchaser,
+    seller,
+    mint,
+    sales_amount,
+    _inserted_timestamp,
+    nft_sales_solsniper_id,
+    inserted_timestamp,
+    modified_timestamp,
+    invocation_id
+FROM
+  {{ source(
+    'solana_silver',
+    'nft_sales_solsniper_v1_events'
+  ) }}

models/silver/parser/silver__decoded_instructions_combined.yml

@@ -17,6 +17,11 @@ models:
         description: "{{ doc('tx_id') }}"
         tests:
           - not_null
+          - dbt_utils.relationships_where:
+              to: ref('silver__nft_sales_solsniper')
+              field: tx_id
+              from_condition: "program_id = 'SNPRohhBurQwrpwAptw1QYtpFdfEKitr4WSJ125cN1g' and event_type = 'executeSolNftOrder' and _inserted_timestamp >= current_date - 7"
+              to_condition: "_inserted_timestamp >= current_date - 7"
       - name: SIGNERS
       - name: SUCCEEDED
       - name: INDEX

models/sources.yml

@@ -87,6 +87,7 @@ sources:
       - name: proposal_votes_marinade
       - name: stake_pool_actions_socean
       - name: _blocks_tx_count
+      - name: nft_sales_solsniper_v1_events
   - name: solana_streamline
     database: solana
     schema: streamline