An 5001/update amm sell (#616)

* wip

* add relationship test

* update ref

* add tests

* revert

* revert

* new model; convert old one to view; update gold

* add events

* update ME signers filter

* update relationship test

data/testing__nft_sales_amm_sell_decoded.csv

@@ -0,0 +1,9 @@
+tx_id,purchaser,seller,mint,sales_amount
+Ep7n5yUXoBiV6J3ntzvCqaGMS92NbQ9zdqS8ZMJVEwVGhKxsomKzGfsKzyHpzrb9s7ipLH4Cn5P88rneZEcm1Uy,6Y9SASCK9Vsa1T84qb6fY16HwkxmAk3mYyxbKs1UQ5cN,EVszcWthU6JidYCPgYc1y2uqoSajkJ3uepWPcK19336b,BwQKXrxKS9qDSvjmU4ZfFv8YnJM2jYXpSQA4yE6KmV6p,4.403131112
+59fgrVMFqY1scM8gumej3kCq6gzhQauso41QzB9sMx72MdH5dQ76Z7ZHcPM8gWyecCB3mriwbng8uPZ5zEKgmVN1,4GJALHbWj3y1MLGzo6MJz3hJyDwQph3nSgE54f9K9dYC,EvCoDUdLCv8D5CBfQwArvxgr2C8WRBosVDg4ZhxCHEpo,3rRg9RtT64msF5nxSUSdbnVR3mC149mM5jNW9VfZjStW,3.108971958
+46c3pCDa8hAuqiRN4SV3zwhuMz7XgDU6qrtsyzhkSXyJwjmeZ1TFsbxqnA15KCnPVLN1zNVUrouWJAskTX7v5uCF,AwQqQ1Xo9VxY64fcjw2toXZQnk94o5pFkxnGRngnqb1u,66Th151qye6WCmQ6nis5SzmBUZkVJ1uogCxMuxm6YLE,YjvHKGnS6vFNCBKtUNd1Jd2dQuovhNbY1iAbYhY22pC,12.617763101
+5cVLZm21xYLbhsi8zN2YJPacuKQjsaxcguJGZ2keP4vrGDKVJvazbvECSMaHoY5qVYWovVP53QC1scGX8rdTPUT,AwQqQ1Xo9VxY64fcjw2toXZQnk94o5pFkxnGRngnqb1u,1PKTvQ2yWWP6gXTq8EP73bWWdGmoyvMQZnQ26xKzZ2G,F3cv38qusZTfLchrNC3NFFxTKmYdRTHdk5dkymXJtYCP,31.09692544
+2yDoFY813NMMrFctpef8wxV4ktEt9Nn8kogJHSA6FBDdHpbezk6X21W2rbHK5tHLHsYBj7sdjNRDpHb9au2nEZT,AwQqQ1Xo9VxY64fcjw2toXZQnk94o5pFkxnGRngnqb1u,4AX2wYzLAaYir9pewgFmotegbPmD51eH3cjaqkTFHxok,CiyGDgU8cDHtu4Pm2otLr9SK8r3jH3xFfJTDYLQVYEXE,31.457249984
+4oyZC4YBe3gkfS4T5FBesosYh5RtUzLmzu4eY9GRmmJiaD3aPGiUAGCMSefsR65rt7kH1AdZgDKp5r9FhFi9XRUs,FpPPhKwQxY1L2M12o7JuyaSooD1J2CLzaD9BP22jyF1L,D8KemM7RsJDzys44HRfS3nJH8B8yJXdqb5Naqa4kECaU,7a1JC1yyVqC7Q57JNpja4cXvobxwx8CyfrA1wxmsjTja,0.049693
+QSbUdMWMi4qWzFoeojb7GLa9MU8TuFFWaRKZZvo69ZJVgPzYUpzi3kZ1HR62ZW4wGxRnzP1HYKZSyJ457g1HFav,CdyBZ5E6wJNYbMWpmTfsBn9iS2z2tp9ch37WTBm5Gj52,BKLBXijVbwJK47fJWHmWyz2VAUnWeZETYLwYUD1ddc3k,2QufyZxx8vrd8ztiuiM4CSRgVVz7eHCqzA1bav4SbdfL,3.861127675
+5RbhAmdeFoWDtRW4bN5FUXZvin2WMjjnbgfdUVw97QVo6mbowdBozKs2ruhZ5EvpKr2pUjHoTTqJdwGeQL2E2H5F,AGFCb7sETaMhvKuq6j3iiVmRLh7WtryAJFgnDGt6LTgN,ENL47paJQUHhtoyEowYTK1Q7kiNmqV2BkrbZJXBXGQMf,7r7ZQBFTngNKGkL84fQxJt7HCTjzWZGLnDVdcu469CQ3,0.276342858

models/gold/nft/nft__fact_nft_sales.sql

@@ -337,7 +337,30 @@ SELECT
         '2000-01-01'
     ) AS modified_timestamp
 FROM
-    {{ ref('silver__nft_sales_amm_sell') }}
+    {{ ref('silver__nft_sales_amm_sell_view') }}
+WHERE
+    block_timestamp::date < '2022-10-30'
+UNION ALL
+SELECT
+    marketplace,
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    program_id,
+    purchaser,
+    seller,
+    mint,
+    sales_amount,
+    NULL as tree_authority,
+    NULL as merkle_tree,
+    NULL as leaf_index,
+    FALSE as is_compressed,
+    nft_sales_amm_sell_decoded_id as fact_nft_sales_id,
+    inserted_timestamp,
+    modified_timestamp
+FROM
+    {{ ref('silver__nft_sales_amm_sell_decoded') }}
 UNION ALL
 SELECT
     'tensorswap',

models/silver/nfts/silver__nft_sales_amm_sell.sql

@@ -3,7 +3,8 @@
     unique_key = "CONCAT_WS('-', tx_id, mint)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE'],
-    tags = ['scheduled_non_core']
+    full_refresh = false,
+    enabled = false,
 ) }}
 
 WITH base_events AS (

models/silver/nfts/silver__nft_sales_amm_sell_decoded.sql

@@ -0,0 +1,311 @@
+-- depends_on: {{ ref('silver__decoded_instructions_combined') }}
+{{ config(
+    materialized = 'incremental',
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
+    merge_exclude_columns = ["inserted_timestamp"],
+    unique_key = "nft_sales_amm_sell_decoded_id",
+    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
+    tags = ['scheduled_non_core']
+) }}
+
+{% if execute %}
+    {% set base_query %}
+    CREATE OR REPLACE temporary TABLE silver.nft_sales_amm_sell_decoded__intermediate_tmp AS
+
+    SELECT
+        block_timestamp,
+        block_id,
+        tx_id,
+        succeeded,
+        signers,
+        INDEX,
+        inner_index,
+        program_id,
+        event_type,
+        decoded_instruction,
+        _inserted_timestamp
+    FROM
+        {{ ref('silver__decoded_instructions_combined') }}
+    WHERE
+        program_id = 'mmm3XBJg5gk8XJxEKBvdgptZz6SgK4tXvn36sodowMc'
+        AND event_type IN (
+            'solMip1FulfillSell',
+            'solFulfillSell',
+            'solFulfillBuy',
+            'solMip1FulfillBuy',
+            'solOcpFulfillBuy',
+            'solExtFulfillBuy',
+            'solOcpFulfillSell'
+        )
+        AND succeeded
+
+{% if is_incremental() %}
+AND _inserted_timestamp >= (
+    SELECT
+        MAX(_inserted_timestamp) - INTERVAL '1 hour'
+    FROM
+        {{ this }}
+)
+{% else %}
+    AND block_timestamp :: DATE >= '2022-10-30' -- refer to legacy model for sales before this date
+{% endif %}
+{% endset %}
+
+{% do run_query(base_query) %}
+{% set between_stmts = fsc_utils.dynamic_range_predicate(
+    "silver.nft_sales_amm_sell_decoded__intermediate_tmp",
+    "block_timestamp::date"
+) %}
+{% endif %}
+
+WITH base_decoded AS (
+    SELECT
+        *
+    FROM
+        silver.nft_sales_amm_sell_decoded__intermediate_tmp
+),
+base_transfers AS (
+    SELECT
+        *
+    FROM
+        {{ ref('silver__transfers') }}
+    WHERE
+        mint = 'So11111111111111111111111111111111111111111'
+        AND succeeded
+        AND {{ between_stmts }}
+),
+coral_cube_sales AS(
+    SELECT
+        block_timestamp,
+        block_id,
+        tx_id,
+        INDEX,
+        inner_index,
+        signers,
+        succeeded,
+        program_id,
+        decoded_instruction,
+        event_type,
+        CASE
+            WHEN event_type IN (
+                'solFulfillSell',
+                'solMip1FulfillSell',
+                'solOcpFulfillSell'
+            ) THEN silver.udf_get_account_pubkey_by_name(
+                'payer',
+                decoded_instruction :accounts
+            )
+            ELSE silver.udf_get_account_pubkey_by_name(
+                'pool',
+                decoded_instruction :accounts
+            )
+        END AS purchaser,
+        CASE
+            WHEN event_type IN (
+                'solFulfillSell',
+                'solMip1FulfillSell',
+                'solOcpFulfillSell'
+            ) THEN silver.udf_get_account_pubkey_by_name(
+                'pool',
+                decoded_instruction :accounts
+            )
+            ELSE silver.udf_get_account_pubkey_by_name(
+                'payer',
+                decoded_instruction :accounts
+            )
+        END AS seller,
+        CASE
+            WHEN event_type IN (
+                'solFulfillSell',
+                'solMip1FulfillSell',
+                'solOcpFulfillSell'
+            ) THEN 'sell'
+            ELSE 'buy'
+        END AS nft_sale_type,
+        'Coral Cube' AS marketplace,
+        silver.udf_get_account_pubkey_by_name(
+            'assetMint',
+            decoded_instruction :accounts
+        ) AS mint,
+        _inserted_timestamp
+    FROM
+        base_decoded
+    WHERE
+        signers [1] = '7RpRDUZBdu5hfmqWvobPazbNeVCagRk5E3Rb8Bm8qRmD'
+        AND purchaser <> '7RpRDUZBdu5hfmqWvobPazbNeVCagRk5E3Rb8Bm8qRmD'
+),
+mev2_sales AS(
+    SELECT
+        block_timestamp,
+        block_id,
+        tx_id,
+        INDEX,
+        inner_index,
+        signers,
+        succeeded,
+        program_id,
+        decoded_instruction,
+        event_type,
+        CASE
+            WHEN event_type IN (
+                'solMip1FulfillSell',
+                'solFulfillSell',
+                'solOcpFulfillSell'
+            ) THEN silver.udf_get_account_pubkey_by_name(
+                'payer',
+                decoded_instruction :accounts
+            )
+            ELSE silver.udf_get_account_pubkey_by_name(
+                'owner',
+                decoded_instruction :accounts
+            )
+        END AS purchaser,
+        CASE
+            WHEN event_type IN (
+                'solMip1FulfillSell',
+                'solFulfillSell',
+                'solOcpFulfillSell'
+            ) THEN silver.udf_get_account_pubkey_by_name(
+                'owner',
+                decoded_instruction :accounts
+            )
+            ELSE silver.udf_get_account_pubkey_by_name(
+                'payer',
+                decoded_instruction :accounts
+            )
+        END AS seller,
+        CASE
+            WHEN event_type IN (
+                'solMip1FulfillSell',
+                'solFulfillSell',
+                'solOcpFulfillSell'
+            ) THEN 'sell'
+            ELSE 'buy'
+        END AS nft_sale_type,
+        'Magic Eden' AS marketplace,
+        silver.udf_get_account_pubkey_by_name(
+            'assetMint',
+            decoded_instruction :accounts
+        ) AS mint,
+        _inserted_timestamp
+    FROM
+        base_decoded
+    WHERE
+        COALESCE(signers[1],signers[0])::STRING = 'NTYeYJ1wr4bpM5xo6zx5En44SvJFAd35zTxxNoERYqd'
+),
+coral_cube_nft_sale_amount AS (
+    SELECT
+        A.tx_id,
+        b.mint,
+        SUM(
+            b.amount
+        ) AS sales_amount
+    FROM
+        coral_cube_sales A
+        LEFT OUTER JOIN base_transfers b
+        ON A.tx_id = b.tx_id
+    WHERE
+        (
+            A.nft_sale_type = 'sell'
+            AND A.purchaser = b.tx_from
+        )
+        OR (
+            A.nft_sale_type = 'buy'
+            AND silver.udf_get_account_pubkey_by_name(
+                'buysideSolEscrowAccount',
+                A.decoded_instruction :accounts
+            ) :: STRING = b.tx_from
+        )
+    GROUP BY
+        1,
+        2
+),
+mev2_nft_sale_amount AS (
+    SELECT
+        A.tx_id,
+        b.mint,
+        SUM(
+            b.amount
+        ) AS sales_amount
+    FROM
+        mev2_sales A
+        LEFT OUTER JOIN base_transfers b
+        ON A.tx_id = b.tx_id
+    WHERE
+        (
+            A.nft_sale_type = 'sell'
+            AND A.purchaser = b.tx_from
+        )
+        OR (
+            A.nft_sale_type = 'buy'
+            AND silver.udf_get_account_pubkey_by_name(
+                'buysideSolEscrowAccount',
+                A.decoded_instruction :accounts
+            ) :: STRING = b.tx_from
+        )
+    GROUP BY
+        1,
+        2
+),
+pre_final AS (
+    SELECT
+        A.block_timestamp,
+        A.block_id,
+        A.tx_id,
+        A.succeeded,
+        A.index,
+        A.inner_index,
+        A.program_id,
+        A.mint,
+        A.purchaser,
+        A.seller,
+        b.sales_amount,
+        A.marketplace,
+        A._inserted_timestamp
+    FROM
+        coral_cube_sales A
+        LEFT JOIN coral_cube_nft_sale_amount b
+        ON A.tx_id = b.tx_id
+    WHERE
+        b.sales_amount IS NOT NULL
+    UNION
+    SELECT
+        A.block_timestamp,
+        A.block_id,
+        A.tx_id,
+        A.succeeded,
+        A.index,
+        A.inner_index,
+        A.program_id,
+        A.mint,
+        A.purchaser,
+        A.seller,
+        b.sales_amount,
+        A.marketplace,
+        A._inserted_timestamp
+    FROM
+        mev2_sales A
+        LEFT JOIN mev2_nft_sale_amount b
+        ON A.tx_id = b.tx_id
+)
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    index,
+    inner_index,
+    program_id,
+    mint,
+    purchaser,
+    seller,
+    sales_amount,
+    marketplace,
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(['tx_id','mint']) }} AS nft_sales_amm_sell_decoded_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    pre_final
+

models/silver/nfts/silver__nft_sales_amm_sell_decoded.yml

@@ -0,0 +1,95 @@
+version: 2
+models:
+  - name: silver__nft_sales_amm_sell_decoded
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TX_ID
+            - MINT
+          where: block_timestamp::date > current_date - 30
+      - compare_model_subset:
+          name: silver__nft_sales_amm_sell_decoded_business_logic_test
+          compare_model: ref('testing__nft_sales_amm_sell_decoded')
+          compare_columns: 
+            - tx_id
+            - purchaser
+            - seller
+            - mint
+            - round(sales_amount,8)
+          model_condition: "where tx_id in ('Ep7n5yUXoBiV6J3ntzvCqaGMS92NbQ9zdqS8ZMJVEwVGhKxsomKzGfsKzyHpzrb9s7ipLH4Cn5P88rneZEcm1Uy',
+            '59fgrVMFqY1scM8gumej3kCq6gzhQauso41QzB9sMx72MdH5dQ76Z7ZHcPM8gWyecCB3mriwbng8uPZ5zEKgmVN1',
+            '2yDoFY813NMMrFctpef8wxV4ktEt9Nn8kogJHSA6FBDdHpbezk6X21W2rbHK5tHLHsYBj7sdjNRDpHb9au2nEZT',
+            '5cVLZm21xYLbhsi8zN2YJPacuKQjsaxcguJGZ2keP4vrGDKVJvazbvECSMaHoY5qVYWovVP53QC1scGX8rdTPUT',
+            '46c3pCDa8hAuqiRN4SV3zwhuMz7XgDU6qrtsyzhkSXyJwjmeZ1TFsbxqnA15KCnPVLN1zNVUrouWJAskTX7v5uCF',
+            '4oyZC4YBe3gkfS4T5FBesosYh5RtUzLmzu4eY9GRmmJiaD3aPGiUAGCMSefsR65rt7kH1AdZgDKp5r9FhFi9XRUs',
+            'QSbUdMWMi4qWzFoeojb7GLa9MU8TuFFWaRKZZvo69ZJVgPzYUpzi3kZ1HR62ZW4wGxRnzP1HYKZSyJ457g1HFav',
+            '5RbhAmdeFoWDtRW4bN5FUXZvin2WMjjnbgfdUVw97QVo6mbowdBozKs2ruhZ5EvpKr2pUjHoTTqJdwGeQL2E2H5F')"
+    recent_date_filter: &recent_date_filter
+      config:
+        where: _inserted_timestamp >= current_date - 7
+    columns:
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 2
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - not_null: *recent_date_filter
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests:
+          - not_null: *recent_date_filter
+      - name: SUCCEEDED
+        description: "{{ doc('tx_succeeded') }}"
+        tests:
+          - not_null: *recent_date_filter
+      - name: PROGRAM_ID 
+        description: "{{ doc('program_id') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: PURCHASER
+        description: "{{ doc('purchaser') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: SELLER
+        description: "{{ doc('seller') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: MINT
+        description: "{{ doc('mint') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: SALES_AMOUNT
+        description: "{{ doc('sales_amount') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: MARKETPLACE
+        description: "{{ doc('marketplace') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('_inserted_timestamp') }}"
+        tests: 
+          - not_null
+      - name: NFT_SALES_AMM_SELL_DECODED_ID
+        description: '{{ doc("pk") }}'   
+        tests: 
+          - unique: *recent_date_filter
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+        tests: 
+          - not_null: *recent_date_filter
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 
+        tests: 
+          - not_null: *recent_date_filter
+      - name: _INVOCATION_ID
+        description: '{{ doc("_invocation_id") }}' 
+        tests: 
+          - not_null: 
+              name: test_silver__not_null_nft_sales_amm_sell_decoded__invocation_id
+              <<: *recent_date_filter

models/silver/nfts/silver__nft_sales_amm_sell_view.sql

@@ -0,0 +1,25 @@
+{{ config(
+  materialized = 'view'
+) }}
+
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    program_id,
+    mint,
+    purchaser,
+    seller,
+    sales_amount,
+    marketplace,
+    _inserted_timestamp,
+    nft_sales_amm_sell_id,
+    inserted_timestamp,
+    modified_timestamp,
+    _invocation_id
+FROM
+  {{ source(
+    'solana_silver',
+    'nft_sales_amm_sell'
+  ) }}

models/silver/parser/silver__decoded_instructions_combined.yml

@@ -166,6 +166,16 @@ models:
                 AND succeeded
                 and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR'
               to_condition: "_inserted_timestamp >= current_date - 7"
+          - dbt_utils.relationships_where:
+              name: dbt_utils_relationships_where_silver__decoded_instructions_combined_nft_sales_amm_sell_decoded_tx_id
+              to: ref('silver__nft_sales_amm_sell_decoded')
+              field: tx_id
+              from_condition: >
+                program_id = 'mmm3XBJg5gk8XJxEKBvdgptZz6SgK4tXvn36sodowMc'
+                AND event_type IN ('solMip1FulfillSell','solFulfillSell','solFulfillBuy','solMip1FulfillBuy','solOcpFulfillBuy','solExtFulfillBuy')
+                AND succeeded
+                and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR'
+              to_condition: "_inserted_timestamp >= current_date - 7"
       - name: SIGNERS
         description: "{{ doc('signers') }}"
       - name: SUCCEEDED

models/sources.yml

@@ -84,6 +84,7 @@ sources:
       - name: _blocks_tx_count
       - name: nft_sales_solsniper_v1_events
       - name: swaps_intermediate_jupiterv6
+      - name: nft_sales_amm_sell
   - name: solana_streamline
     database: solana
     schema: streamline