Hot fix/nft price exclude burns (#229)

* add test cases

* filter out burn actions

* fix test

* adjust incremental date

* add test cases

* remove null rows

* join post_token_bal using tx_id

* fix distinct logic

* revert distinct

* exclude models from incremental

.github/workflows/dbt_run_incremental.yml

@@ -44,6 +44,6 @@ jobs:
           dbt run-operation run_sp_refresh_external_tables_full
           dbt run -s models/silver/silver__transactions.sql models/silver/silver__blocks.sql models/silver/silver__votes.sql models/silver/silver___inner_instructions.sql models/silver/silver___instructions.sql models/silver/silver__events.sql models/silver/silver___all_undecoded_instructions_data.sql
           dbt run-operation run_sp_refresh_external_tables_full
-          dbt run -s ./models --exclude models/core models/silver/silver__transactions.sql models/silver/silver__blocks.sql models/silver/silver__votes.sql models/silver/silver___inner_instructions.sql models/silver/silver___instructions.sql models/silver/silver__events.sql models/silver/silver___all_undecoded_instructions_data.sql tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql
+          dbt run -s ./models --exclude models/core models/silver/silver__transactions.sql models/silver/silver__blocks.sql models/silver/silver__votes.sql models/silver/silver___inner_instructions.sql models/silver/silver___instructions.sql models/silver/silver__events.sql models/silver/silver___all_undecoded_instructions_data.sql tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql models/silver/nfts/silver__nft_mint_price_generic.sql+
           dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s ./models/core --exclude models/core/core__ez_signers.sql
           

data/testing__nft_mint_price.csv

@@ -7,4 +7,10 @@ metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,8yGGqtYud82xsUF6LYQtbsXjuXcF5JrbZmpY
 metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,GwQ1d3oJDvAyhTTBUeUX9JW5dhjqYxYqzaG2HNkyYcWd,5Awi9epmw6EYXzLnzkSu7qJKiZ4HnnDsJ3ix9Wxrt6DC,So11111111111111111111111111111111111111111,0.013683361
 metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,GwQ1d3oJDvAyhTTBUeUX9JW5dhjqYxYqzaG2HNkyYcWd,5Awi9epmw6EYXzLnzkSu7qJKiZ4HnnDsJ3ix9Wxrt6DC,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,16250000
 metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,2jPtkhNbYP55Brb9bBPVwaRPFnMZHsu2YEeHHybFxwEb,AXJFRmLT11RnuFEgYC35F9tktdL77xX8y1b4HEbXPJkf,So11111111111111111111111111111111111111111,0.013683361
-metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,2jPtkhNbYP55Brb9bBPVwaRPFnMZHsu2YEeHHybFxwEb,AXJFRmLT11RnuFEgYC35F9tktdL77xX8y1b4HEbXPJkf,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,16250000
+metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,2jPtkhNbYP55Brb9bBPVwaRPFnMZHsu2YEeHHybFxwEb,AXJFRmLT11RnuFEgYC35F9tktdL77xX8y1b4HEbXPJkf,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,16250000
+metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,86teyjAD1WjMTmw68wHu6LTWd6nvNcYkHRAQoqqJsdyA,3SQcD75fEJsDPgGCPsqpbD4N2XyDHCCcLDB26UiUU6wa,GENEtH5amGSi8kHAtQoezp1XEXwZJ8vcuePYnXdKrMYz,5
+metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,86teyjAD1WjMTmw68wHu6LTWd6nvNcYkHRAQoqqJsdyA,3SQcD75fEJsDPgGCPsqpbD4N2XyDHCCcLDB26UiUU6wa,So11111111111111111111111111111111111111111,0.020942641
+metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,5d3TUi4oSB2NMazfcWzKXgeinJKtiJKAUEbsFUbmnZnq,7onpWcfER4oz1KLPJMB6mRCXx9EXvDAWsoy4ZnVCkWun,GkpbHQu2zYmJxyp93p9wTX3uHjsFt8ZGeomVwZkGwXLH,5
+metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,5d3TUi4oSB2NMazfcWzKXgeinJKtiJKAUEbsFUbmnZnq,7onpWcfER4oz1KLPJMB6mRCXx9EXvDAWsoy4ZnVCkWun,So11111111111111111111111111111111111111111,0.020942641
+metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,9xmSt5gqCJ1yfcFJicrdkzTi6ujcQixDN8hXTZqs4kDH,SnFYS4FwpRyUziz9fQ68v5A2aGKqRDaNAv5AVaMPQja,So11111111111111111111111111111111111111111,0.011456161
+metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s,J9UMwZgP1r7HNyCWrdvd54HThWhA42z4VbCd9yz48GFq,5f7x2JEiJ2jq1xwVTf4fMs8b5ecBt6s5HgAkajuiK5yh,So11111111111111111111111111111111111111111,0.011456161

data/testing__nft_mints.csv

@@ -27,4 +27,10 @@ Ar9mEJ7qJ1uAgkWduKGo3UYVbPMdKrmWxs5z2wgZgbgN,GenoS3ck8xbDvYEZ8RxMG3Ln2qcyoAN8CTe
 5Awi9epmw6EYXzLnzkSu7qJKiZ4HnnDsJ3ix9Wxrt6DC,GwQ1d3oJDvAyhTTBUeUX9JW5dhjqYxYqzaG2HNkyYcWd,So11111111111111111111111111111111111111111,0.013683361
 5Awi9epmw6EYXzLnzkSu7qJKiZ4HnnDsJ3ix9Wxrt6DC,GwQ1d3oJDvAyhTTBUeUX9JW5dhjqYxYqzaG2HNkyYcWd,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,16250000
 AXJFRmLT11RnuFEgYC35F9tktdL77xX8y1b4HEbXPJkf,2jPtkhNbYP55Brb9bBPVwaRPFnMZHsu2YEeHHybFxwEb,So11111111111111111111111111111111111111111,0.013683361
-AXJFRmLT11RnuFEgYC35F9tktdL77xX8y1b4HEbXPJkf,2jPtkhNbYP55Brb9bBPVwaRPFnMZHsu2YEeHHybFxwEb,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,16250000
+AXJFRmLT11RnuFEgYC35F9tktdL77xX8y1b4HEbXPJkf,2jPtkhNbYP55Brb9bBPVwaRPFnMZHsu2YEeHHybFxwEb,DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263,16250000
+7onpWcfER4oz1KLPJMB6mRCXx9EXvDAWsoy4ZnVCkWun,5d3TUi4oSB2NMazfcWzKXgeinJKtiJKAUEbsFUbmnZnq,So11111111111111111111111111111111111111111,0.020942641
+7onpWcfER4oz1KLPJMB6mRCXx9EXvDAWsoy4ZnVCkWun,5d3TUi4oSB2NMazfcWzKXgeinJKtiJKAUEbsFUbmnZnq,GkpbHQu2zYmJxyp93p9wTX3uHjsFt8ZGeomVwZkGwXLH,5
+3SQcD75fEJsDPgGCPsqpbD4N2XyDHCCcLDB26UiUU6wa,86teyjAD1WjMTmw68wHu6LTWd6nvNcYkHRAQoqqJsdyA,So11111111111111111111111111111111111111111,0.020942641
+3SQcD75fEJsDPgGCPsqpbD4N2XyDHCCcLDB26UiUU6wa,86teyjAD1WjMTmw68wHu6LTWd6nvNcYkHRAQoqqJsdyA,GENEtH5amGSi8kHAtQoezp1XEXwZJ8vcuePYnXdKrMYz,5
+SnFYS4FwpRyUziz9fQ68v5A2aGKqRDaNAv5AVaMPQja,9xmSt5gqCJ1yfcFJicrdkzTi6ujcQixDN8hXTZqs4kDH,So11111111111111111111111111111111111111111,0.011456161
+5f7x2JEiJ2jq1xwVTf4fMs8b5ecBt6s5HgAkajuiK5yh,J9UMwZgP1r7HNyCWrdvd54HThWhA42z4VbCd9yz48GFq,So11111111111111111111111111111111111111111,0.011456161

models/silver/nfts/silver__nft_mint_price.yml

@@ -22,7 +22,11 @@ models:
             'A3PFimLjQFwFt9mHWLuXWQwfpaBBpp2CEp6uTR6yccKr',
             '4CLQnbNKynn6qXF1S4woTxbiuPYREoEYbkXtvoYyzvxQ',
             '5Awi9epmw6EYXzLnzkSu7qJKiZ4HnnDsJ3ix9Wxrt6DC',
-            'AXJFRmLT11RnuFEgYC35F9tktdL77xX8y1b4HEbXPJkf')"
+            'AXJFRmLT11RnuFEgYC35F9tktdL77xX8y1b4HEbXPJkf',
+            '3SQcD75fEJsDPgGCPsqpbD4N2XyDHCCcLDB26UiUU6wa',
+            '7onpWcfER4oz1KLPJMB6mRCXx9EXvDAWsoy4ZnVCkWun',
+            '5f7x2JEiJ2jq1xwVTf4fMs8b5ecBt6s5HgAkajuiK5yh',
+            'SnFYS4FwpRyUziz9fQ68v5A2aGKqRDaNAv5AVaMPQja')"
     columns:
       - name: BLOCK_TIMESTAMP
         description: "block timestamp of the latest transaction with a mint price"

models/silver/nfts/silver__nft_mint_price_generic.sql

@@ -39,12 +39,13 @@ AND
                 {{ this }}
         )
 {% elif is_incremental() %}
-AND _inserted_timestamp >= (
-    SELECT
-        MAX(_inserted_timestamp)
-    FROM
-        {{ this }}
-)
+-- AND _inserted_timestamp >= (
+--     SELECT
+--         MAX(_inserted_timestamp)
+--     FROM
+--         {{ this }}
+-- )
+and block_timestamp::date >= '2022-04-01'
 {% else %}
 AND 
     block_timestamp :: DATE BETWEEN '2021-06-02'
@@ -54,7 +55,8 @@ AND
 ),
 base_ptb AS (
     SELECT
-        distinct mint AS mint_paid,
+        mint AS mint_paid,
+        tx_id,
         account,
         DECIMAL
     FROM
@@ -84,12 +86,14 @@ WHERE
                 {{ this }}
         ) 
 {% elif is_incremental() %}
-WHERE _inserted_timestamp >= (
-    SELECT
-        MAX(_inserted_timestamp)
-    FROM
-        {{ this }}
-)
+-- WHERE _inserted_timestamp >= (
+--     SELECT
+--         MAX(_inserted_timestamp)
+--     FROM
+--         {{ this }}
+-- )
+where block_timestamp::date >= '2022-04-01'
+
 {% else %}
 WHERE
     block_timestamp :: DATE BETWEEN '2021-06-02'
@@ -199,6 +203,7 @@ mint_price_events AS (
     FROM
         metaplex_events me
         LEFT JOIN TABLE(FLATTEN(inner_instruction :instructions)) i
+    where i.value:parsed:type <> 'burn'
     group by 1,2,3,4,5,6,7,8,9,10,11,12
 ),
 pre_final as (
@@ -210,7 +215,7 @@ pre_final as (
         ) AS mint_currency,
         COALESCE(p.decimal, 9) as decimal
     from mint_price_events e
-    LEFT OUTER JOIN base_ptb p on e.token_account = p.account
+    LEFT OUTER JOIN base_ptb p on e.token_account = p.account and e.tx_id = p.tx_id
     where (temp_destination <> temp_source) or (temp_destination is null) or (temp_source is null)
 )
 SELECT

models/silver/nfts/silver__nft_mints.yml

@@ -24,7 +24,9 @@ models:
             'CmejwsHB1hcQPbY7y4EjYD5gxWJEXn9G4PkXLpLoTus1','JBzLjej5DEtVupwVtn6y1mnG4HFEkm5PVxQYFNYMJzhc','J5EFUxYPxAyx5oCUUgmCrESHjhmXuzrYsscoxRvVW5Vz',
             'DCahWcCagbsW8HAqu4QMDaQZn212MrA7aN4yLNKYn3tY','58MAJPUgfSyehsKggYEeWjpNZaKw5qxnLNZ4EKeRicmq','Dq6H4jMXQkHEf3q3BocptwF7Nh6V9FZwABXuVRgRXtts',
             'Ar9mEJ7qJ1uAgkWduKGo3UYVbPMdKrmWxs5z2wgZgbgN','4CLQnbNKynn6qXF1S4woTxbiuPYREoEYbkXtvoYyzvxQ','5huRn5xdFZVi8TY2WEYyqimMiBJQPP1uQQcoFqNZb2oG',
-            '5AbpcVMnyxgK1UfZRrYDD5iiFWWBXUVnAqLifa9n7Bm8','5Awi9epmw6EYXzLnzkSu7qJKiZ4HnnDsJ3ix9Wxrt6DC', 'AXJFRmLT11RnuFEgYC35F9tktdL77xX8y1b4HEbXPJkf')"
+            '5AbpcVMnyxgK1UfZRrYDD5iiFWWBXUVnAqLifa9n7Bm8','5Awi9epmw6EYXzLnzkSu7qJKiZ4HnnDsJ3ix9Wxrt6DC', 'AXJFRmLT11RnuFEgYC35F9tktdL77xX8y1b4HEbXPJkf',
+            '7onpWcfER4oz1KLPJMB6mRCXx9EXvDAWsoy4ZnVCkWun','3SQcD75fEJsDPgGCPsqpbD4N2XyDHCCcLDB26UiUU6wa', 'SnFYS4FwpRyUziz9fQ68v5A2aGKqRDaNAv5AVaMPQja',
+            '5f7x2JEiJ2jq1xwVTf4fMs8b5ecBt6s5HgAkajuiK5yh')"
     columns:
       - name: BLOCK_TIMESTAMP
         description: "{{ doc('block_timestamp') }}"