From 43264daa2f5c856cd76968825c60ee15374f1697 Mon Sep 17 00:00:00 2001 From: flipside-kellen Date: Thu, 14 Jul 2022 15:10:02 -0700 Subject: [PATCH 1/2] automated --- commands.txt | 1 + convert_to_rdata.R | 35 +- format_data.py | 231 +- load_data.py | 812 ++- objects/saved_params.pickle | Bin 3849 -> 9102 bytes prepare_data.py | 143 + scrape_sol_nfts.py | 290 +- scratch.py | 45 +- scratch.sql | 4917 ++++++++++++++++- solana_model.py | 1162 ++-- utils.py | 20 + viz/.DS_Store | Bin 6148 -> 6148 bytes viz/add_sales.py | 42 +- .../kellen/update_nft_deal_score_data4.dcf | 6 +- .../kellen/nft-deal-score2.dcf | 6 +- .../kellen/update_nft_deal_score_data4.dcf | 14 + viz/scrape_terra_nfts.py | 1 + viz/server.R | 8 +- viz/ui.R | 6 +- viz/update_data.R | 176 +- viz/update_nft_deal_score_data.RMD | 113 +- viz/update_nft_labels.R | 566 ++ viz/upload_solana_nft_labels.py | 655 +++ viz/utils.py | 4 + 24 files changed, 8325 insertions(+), 928 deletions(-) create mode 100644 viz/rsconnect/science.flipsidecrypto.xyz/kellen/update_nft_deal_score_data4.dcf create mode 100644 viz/update_nft_labels.R create mode 100644 viz/upload_solana_nft_labels.py diff --git a/commands.txt b/commands.txt index eeaeaa50..23fe3e28 100644 --- a/commands.txt +++ b/commands.txt @@ -9,6 +9,7 @@ sudo cp ~/nft-deal-score/viz/nft_deal_score_data.RData /srv/shiny-server/nft-dea sudo cp ~/nft_deal_score_listings_data.RData /rstudio-data sudo cp ~/nft_deal_score_sales_data.RData /rstudio-data +sudo cp ~/nft_deal_score_sales.csv /rstudio-data sudo cp ~/nft_deal_score_data.RData /rstudio-data sudo cp ~/nft_deal_score_listings.csv /rstudio-data sudo cp ~/nft_deal_score_sales.csv /rstudio-data diff --git a/convert_to_rdata.R b/convert_to_rdata.R index 67cbb582..3751f492 100644 --- a/convert_to_rdata.R +++ b/convert_to_rdata.R @@ -27,7 +27,7 @@ attributes[, feature_name := trimws(feature_name) ] attributes[, feature_value := trimws(as.character(feature_value)) ] feature_values <- read_csv('feature_values.csv') sales <- read_csv('model_sales.csv') -listings <- read_csv('listings.csv') +listings <- read.csv('/Users/kellenblumberg/git/nft-deal-score/viz/nft_deal_score_listings.csv') %>% as.data.table() coefsdf <- read_csv('coefsdf.csv') tokens <- read_csv('tokens.csv') tokens[, token_id := clean_token_id] @@ -39,18 +39,25 @@ listings <- listings[ !(collection == 'Solana Monkey Business' & token_id == 953 tokens[, token_id := as.numeric(token_id)] # manual adjustments to price -ids_1 <- attributes[ (collection == 'Aurory') & (feature_value == 'Solana Blob') ]$token_id -pred_price[ collection == 'Aurory' & token_id %in% eval(ids_1), pred_price := (pred_price * 0.8) ] +# ids_1 <- attributes[ (collection == 'Aurory') & (feature_value == 'Solana Blob') ]$token_id +# pred_price[ collection == 'Aurory' & token_id %in% eval(ids_1), pred_price := (pred_price * 0.8) ] -ids_2 <- attributes[ (collection == 'Aurory') & (feature_value == 'Long Blob Hair ') ]$token_id -pred_price[ collection == 'Aurory' & token_id %in% eval(ids_2), pred_price := (pred_price * 0.90) ] +# ids_2 <- attributes[ (collection == 'Aurory') & (feature_value == 'Long Blob Hair ') ]$token_id +# pred_price[ collection == 'Aurory' & token_id %in% eval(ids_2), pred_price := (pred_price * 0.90) ] -ids_3 <- attributes[ (collection == 'Aurory') & (grepl( 'Mask', feature_value, fixed = TRUE)) ]$token_id -pred_price[ collection == 'Aurory' & token_id %in% eval(ids_3), pred_price := (pred_price * 0.975) ] +# ids_3 <- attributes[ (collection == 'Aurory') & (grepl( 'Mask', feature_value, fixed = TRUE)) ]$token_id +# pred_price[ collection == 'Aurory' & token_id %in% eval(ids_3), pred_price := (pred_price * 0.975) ] -sales[collection == 'Cets On Creck', collection := 'Cets on Creck'] -pred_price[collection == 'Cets On Creck', collection := 'Cets on Creck'] +# sales[collection == 'Cets On Creck', collection := 'Cets on Creck'] +# pred_price[collection == 'Cets On Creck', collection := 'Cets on Creck'] listings[collection == 'Cets On Creck', collection := 'Cets on Creck'] +cols <- c( 'Citizens By Solsteads' ) +# sales[, tmp := tolower(coll)] +for (col in cols) { + sales[ tolower(collection) == eval(tolower(col)), collection := eval(col) ] + pred_price[ tolower(collection) == eval(tolower(col)), collection := eval(col) ] + listings[ tolower(collection) == eval(tolower(col)), collection := eval(col) ] +} sort(unique(listings$collection)) @@ -58,10 +65,13 @@ sort(unique(pred_price$collection)) sort(unique(sales$collection)) # filter for only collections that have all data -a <- unique(pred_price[, list(collection)]) -b <- unique(sales[, list(collection)]) -c <- unique(listings[, list(collection)]) +a <- unique(pred_price[, list(collection)][order(collection)]) +b <- unique(sales[, list(collection)][order(collection)]) +c <- unique(listings[, list(collection)][order(collection)]) d <- merge(merge(a, b), c) +d <- d[order(collection)] +d <- d[ collection %in% c('Aurory','Bubblegoose Ballers','Catalina Whale Mixer','Cets on Creck','DeGods','Degen Apes','Famous Fox Federation','Meerkat Millionaires','Okay Bears','Pesky Penguins','Primates','SOLGods','Solana Monkey Business','Stoned Ape Crew','ThugbirdzcMAYC') ] +write.csv(d, '~/Downloads/tmp.csv', row.names=F) pred_price <- merge(pred_price, d, by=c('collection')) attributes <- merge(attributes, d, by=c('collection')) @@ -91,6 +101,7 @@ save( , tokens , file = paste0(file.location,'nft_deal_score_data.Rdata') ) + # save( # listings # , file = paste0(file.location,'nft_deal_score_listings_data.Rdata') diff --git a/format_data.py b/format_data.py index d274c028..a113009f 100644 --- a/format_data.py +++ b/format_data.py @@ -4,6 +4,7 @@ import os import math import json from typing import Collection +from nbformat import write import pandas as pd import snowflake.connector @@ -364,15 +365,37 @@ def solana(): + query = ''' + SELECT DISTINCT project_name + FROM solana.dim_nft_metadata + ''' + seen = ctx.cursor().execute(query) + seen = pd.DataFrame.from_records(iter(seen), columns=[x[0] for x in seen.description]) + seen = clean_colnames(seen) + seen = list(seen.project_name.values) + seen = [ x.lower() for x in seen ] + metadata = pd.read_csv('./data/metadata.csv') + len(metadata) # print(sorted(metadata.collection.unique())) # metadata = metadata[metadata.collection == collection] # print(sorted(metadata.collection.unique())) metadata = metadata[-(metadata.feature_name.isin(['adj_nft_rank_0','adj_nft_rank_1','adj_nft_rank_2','nft_rank']))] + metadata[['collection']].drop_duplicates().to_csv('~/Downloads/tmp.csv', index=False) len(metadata.token_id.unique()) - id_map = pd.read_csv('./data/mint_to_token_id_map.csv') + # id_map = pd.read_csv('./data/mint_to_token_id_map.csv') id_map = pd.read_csv('./data/tokens.csv') - cs = ['Stoned Ape Crew'] + tokens = pd.read_csv('./data/tokens.csv') + tokens.collection.unique() + len(tokens.collection.unique()) + cs = [ x for x in id_map.collection.unique() if not x.lower() in seen ] + len(id_map.collection.unique()) + len(cs) + id_map = id_map[id_map.collection.isin(cs)] + metadata = metadata[metadata.collection.isin(cs)] + + # cs = metadata[metadata.chain.fillna('Solana') == 'Solana'].collection.unique() + cs = metadata.collection.unique() id_map = id_map[id_map.collection.isin(cs)] metadata = metadata[metadata.collection.isin(cs)] sorted(id_map.collection.unique()) @@ -399,51 +422,180 @@ def solana(): # sorted(metadata.feature_name.unique()) # metadata[['collection']].drop_duplicates().to_csv('~/Downloads/tmp.csv', index=False) + # Python code to convert into dictionary + def Convert(tup, di): + di = dict(tup) + return di + metadata = metadata[-metadata.collection.isin(['LunaBulls', 'Levana Dragon Eggs'])] + metadata['token_id'] = metadata.token_id.astype(float) metadata['token_id'] = metadata.token_id.astype(int) metadata.groupby(['collection','feature_name']).token_id.count() metadata.head() + metadata[metadata.mint_address.isnull()].collection.unique() assert(len(metadata[metadata.mint_address.isnull()]) == 0) - for collection in metadata.collection.unique(): - print(collection) + dirs = sorted(list(set(os.listdir('./data/metadata/')).intersection(set(metadata.collection.unique())))) + sorted(list(metadata.collection.unique())) + # collection = 'Bubblegoose Ballers' + it = 0 + tot = len(metadata.collection.unique()) + data = [] + for collection in metadata.collection.unique()[:1]: + print('#{} / {}: {}'.format(it, tot, collection)) mdf = metadata[metadata.collection == collection] - results = [] - for token_id in sorted(mdf.token_id.unique()): - if token_id % 1000 == 1: - print(token_id, len(results)) - cur = mdf[mdf.token_id == token_id] - token_metadata = {} - # m = mints[(mints.collection == collection) & (mints.token_id == token_id) ] - m = metadata[(metadata.collection == collection) & (metadata.token_id == token_id) ] - m = m.fillna('None') - if not len(m): - print(token_id) - continue - # mint_address = m.mint_address.values[0] if 'mint_address' in m.columns else '' - mint_address = m.mint_address.values[0] - for row in cur.iterrows(): - row = row[1] - token_metadata[row['feature_name']] = row['feature_value'] + df.groupby('Column1')[['Column2', 'Column3']].apply(lambda g: g.values.tolist()).to_dict() + mdf.head(20).groupby(['collection','image_url','token_id'])[[ 'feature_name','feature_value' ]].apply(lambda g: g.values.tolist()).to_dict() - d = { - 'commission_rate': None - , 'mint_address': mint_address - , 'token_id': token_id - , 'contract_address': mint_address - , 'contract_name': row['collection'] - , 'created_at_block_id': 0 - , 'created_at_timestamp': str('2021-01-01') - , 'created_at_tx_id': '' - , 'creator_address': mint_address - , 'creator_name': row['collection'] - , 'image_url': 'None' - , 'project_name': row['collection'] - , 'token_id': int(token_id) - , 'token_metadata': token_metadata - , 'token_metadata_uri': row['image_url'] - , 'token_name': row['collection'] + mdf.head(20).groupby(['collection','image_url','token_id'])[[ 'feature_name','feature_value' ]].apply(lambda g: list(map(tuple, g.values.tolist())) ).to_dict() + + mdf.head(20).groupby(['collection','image_url','token_id'])[[ 'feature_name','feature_value' ]].apply(lambda g: Convert(list(map(tuple, g.values.tolist())), {}) ).to_dict() + a = mdf.head(20).groupby(['collection','mint_address','token_id','image_url'])[[ 'feature_name','feature_value' ]].apply(lambda g: Convert(list(map(tuple, g.values.tolist())), {}) ).reset_index() + + + a = metadata.groupby(['collection','mint_address','token_id','image_url'])[[ 'feature_name','feature_value' ]].apply(lambda g: Convert(list(map(tuple, g.values.tolist())), {}) ).reset_index() + a.columns = ['collection','mint_address','token_id','image_url', 'token_metadata'] + a['commission_rate'] = None + a['contract_address'] = a.mint_address + a['contract_name'] = a.collection + a['created_at_block_id'] = 0 + a['created_at_timestamp'] = '2021-01-01' + a['created_at_tx_id'] = '' + a['creator_address'] = a.mint_address + a['creator_name'] = a.collection + a['project_name'] = a.collection + a['token_metadata_uri'] = a.image_url + a['token_name'] = a.collection + a.to_csv('./data/metadata/results.csv', index=False) + a['n'] = range(len(a)) + a['n'] = a.n.apply(lambda x: int(x/50) ) + a['token_id'] = a.token_id.astype(int) + cols = ['collection', 'mint_address', 'token_id', 'image_url', 'token_metadata', + 'commission_rate', 'contract_address', 'contract_name', + 'created_at_block_id', 'created_at_timestamp', 'created_at_tx_id', + 'creator_address', 'creator_name', 'project_name', 'token_metadata_uri', + 'token_name'] + + n = 100000 + tot = int(len(a) / n) + 1 + for i in range(0, len(a), n): + ind = int(i/n) + print('#{} / {}'.format(ind, tot)) + g = a.head(i+n).tail(n).to_dict('records') + txt = [ + { + "model": { + "blockchain": "solana", + "sinks": [ + { + "destination": "{database_name}.silver.nft_metadata", + "type": "snowflake", + "unique_key": "blockchain || contract_address || token_id" + } + ], + }, + "results": g[x:x+50] + } + for x in range(0, len(g), 50) + ] + w = pd.DataFrame({'ind': range(len(txt)), 'results':[json.dumps(x) for x in txt] }) + # w['results'] = w.results.apply(lambda x: x[1:-1] ) + w.to_csv('./data/metadata/results/{}.csv'.format(ind), index=False) + # with open('./data/metadata/results/{}.json'.format(i), 'w') as outfile: + # json.dump(results[i:i+100000], outfile) + + g = a.head(200).groupby('n')[cols].apply(lambda g: Convert(list(map(tuple, g.values.tolist())), {}) ).to_dict() + g = a.head(200).groupby('n')[cols].apply(lambda g: (list(map(tuple, g.values.tolist())), {}) ) + g = a.head(200).groupby('n')[cols].apply(lambda g: g.values.tolist()).reset_index() + g = a.head(200).to_dict('records') + sorted(a.collection.unique()) + g = a[a.collection == 'Jungle Cats'].head(20000).to_dict('records') + txt = [ + { + "model": { + "blockchain": "solana", + "sinks": [ + { + "destination": "{database_name}.silver.nft_metadata", + "type": "snowflake", + "unique_key": "blockchain || contract_address || token_id" + } + ], + }, + "results": g[i:i+50] } - results.append(d) + for i in range(0, len(g), 50) + ] + w = pd.DataFrame({'ind': range(len(txt)), 'results':[json.dumps(x) for x in txt] }) + # w['results'] = w.results.apply(lambda x: x[1:-1] ) + w.to_csv('./data/metadata/results.csv', index=False) + with open('./data/metadata/results.txt', 'w') as outfile: + outfile.write(json.dumps(txt)) + g = list(a.head(200).values) + results = a.to_dict('records') + for i in range(0, len(results), 100000): + print(i) + with open('./data/metadata/results/{}.json'.format(i), 'w') as outfile: + json.dump(results[i:i+100000], outfile) + + n = 50 + r = math.ceil(len(results) / n) + for i in range(r): + print('#{} / {}'.format(i, r)) + newd = { + "model": { + "blockchain": "solana", + "sinks": [ + { + "destination": "{database_name}.silver.nft_metadata", + "type": "snowflake", + "unique_key": "blockchain || contract_address || token_id" + } + ], + }, + "results": results[(i * n):((i * n)+r)] + } + data += [ json.dumps(newd) ] + with open('./data/metadata/results/{}.txt'.format(collection, i), 'w') as outfile: + outfile.write(json.dumps(newd)) + + + # results = [] + # for token_id in sorted(mdf.token_id.unique()): + # if token_id % 1000 == 1: + # print(token_id, len(results)) + # cur = mdf[mdf.token_id == token_id] + # token_metadata = {} + # # m = mints[(mints.collection == collection) & (mints.token_id == token_id) ] + # m = metadata[(metadata.collection == collection) & (metadata.token_id == token_id) ] + # m = m.fillna('None') + # if not len(m): + # print(token_id) + # continue + # # mint_address = m.mint_address.values[0] if 'mint_address' in m.columns else '' + # mint_address = m.mint_address.values[0] + # for row in cur.iterrows(): + # row = row[1] + # token_metadata[row['feature_name']] = row['feature_value'] + + # d = { + # 'commission_rate': None + # , 'mint_address': mint_address + # , 'token_id': token_id + # , 'contract_address': mint_address + # , 'contract_name': row['collection'] + # , 'created_at_block_id': 0 + # , 'created_at_timestamp': str('2021-01-01') + # , 'created_at_tx_id': '' + # , 'creator_address': mint_address + # , 'creator_name': row['collection'] + # , 'image_url': row['image_url'] + # , 'project_name': row['collection'] + # , 'token_id': int(token_id) + # , 'token_metadata': token_metadata + # , 'token_metadata_uri': row['image_url'] + # , 'token_name': row['collection'] + # } + # results.append(d) print('Uploading {} results'.format(len(results))) dir = './data/metadata/{}/'.format(collection) @@ -466,6 +618,7 @@ def solana(): }, "results": results[(i * n):((i * n)+r)] } + data += [ json.dumps(newd) ] with open('./data/metadata/{}/{}.txt'.format(collection, i), 'w') as outfile: outfile.write(json.dumps(newd)) diff --git a/load_data.py b/load_data.py index f85ba5ac..6cc59bd2 100644 --- a/load_data.py +++ b/load_data.py @@ -1,9 +1,8 @@ -import collections import re import os import json +import time import math -from tkinter import SEL import requests import pandas as pd import urllib.request @@ -11,11 +10,24 @@ import snowflake.connector from bs4 import BeautifulSoup from time import sleep +import cloudscraper + +from theblockchainapi import SolanaAPIResource, SolanaNetwork, SearchMethod + +# Get an API key pair for free here: https://dashboard.blockchainapi.com/api-keys +MY_API_KEY_ID = 'sLbjx8YFYdTtUuH' +MY_API_SECRET_KEY = 'p24pFaM9lLbWscN' +BLOCKCHAIN_API_RESOURCE = SolanaAPIResource( + api_key_id=MY_API_KEY_ID, + api_secret_key=MY_API_SECRET_KEY +) os.chdir('/Users/kellenblumberg/git/nft-deal-score') from solana_model import just_float -from utils import clean_name, clean_token_id, format_num +from utils import clean_name, clean_token_id, format_num, merge + + ######################### # Connect to DB # @@ -100,6 +112,19 @@ def add_collection_steps(): # 5. run model pass +def create_upload_file(): + cols = [ 'collection','mint_address' ] + a = pd.read_csv('./data/mints-2022-06-13-2pm.csv')[cols] + b = pd.read_csv('~/Downloads/manual_labels.csv') + b.columns = cols + c = pd.read_csv('~/Downloads/solscan_collections.csv')[cols] + d = pd.read_csv('./data/tokens.csv')[cols] + df = pd.concat([a, b, c, d]).drop_duplicates(subset=['mint_address'], keep='last') + df.to_csv('~/Downloads/mints-2022-06-13-5pm.csv', index=False) + tmp = pd.read_csv('~/Downloads/mints-2022-06-13-5pm.csv') + tmp[tmp.mint_address == 'EhuVN896QVypRreAt6mcJr6eKkKunVzsgSRz7qt4oeBr'] + + def manual_clean(): for c in [ 'pred_price', 'attributes', 'feature_values', 'model_sales', 'listings', 'coefsdf', 'tokens' ]: df = pd.read_csv('./data/{}.csv'.format(c)) @@ -108,7 +133,149 @@ def manual_clean(): df['clean_token_id'] = df.token_id df.to_csv('./data/{}.csv'.format(c), index=False) + +def pull_from_solscan(): + + todo = [ + ['50a75e6d3d0b6d4a72b2f745fdba4b1c28bc774ca9629fe8e36053ae2fb396f8','Degen Egg'] + , ['45e3f45d695e9e8775eed480cb0f5a6a957d47dcb3ed3800e454846dca9ab7fc','Genopets'] + , ['a437071c6f9679e8431a072ae39421262bf289cc6ead21e38190d5b7b409e7f7','Shin Sengoku'] + , ['d38349f2704e8cd1c538cc48fbea4b3e2596ac8da14b62c0eb3c07aeda7ae75e','SolStein'] + , ['9e0593a4842ceb9ccdc510e6ffdf0d84f736bff2b58d5803c5002ace17df9fe0','Zillaz NFT'] + , ['895d8f01108fbb6b28c5e32027c9c98e3054241927c8e59c304fa4763c5c88ea','enviroPass Tier 02'] + , ['59c2a35d902f85feec4c774df503a0df2be263f763dcbcb73bce50c999fc2c78','The Fracture'] + , ['e8dfb059b1dfc71cf97342a1c46793bc5e154909416a93a155929da5bba44a57','Suteki'] + , ['271e0d68d069d80afbcb916e877831b060933b97e7b02e1cfb77e74b228b4745','Chillchat'] + ] + start = time.time() + data = [] + meta = [] + it = 0 + tot = len(todo) + for collectionId, collection in todo: + it += 1 + print('#{} / {}'.format(it, tot)) + # collectionId = j['data']['collectionId'] + # collection = j['data']['collection'] + offset = 0 + limit = 500 + while True: + print(offset) + url = 'https://api.solscan.io/collection/nft?sortBy=nameDec&collectionId={}&offset={}&limit={}'.format(collectionId, offset, limit) + r = requests.get(url) + js = r.json()['data'] + offset += limit + if len(js) == 0: + break + for j in js: + data += [[ collectionId, collection, j['info']['mint'] ]] + m = j['info']['meta'] + m['mint_address'] = j['info']['mint'] + # m['name'] = row['name'] + # m['update_authority'] = update_authority + meta += [ m ] + it += 1 + end = time.time() + print('Finished {} / {} in {} minutes'.format(it, tot, round((end - start) / 60.0, 1))) + df = pd.DataFrame(data, columns=['collection_id','collection','mint_address']) + df.to_csv('~/Downloads/solscan_collections.csv', index=False) + df[['collection','mint_address']].to_csv('~/Downloads/mints-2022-06-14-8am.csv', index=False) + df.groupby('collection').mint_address.count() + +def collecitons_from_missing_tokens(): + query = ''' + WITH base AS ( + SELECT block_timestamp::date AS date + , s.* + , ROW_NUMBER() OVER (ORDER BY sales_amount DESC) AS rn + FROM solana.fact_nft_sales s + LEFT JOIN solana.dim_labels l on s.mint = l.address + WHERE marketplace in ('magic eden v1', 'magic eden v2') + AND block_timestamp >= '2022-01-01' + AND l.address IS NULL + AND sales_amount >= 10 + ) + SELECT * + FROM base + WHERE rn % 20 = 0 + ORDER BY sales_amount DESC + LIMIT 500 + ''' + missing = ctx.cursor().execute(query) + missing = pd.DataFrame.from_records(iter(missing), columns=[x[0] for x in missing.description]) + missing = clean_colnames(missing) + missing.head() + + headers = { + 'Authorization': 'Bearer 9c39e05c-db3c-4f3f-ac48-84099111b813' + } + it = 0 + tot = len(missing) + data = [] + for m in missing.mint.unique(): + it += 1 + if it % 10 == 0: + print('#{} / {} ({})'.format(it, tot, len(data))) + url = 'https://api-mainnet.magiceden.dev/v2/tokens/{}'.format(m) + r = requests.get(url, headers=headers) + j = r.json() + data.append(j) + pass + df = pd.DataFrame(data) + df.head()[['collection','mintAddress']] + df.to_csv('~/Downloads/tmp.csv', index=False) + need = df.groupby(['collection','updateAuthority']).mintAddress.count().reset_index().sort_values('mintAddress', ascending=0) + need = need[need.mintAddress > 1].rename(columns={'updateAuthority':'update_authority'}) + need.to_csv('~/Downloads/missing.csv', index=False) + need.head() + sorted(need.collection.unique()) + need['collection'] = need.collection.apply(lambda x: re.sub('_', ' ', x.title()).strip() ) + need['collection'] = need.collection.apply(lambda x: re.sub('\|', '-', x).strip() ) + need['collection'] = need.collection.apply(lambda x: re.sub('\)', '', x).strip() ) + need['collection'] = need.collection.apply(lambda x: re.sub('\(', '', x).strip() ) + need['collection'] = need.collection.apply(lambda x: re.sub('\'', '', x).strip() ) + + us = sorted(g[g.mintAddress > 1].updateAuthority.unique()) + tot = len(us) + it = 0 + for u in us: + it += 1 + print('#{} / {} ({})'.format(it, tot, len(data))) + + nfts = BLOCKCHAIN_API_RESOURCE.search_nfts( + update_authority = u + , update_authority_search_method = SearchMethod.EXACT_MATCH + ) + print(u, len(nfts)) + for n in nfts: + m = n['nft_metadata'] + data += [[ m['update_authority'], m['mint'], m['data']['symbol'], m['data']['name'] ]] + +def manual_tags(): + d = { + 'daaLrDfvcT4joui5axwR2gCkGAroruJFzyVsacU926g': 'Degenerate Ape Kindergarten' + , 'FbfGrZ3LKuGSsayK57DetzzyN7qKeNnDuLMu5bBSocwF': 'Botheads' + } + a = 'FbfGrZ3LKuGSsayK57DetzzyN7qKeNnDuLMu5bBSocwF' + c = 'Botheads' + labels = pd.DataFrame() + for a, c in d.items(): + query = ''' + SELECT DISTINCT instructions[1]:parsed:info:mint::string AS mint_address + FROM solana.fact_transactions + WHERE instructions[1]:parsed:info:mintAuthority = '{}' + '''.format(a) + df = ctx.cursor().execute(query) + df = pd.DataFrame.from_records(iter(df), columns=[x[0] for x in df.description]) + df = clean_colnames(df) + df['collection'] = c + labels = labels.append(df) + labels.to_csv('~/Downloads/manual_labels.csv', index=False) + def mints_from_me(): + ################################## + # Get All ME Collections # + ################################## headers = { 'Authorization': 'Bearer 9c39e05c-db3c-4f3f-ac48-84099111b813' } @@ -144,6 +311,9 @@ def mints_from_me(): # lp_df.to_csv('./data/me_lp_collections.csv', index=False) # lp_df = pd.read_csv('./data/me_lp_collections.csv') + ########################################### + # Get 1 Mint From Each Collection # + ########################################### it = 0 l_data = [] old_l_df = pd.read_csv('./data/me_mints.csv') @@ -154,7 +324,7 @@ def mints_from_me(): it += 1 row = row[1] print('Listings on {}...'.format(row['symbol'])) - url = 'https://api-mainnet.magiceden.dev/v2/collections/{}/listings?offset=0&limit=1'.format(row['symbol']) + url = 'https://api-mainnet.magiceden.dev/v2/collections/{}/activities?offset=0&limit=1'.format(row['symbol']) if row['symbol'] in seen: print('Seen') continue @@ -218,9 +388,38 @@ def mints_from_me(): # l_df = pd.DataFrame(l_data, columns=['symbol','name','mint_address']) # l_df.to_csv('./data/me_mints.csv', index=False) + # get missing collections + query = ''' + WITH base AS ( + SELECT block_timestamp::date AS date + , s.* + , ROW_NUMBER() OVER (ORDER BY sales_amount DESC) AS rn + FROM solana.fact_nft_sales s + LEFT JOIN solana.dim_labels l on s.mint = l.address + WHERE marketplace in ('magic eden v1', 'magic eden v2') + AND block_timestamp >= '2022-01-01' + AND block_timestamp <= '2022-05-20' + AND l.address IS NULL + AND sales_amount > 20 + ) + SELECT * + FROM base + WHERE rn % 50 = 1 + LIMIT 100 + ''' + missing = ctx.cursor().execute(query) + missing = pd.DataFrame.from_records(iter(missing), columns=[x[0] for x in missing.description]) + missing = clean_colnames(missing) + + ###################################################### + # Get Update Authorities For All Collections # + ###################################################### l_df = pd.read_csv('./data/me_mints.csv') + len(l_df) + l_df.head() m_old = pd.read_csv('./data/me_update_authorities.csv') - m_data = list(m_old.values) + m_old['seen'] = 1 + m_data = list(m_old[['symbol','name','update_authority']].values) seen = [ x[0] for x in m_data ] print('Seen {} m_data'.format(len(seen))) l_df = l_df[-l_df.symbol.isin(seen)] @@ -258,19 +457,583 @@ def mints_from_me(): m_df.to_csv('./data/me_update_authorities.csv', index=False) m_df = pd.DataFrame(m_data, columns=['symbol','name','update_authority']) m_df = m_df.drop_duplicates() - print('Adding {} rows to me_mints'.format(len(m_df) - len(m_old))) + print('Adding {} rows to me_update_authorities'.format(len(m_df) - len(m_old))) m_df.to_csv('./data/me_update_authorities.csv', index=False) + m_df.tail(134).head(20) + m_df = m_df.tail(134) + + query = ''' + SELECT DISTINCT project_name, LOWER(project_name) AS lower_name + FROM crosschain.address_labels + WHERE blockchain = 'solana' + AND label_subtype = 'nf_token_contract' + AND project_name IS NOT NULL + ''' + + labels = ctx.cursor().execute(query) + labels = pd.DataFrame.from_records(iter(labels), columns=[x[0] for x in labels.description]) + labels = clean_colnames(labels) + labels.to_csv('~/Downloads/tmp-la.csv', index=False) + + ###################################################### + # Get Update Authorities For All Collections # + ###################################################### m_df = pd.read_csv('./data/me_update_authorities.csv') - def f(x): - x = re.sub('\(|\)', '', x) - x = re.sub(' ', '_', x) - x = re.sub('\'', '', x) - return(x) - m_df['collection'] = m_df.name.apply(lambda x: f(x) ) + m_df['seen'] = (-m_df.name.isin(m_df.name.tail(134).values)).astype(int) + m_df['lower_name'] = m_df.name.apply(lambda x: x.lower() ) + seen = list(labels.lower_name.unique()) + m_df['seen'] = m_df.lower_name.isin(seen).astype(int) + n_auth = m_df.groupby('update_authority').name.count().reset_index().rename(columns={'name':'n_auth'}) + m_df = m_df.merge(n_auth) + len(m_df[m_df.seen == 0]) + len(m_df[ (m_df.seen == 0) & (m_df.n_auth == 1)]) + len(m_df[ (m_df.seen == 0) & (m_df.n_auth > 1)]) - x = 'asf (asf)' - f(x) + m_df.to_csv('~/Downloads/tmp-m_df.csv', index=False) + len(m_df.name.unique()) + + need = list(m_df[m_df.seen == 0].update_authority.unique()) + need = list(m_df[ (m_df.seen == 0) & (m_df.n_auth == 1) ].update_authority.unique()) + len(need) + # need = need + [ + # need = [ + # 'CDgbhX61QFADQAeeYKP5BQ7nnzDyMkkR3NEhYF2ETn1k' # taiyo + # , 'DC2mkgwhy56w3viNtHDjJQmc7SGu2QX785bS4aexojwX' # DAA + # , 'daaLrDfvcT4joui5axwR2gCkGAroruJFzyVsacU926g' # Degen Egg + # , 'BL5U8CoFPewr9jFcKf3kE1BhdFS1J59cwGpeZrm7ZTeP' # Skullbot + # , 'DRGNjvBvnXNiQz9dTppGk1tAsVxtJsvhEmojEfBU3ezf' # Boryoku + # , '7hYkx2CNGRB8JE7X7GefX1ak1dqe7GxgYKbpfj9moE9D' # mindfolk + # , 'CjwNEVQFKk8YzZLCvvw6sNrjxiQW8dYDSzhTph18T7g5' # jelly rascals + # , 'EcxEqUj4RNgdGJwPE3ktsM99Ea9ThPmXHUV5g37Qm4ju' # women monkey + # , 'EQSoRhbN9fEEYXKEE5Lg63Mqf17P3JydcWTvDhdMJW1N' # hydrascripts + # , '75CPiM9ywLgxhii9SQsNoA1SH3h66o5EhrYsazHR5Tqk' # hydrascripts + # , 'aury7LJUae7a92PBo35vVbP61GX8VbyxFKausvUtBrt' # aurory + # , 'ET3LWbEL6q4aUSjsX5xLyWktCwqKh6qsQE5j6TDZtZBY' # enviropass + # , '8ERR2gYrvXcJFuoNAbPRvHXtrJnAXXHgXKkVviwz9R6C' # enviroPass + # , 'GRDCbZBP1x2JxYf3rQQoPFGzF57LDPy7XtB1gEMaCqGV' # Space Robots + # , 'GenoS3ck8xbDvYEZ8RxMG3Ln2qcyoAN8CTeZuaWgAoEA' # Genopet + # , 'STEPNq2UGeGSzCyGVr2nMQAzf8xuejwqebd84wcksCK' # stepn + # , 'HcS8iaEHwUino8wKzcgC16hxHodnPCyacVYUdBaSZULP' # BASC + # , 'AvkbtawpmMSy571f71WsWEn41ATHg5iHw27LoYJdk8QA' # THUG + # , 'GH4QhJznKEHHv44AqEH5SUohkUauWyAFtu5u8zUWUKL4' # StepN Shoebox + # , 'FTQmhcD7SNBWrVxTgQMFr7xL2aA6adfAJJPBxGKU4VsZ' # Solstien + # ] + need = m_df[m_df.update_authority.isin(need)] + + # m_df[m_df.lower_name.isin(seen)] + # m_df[-m_df.lower_name.isin(seen)] + # tmp = m_df[['update_authority','collection']].drop_duplicates().groupby(['update_authority']).collection.count().reset_index().rename(columns={'collection':'n_collection'}) + # tmp = tmp.sort_values('n_collection', ascending=0) + # m_df = m_df.merge(tmp) + # m_df = m_df.sort_values(by=['n_collection','update_authority','collection'], ascending=[0,0,0]) + l_df = pd.read_csv('./data/me_mints.csv') + fix = need.merge(l_df[[ 'name','mint_address' ]]) + # len(need.name.unique()) + # len(fix.name.unique()) + # fix = fix.sort_values(by=['update_authority','collection'], ascending=[0,0]) + # fix.head() + + + # seen = [] + # data = [] + # meta = [] + + # fix = fix[-(fix.name.isin(seen))] + # start = time.time() + # it = 0 + # tot = len(fix) + # scraper = cloudscraper.create_scraper() + # # for each collection + # for row in fix.iterrows(): + # row = row[1] + # print(row['name']) + # if row['name'] in seen: + # print('Seen') + # continue + # url = 'https://api.solscan.io/nft/detail?mint={}'.format(row['mint_address']) + # t = scraper.get(url).text + # j = json.loads(t) + # # r = requests.get(url) + # # j = r.json() + # j['data'] + # if not j['success']: + # print('Error') + # print(r) + # print(j) + # sleep(1) + # continue + # update_authority = j['data']['updateAuthority'] + # collectionId = j['data']['collectionId'] + # collection = j['data']['collection'] + # offset = 0 + # limit = 500 + # while True: + # print(offset) + # url = 'https://api.solscan.io/collection/nft?sortBy=nameDec&collectionId={}&offset={}&limit={}'.format(collectionId, offset, limit) + # r = requests.get(url) + # js = r.json()['data'] + # offset += limit + # if len(js) == 0: + # break + # for j in js: + # data += [[ update_authority, collectionId, collection, row['symbol'], row['name'], row['collection'], j['info']['mint'] ]] + # m = j['info']['meta'] + # m['mint_address'] = j['info']['mint'] + # m['name'] = row['name'] + # m['update_authority'] = update_authority + # meta += [ m ] + # it += 1 + # end = time.time() + # print('Finished {} / {} in {} minutes'.format(it, tot, round((end - start) / 60.0, 1))) + + # old = pd.read_csv('./data/nft_label_tokens.csv') + # token_df = pd.DataFrame(data, columns=['update_authority','collectionId','solscan_collection','symbol','name','collection','mint']) + # token_df = token_df.append(old).drop_duplicates() + # token_df.to_csv('./data/nft_label_tokens.csv', index=False) + + # old = pd.read_csv('./data/nft_label_metadata.csv') + # meta_df = pd.DataFrame(meta) + # meta_df = meta_df.append(old).drop_duplicates() + # meta_df.to_csv('./data/nft_label_metadata.csv', index=False) + # seen = list(token_df.name.unique()) + + # m_df.to_csv('~/Downloads/tmp.csv', index=False) + # tmp[tmp.collection > 1] + # m_df.head() + # def f(x): + # x = re.sub('\(|\)', '', x) + # x = re.sub(' ', '_', x) + # x = re.sub('\'', '', x) + # return(x) + # m_df['collection'] = m_df.name.apply(lambda x: f(x) ) + + # x = 'asf (asf)' + # f(x) + + # query = ''' + # WITH base AS ( + # SELECT * + # , ROW_NUMBER() OVER (PARTITION BY project_name ORDER BY insert_date DESC) AS rn + # FROM crosschain.address_labels + # WHERE blockchain = 'solana' + # AND label_subtype = 'nf_token_contract' + # ) + # SELECT * + # FROM base + # ''' + + # examples = ctx.cursor().execute(query) + # examples = pd.DataFrame.from_records(iter(examples), columns=[x[0] for x in examples.description]) + # examples = clean_colnames(examples) + # examples.head() + # examples[examples.address_name == 'paradisedao'].head() + # examples[examples.address == 'GUXSatf5AAFKmuQgSgn4GoGzBEhwJ9WAQRxeVt1vZvkb'].head() + # # m_df = pd.read_csv('./data/me_update_authorities.csv') + # # fix = m_df[m_df.n_collection > 1].merge(examples[[ 'address','address_name' ]].rename(columns={'address_name':'name'}) ) + # fix = m_df[m_df.n_collection > 1].merge(examples[[ 'address','address_name' ]].rename(columns={'address_name':'name'}) ) + # len(m_df[m_df.n_collection > 1].name.unique()) + # len(fix.name.unique()) + + # j = list(fix.address.unique()) + # with open('./data/fix_mints.json', 'w') as f: + # json.dump(j, f) + + # seen = list(examples.address.unique()) + # seen = [] + # need = df[-df.mint_address.isin(seen)].sort_values(['collection','mint_address']) + # CDgbhX61QFADQAeeYKP5BQ7nnzDyMkkR3NEhYF2ETn1k - taiyo + # DC2mkgwhy56w3viNtHDjJQmc7SGu2QX785bS4aexojwX - DAA + # DRGNjvBvnXNiQz9dTppGk1tAsVxtJsvhEmojEfBU3ezf - Boryoku + # 7hYkx2CNGRB8JE7X7GefX1ak1dqe7GxgYKbpfj9moE9D - mindfolk + # CjwNEVQFKk8YzZLCvvw6sNrjxiQW8dYDSzhTph18T7g5 - mindfolk + need = fix.copy().rename(columns={'name':'collection'}) + # need = need.drop_duplicates(subset=['update_authority']).sort_values('collection').head(7).tail(1) + need = need.drop_duplicates(subset=['update_authority']).sort_values('collection') + need['collection'] = need.collection.apply(lambda x: re.sub('\|', '-', x).strip() ) + need['collection'] = need.collection.apply(lambda x: re.sub('\)', '', x).strip() ) + need['collection'] = need.collection.apply(lambda x: re.sub('\(', '', x).strip() ) + need['collection'] = need.collection.apply(lambda x: re.sub('\'', '', x).strip() ) + need.collection.unique() + # need = need.drop_duplicates(subset=['collection']).sort_values('collection') + n = 0 + # 1310 - 310 + # need = need.tail(n).head(300).tail(25) + # need = need.tail(1009).head(17) + # need = need.tail(1009 - 17).head(17) + # 1-285, 1310-975 + len(need) + # print(n) + + mfiles = ['/data/mints/{}/{}_mint_accounts.json'.format(re.sub(' |-', '_', collection), update_authority) for collection, update_authority in zip(need.collection.values, need.update_authority.values) ] + seen = [ x for x in mfiles if os.path.exists(x) ] + seen = [] + + # for update authorities that have only 1 collection, we can just check metaboss once + rpc = 'https://red-cool-wildflower.solana-mainnet.quiknode.pro/a1674d4ab875dd3f89b34863a86c0f1931f57090/' + # need = need.tail(400) + it = 0 + tot = len(need) + for row in need.iterrows(): + it += 1 + row = row[1] + collection = row['collection'] + print('#{} / {}: {}'.format(it, tot, collection)) + # if collection in seen: + # continue + update_authority = row['update_authority'] + # print('Working on {}...'.format(collection)) + collection_dir = re.sub(' |-', '_', collection) + + dir = './data/mints/{}/'.format(collection_dir) + mfile = '{}{}_mint_accounts.json'.format(dir, update_authority) + if not os.path.exists(dir): + print(collection) + os.makedirs(dir) + # elif len(os.listdir(dir)) and os.path.exists(mfile): + # print('Already have {}.'.format(collection)) + # print('Seen') + # continue + seen.append(update_authority) + os.system('metaboss -r {} -t 300 snapshot mints --update-authority {} --output {}'.format(rpc, update_authority, dir)) + + # write the mints to csv + data = [] + for path in os.listdir('./data/mints/'): + if os.path.isdir('./data/mints/'+path): + collection = re.sub('_', ' ', path).strip() + for fname in os.listdir('./data/mints/'+path): + f = './data/mints/'+path+'/'+fname + if os.path.isfile(f) and '.json' in f: + with open(f) as file: + j = json.load(file) + for m in j: + data += [[ collection, m ]] + df = pd.DataFrame(data, columns=['collection','mint_address']) + df.collection.unique() + df.to_csv('./data/single_update_auth_labels.csv', index=False) + + ################################ + # Multiple Authorities # + ################################ + rpc = 'https://red-cool-wildflower.solana-mainnet.quiknode.pro/a1674d4ab875dd3f89b34863a86c0f1931f57090/' + need = list(m_df[ (m_df.seen == 0) & (m_df.n_auth > 1) ].update_authority.unique()) + need = m_df[m_df.update_authority.isin(need)] + fix = need.merge(l_df[[ 'name','mint_address' ]]) + need = fix.copy().rename(columns={'name':'collection'}) + need = need.sort_values('collection').drop_duplicates(subset=['update_authority'], keep='first') + i = 5 + sz = 112 + t = len(need) - (sz * (i - 1)) if sz * i > len(need) else sz + print(t) + need = need.head(sz * i).tail(t) + # need = need.head(150 * 2).tail(150) + # need = need.head(150 * 3).tail(150) + # need = need.head(150 * 4).tail(150) + need['collection'] = need.collection.apply(lambda x: re.sub('\|', '-', x).strip() ) + need['collection'] = need.collection.apply(lambda x: re.sub('\)', '', x).strip() ) + need['collection'] = need.collection.apply(lambda x: re.sub('\(', '', x).strip() ) + need['collection'] = need.collection.apply(lambda x: re.sub('\'', '', x).strip() ) + need.collection.unique() + it = 0 + a = [] + print(i) + for row in need.iterrows(): + it += 1 + # if it < 20: + # continue + # if it % 100 == 0: + # print('#{}/{}'.format(it, len(m_df))) + print('#{}/{}'.format(it, len(need))) + row = row[1] + collection = row['collection'] + if collection in seen: + continue + update_authority = row['update_authority'] + print('Working on {}...'.format(collection)) + collection_dir = re.sub(' |-', '_', collection) + + dir = './data/mints/{}/'.format(collection_dir) + mfile = '{}{}_mint_accounts.json'.format(dir, update_authority) + if not os.path.exists(dir): + print(collection) + os.makedirs(dir) + # elif len(os.listdir(dir)) and os.path.exists(mfile): + # print('Already have {}.'.format(collection)) + # print('Seen') + # continue + print('LETS GOOO') + a.append(update_authority) + os.system('metaboss -r {} -t 300 snapshot mints --update-authority {} --output {}'.format(rpc, update_authority, dir)) + + # len(need) + # len(need.drop_duplicates(subset=['mint_address'])) + # len(need.collection.unique()) + # tot = len(need.collection.unique()) + # it = 0 + # # for each collection, get all the mints from metaboss + # for c in need.collection.unique(): + # it += 1 + # print('#{} / {}: {}'.format(it, tot, c)) + # dir = './data/fix_labels_1/{}/'.format(re.sub(' ', '_', c)) + odir = dir+'output/' + # if not os.path.exists(dir): + # print('Making dir {}'.format(dir)) + # os.makedirs(dir) + if not os.path.exists(odir): + print('Making dir {}'.format(odir)) + os.makedirs(odir) + # elif os.path.exists(dir+'mints.json'): + # print('Already Seen') + # continue + # ms = list(need[need.collection == c].mint_address.unique()) + # with open(dir+'mints.json', 'w') as f: + # json.dump(ms, f) + os.system('metaboss -r {} -t 300 decode mint --list-file {} --output {}'.format(rpc, mfile, odir )) + + ################################################## + # Load All The Mints for Each Collection # + ################################################## + # now that we have the mints, create a data frame with the info for each mint in each collection + data = [] + seen = [ x[1] for x in data ] + it = 0 + dirs = os.listdir('./data/mints/') + for path in dirs: + print(it) + it += 1 + if os.path.isdir('./data/mints/'+path): + collection = re.sub('_', ' ', path).strip() + if not os.path.exists('./data/mints/'+path+'/output/'): + continue + fnames = os.listdir('./data/mints/'+path+'/output/') + print(collection, len(fnames)) + for fname in fnames: + f = './data/mints/'+path+'/output/'+fname + if fname[:-5] in seen: + continue + if os.path.isfile(f) and '.json' in f: + try: + with open(f) as file: + j = json.load(file) + data += [[ collection, fname, j['name'], j['symbol'], j['uri'] ]] + except: + print('Error {}'.format(fname[:-5])) + + ################################################## + # Load All The Mints for Each Collection # + ################################################## + new_mints = pd.DataFrame(data, columns=['collection','mint_address','name','symbol','uri']) + # tmp = tmp[-(tmp.collection.isin(['Dskullys','Decimusdynamics']))] + n = len(new_mints[(new_mints.uri.isnull()) | (new_mints.uri == '')]) + tot = len(new_mints) + pct = round(n * 100 / tot, 1) + print('{} ({}%) rows have no uri'.format(n, pct)) + new_mints = new_mints[new_mints.uri != ''] + + # function to clean the name of each NFT (remove the number) + def f_cn(x): + if not x or x != x: + return(x) + if '#' in x[-6:]: + x = ''.join(re.split('#', x)[:-1]).strip() + elif bool(re.match('.+\s+[0-9]+', x)): + x = ' '.join(re.split(' ', x)[:-1]).strip() + return(x) + new_mints['clean_name'] = new_mints.name.apply(lambda x: f_cn(x) ) + + # determine for each collection if we should look at collection-name-symbol, collection-symbol, or just collection to determine what collection it actuallly belongs to + # this logic is because e.g. some only have a few names in the collection so we can iterate, but some have a different name for each NFT, so we assume its the same collection for all + a = new_mints.drop_duplicates(subset=['collection','clean_name','symbol']).groupby(['collection']).uri.count().reset_index().sort_values('uri', ascending=0) + symbol_only = a[a.uri > 10].collection.unique() + b = new_mints[new_mints.collection.isin(symbol_only)].drop_duplicates(subset=['collection','symbol']).groupby(['collection']).uri.count().reset_index().sort_values('uri', ascending=0) + collection_only = b[b.uri > 10].collection.unique() + + # now get the info for each collection-name-symbol combo + g1 = new_mints[ (-(new_mints.collection.isin(symbol_only))) & (-(new_mints.collection.isin(collection_only))) ].groupby(['collection','clean_name','symbol']).head(1).reset_index() + g2 = new_mints[ ((new_mints.collection.isin(symbol_only))) & (-(new_mints.collection.isin(collection_only))) ].groupby(['collection','symbol']).head(1).reset_index() + g3 = new_mints[ (-(new_mints.collection.isin(symbol_only))) & ((new_mints.collection.isin(collection_only))) ].groupby(['collection']).head(1).reset_index() + g = g1.append(g2).append(g3).drop_duplicates(subset=['mint_address']) + print('{} Total: {} all, {} collection-symbol {} collection'.format(len(g), len(g1), len(g2), len(g3))) + g.to_csv('~/Downloads/tmp-g.csv', index=False) + + # iterate over each row to get what collection they are actually in + # by pulling data from the uri + uri_data = [] + it = 0 + tot = len(g) + print(tot) + errs = [] + seen = [ x['uri'] for x in uri_data ] + # for row in g.iterrows(): + for row in g[ -(g.uri.isin(seen)) ].iterrows(): + row = row[1] + it += 1 + if it % 100 == 0: + uri_df = pd.DataFrame(uri_data)[[ 'collection','name','symbol','row_symbol','row_collection','uri','row_clean_name','mint_address' ]] + uri_df.to_csv('~/Downloads/uri_df.csv', index=False) + print('#{} / {}: {}'.format(it, tot, row['collection'])) + try: + r = requests.get(row['uri']) + j = r.json() + j['uri'] = row['uri'] + j['row_collection'] = row['collection'] + j['row_clean_name'] = row['clean_name'] + j['row_symbol'] = row['symbol'] + j['mint_address'] = row['mint_address'] + uri_data += [j] + except: + print('Error') + errs.append(row) + uri_df = pd.DataFrame(uri_data)[[ 'collection','name','symbol','row_symbol','row_collection','uri','row_clean_name','mint_address' ]] + uri_df.to_csv('~/Downloads/uri_df.csv', index=False) + + # for each row, parse the json from the uri + uri_df = pd.read_csv('~/Downloads/uri_df.csv') + def f(x, c): + x = str(x) + try: + n = json.loads(re.sub("'", "\"", x))[c] + if type(n) == list: + return(n[0]) + return(n) + except: + try: + return(json.loads(re.sub("'", "\"", x))[c]) + except: + try: + return(json.loads(re.sub("'", "\"", x))[0][c]) + except: + try: + return(json.loads(re.sub("'", "\"", x))[0]) + except: + return(x) + # parse the json more + uri_df['parsed_collection'] = uri_df.collection.apply(lambda x: f(x, 'name') ) + uri_df['parsed_family'] = uri_df.collection.apply(lambda x: f(x, 'family') ) + uri_df['clean_name'] = uri_df.name.apply( lambda x: f_cn(x) ) + # calculate what the collection name is + uri_df['use_collection'] = uri_df.parsed_collection.replace('', None).fillna( uri_df.clean_name )#.fillna( uri_df.row_symbol ) + uri_df[uri_df.use_collection == 'nan'][['use_collection','parsed_collection','parsed_family','clean_name','name','collection','symbol','row_symbol','row_collection']].head() + uri_df[uri_df.use_collection == 'nan'][['use_collection','parsed_collection','parsed_family','clean_name','name','collection','symbol','row_symbol','row_collection']].to_csv('~/Downloads/tmp.csv', index=False) + len(uri_df) + + # clean the collection name + def f1(x): + try: + if len(x['use_collection']) == 1: + return(x['clean_name']) + if bool(re.match('.+\s+#[0-9]+', x['use_collection'])): + return(''.join(re.split('#', x['use_collection'])[:-1]).strip()) + if '{' in x['use_collection']: + return(x['clean_name']) + return(x['use_collection'].strip().title()) + except: + return(x['use_collection'].strip().title()) + uri_df['tmp'] = uri_df.apply(lambda x: f1(x), 1 ) + uri_df[uri_df.tmp == 'Nan']['use_collection','tmp'] + uri_df['use_collection'] = uri_df.apply(lambda x: f1(x), 1 ) + sorted(uri_df.use_collection.unique())[:20] + sorted(uri_df.use_collection.unique())[-20:] + + # clean the mint_address + uri_df['mint_address'] = uri_df.mint_address.apply(lambda x: re.sub('.json','', x)) + uri_df.head() + uri_df = uri_df.fillna('None') + + for i in range(2): + # for each collection-name-symbol combo, see how many have multiple mappings + a = uri_df.copy().fillna('None') + a = a[['row_collection','row_clean_name','row_symbol','use_collection']].drop_duplicates().groupby(['row_collection','row_clean_name','row_symbol']).use_collection.count().reset_index().rename(columns={'use_collection':'n_1'}) + uri_df = merge(uri_df, a, ensure=True) + + # for each collection-symbol combo, see how many have multiple mappings + a = uri_df.copy().fillna('None') + a = a[['row_collection','row_symbol','use_collection']].drop_duplicates().groupby(['row_collection','row_symbol']).use_collection.count().reset_index().rename(columns={'use_collection':'n_2'}) + uri_df = merge(uri_df, a, ensure=True) + + # for each collection combo, see how many have multiple mappings + a = uri_df.copy().fillna('None') + a = a[['row_collection','use_collection']].drop_duplicates().groupby(['row_collection']).use_collection.count().reset_index().rename(columns={'use_collection':'n_3'}) + uri_df = merge(uri_df, a, ensure=True) + + uri_df['n'] = uri_df.apply(lambda x: x['n_3'] if x['row_collection'] in collection_only else x['n_2'] if x['row_collection'] in symbol_only else x['n_1'], 1 ) + print('{} / {} ({}%) have multiple collection-name-symbol mappings'.format(len(uri_df[uri_df.n > 1]), len(uri_df), round( 100.0 * len(uri_df[uri_df.n > 1]) / len(uri_df)))) + + # if there is multiple, use the parsed_family instead of the use_collection + uri_df['use_collection'] = uri_df.apply(lambda x: x['use_collection'] if x['n'] == 1 else x['parsed_family'], 1 ) + del uri_df['n_1'] + del uri_df['n_2'] + del uri_df['n_3'] + + # only take rows where there is a single mapping + m = uri_df[uri_df.n==1][[ 'use_collection','row_collection','row_clean_name','row_symbol' ]].dropna().drop_duplicates() + m.columns = [ 'use_collection','collection','clean_name','symbol' ] + + m_1 = new_mints[ (-(new_mints.collection.isin(symbol_only))) & (-(new_mints.collection.isin(collection_only))) ].fillna('').merge(m.fillna(''), how='left') + m_2 = new_mints[ ((new_mints.collection.isin(symbol_only))) & (-(new_mints.collection.isin(collection_only))) ][[ 'collection','mint_address','symbol' ]].fillna('').merge(m.fillna(''), how='left') + m_3 = new_mints[ (-(new_mints.collection.isin(symbol_only))) & ((new_mints.collection.isin(collection_only))) ][[ 'collection','mint_address' ]].fillna('').merge(m.fillna(''), how='left') + len(m_1) + len(m_2) + len(m_3) + len(new_mints) + # m = new_mints.fillna('').merge(m.fillna(''), how='left') + m = m_1.append(m_2).append(m_3) + print('After all this, we have {}% of the mints'.format( round(len(m) * 100 / len(new_mints)) )) + len(new_mints) + len(m) + m['mint_address'] = m.mint_address.apply(lambda x: re.sub('.json', '', x) ) + m = m[['mint_address','use_collection']].dropna().drop_duplicates() + m.columns = ['mint_address','collection'] + + m[m.collection.isnull()].head() + m[m.collection=='Nan'].head() + + m = m[m.collection != 'Nan'] + + tmp = m.groupby('collection').mint_address.count().reset_index().sort_values('mint_address', ascending=0) + tmp.head() + + m.to_csv('./data/mult_update_auth_labels.csv', index=False) + ################ + # DONE # + ################ + + + + tokens = m.append(pd.read_csv('./data/tokens.csv')[['collection','mint_address']]).drop_duplicates(subset=['mint_address'], keep='last') + tokens.to_csv('./data/mints-2022-06-13-2pm.csv', index=False) + + tokens.head() + + m.to_csv('./data/mints-2022-06-09.csv', index=False) + m = pd.read_csv('./data/mints-2022-06-09.csv') + m.groupby('collection').head(1).to_csv('~/Downloads/tmp.csv', index=False) + len(m) + len(m.mint_address.unique()) + m.head() + m.head() + # m = m.merge(symbol_map, how='left', on='symbol') + # m['use_collection'] = m.use_collection_x.fillna(m.use_collection_y) + len(new_mints) + len(m) + len(m[m.use_collection.isnull()]) + len(m[m.use_collection.isnull()]) / len(m) + len(m[m.use_collection_x.isnull()]) / len(m) + m[m.use_collection.isnull()].fillna('').drop_duplicates(subset=['collection','clean_name','symbol']).to_csv('~/Downloads/tmp-3.csv', index=False) + m[m.use_collection.isnull()].drop_duplicates(subset=['collection']).to_csv('~/Downloads/tmp-3.csv', index=False) + + a = uri_df[(uri_df.parsed_collection.isnull()) | (uri_df.parsed_collection == '')].groupby('row_clean_name').uri.count().reset_index() + a = uri_df[(uri_df.parsed_collection.isnull()) | (uri_df.parsed_collection == '')] + uri_df.head() + uri_df['row_clean_name'] = uri_df.row_clean_name.apply(lambda x: f_cn(x) ) + id_map = uri_df + a.to_csv('~/Downloads/tmp-1.csv', index=False) + len(uri_df) + n = uri_df.groupby() + uri_df + uri_df + uri_df.head() + uri_df[['symbol','collection','']] + uri_df.head() query = ''' SELECT DISTINCT project_name @@ -294,6 +1057,26 @@ def mints_from_me(): [x for x in seen if not x in m_df.tmp.unique()][:11] m_df[m_df.symbol == 'apesquad'] m_df[m_df.symbol == 'chimp_frens'] + url = 'https://api.solscan.io/nft/detail?mint=D5pT5HYPeQkHD6ryoHxnc2jdcUMYmjs6sS6LswbSDsuy' + us = sorted(m_df[m_df.n_collection > 1].update_authority.unique()) + u = us[1] + m_df[m_df.update_authority == u] + m_df[m_df.mint == 'G3xiAFZEp49BJc8nNrDJxwTXZ34teKH7CRf5KTGakxte'] + data = [] + for u in us[:10]: + nfts = BLOCKCHAIN_API_RESOURCE.search_nfts( + update_authority = u + , update_authority_search_method = SearchMethod.EXACT_MATCH + ) + print(u, len(nfts)) + for n in nfts: + m = n['nft_metadata'] + data += [[ m['update_authority'], m['mint'], m['data']['symbol'], m['data']['name'] ]] + nft_df = pd.DataFrame(data, columns=['update_authority','mint','symbol','name']) + len(nft_df.update_authority.unique()) + nft_df['collection'] = nft_df.name.apply(lambda x: re.split('#', x)[0].strip() ) + nft_df.groupby(['symbol','collection']).mint.count() + nft_df.groupby(['symbol','name']).mint.count() print(len(seen)) # m_df = m_df.merge(lp_df) len(m_df) @@ -335,7 +1118,6 @@ def mints_from_me(): # os.makedirs(dir_mints) # os.system('metaboss -r {} -t 300 decode mint --list-file {} --output {}'.format(rpc, fname, dir_mints)) - data = [] for path in os.listdir('./data/mints/'): if os.path.isdir('./data/mints/'+path): diff --git a/objects/saved_params.pickle b/objects/saved_params.pickle index b51eb948be82f7c81e1cd1d8d779ea5973bedc57..112e785a59a56b664e1cc340105a387f0189fa10 100644 GIT binary patch literal 9102 zcmc&)TWB0r7*0aBZ8mArOWG!4>)6^0f~8HtOCOre?wVCMNmG*|zI1z%J>8kv*_mZ7 z+N3B#tN1VoVp=Z`N*@c>7ZIxXkQYIGvv68ah`hyEf$B}UcH1ZTd5)2Fv%Nfx0gP1 z79-m+$rF=Ka6a|m2=Ra#cl7GxhffWoM%($v+;+!ZmmPgMmk@}Lub;g2=I-l_o6%^= zgN4lQGb|DbftYM74&_$jFjyFQ=*gS69?m_{XgZ4n`8bTj`aCWeWh*uka;lhW+<`ft zDrBzSyI@Kk9Wl*qIsrsMVnW=uJH6|zEH^bx=WMHdvkyo=;_H_7)SB;OLr z_oL+Xp16Du7LNbxY@Rr^O99M|z;f$2eg)}B%A2TS=+d*53T%2pnGR$c;utY~GGh6V z$np{35FT)Th~svTJ37ib8kli0P!nPuHMlG^Sbop`T7CT<5b6M31R+z{s?JMOSg$yW z4uqLdE3hFa1eguwG(p5FF#6>jm79dxS|M0!YMF{DzOBR!BqHqsOVm6v3sVQE4+hfTovVi$yu zD__K{ZB1{KPf9P_Bu9eXWo=^VCfs-1o_~ zR9iBoCAoG@EK`W>`@f~8({OCcVcU{U=(JYmstFMh0dT318UI0IyEgZ-RgB2xR)pIU z*c%wEm!iV0H;=~@X-YAX4Qrw#(l(F(l%WA`4h13YDC>!9QLMUyfIIHM1jc3+*`x?H zO)+$&P%R&aZz|{jwcry;pcWF17i!Q1>I+VB+zrzw#zk5grC655#j+eNmJOqpZ6ZLDg;aWR;pf6<1LxycL9z801_fwYvv z*_btK={bzbu|e@39}#q1e?wT6se4GsifMH(UAolA=+dd~qf1%c?|pWFEHM`-|8$_c zMcm<9#c{V~o!MCpR}90#QWj|%2IPcz??|q>f^Ft9foGRj<{5yrh-X|o5`9Eeos>hS zjWtoKfH_FSA()&!hf-~%N-*B=ffADT_MBoX3s|?L?7T$oHkM6L7~dPi+62Y`%)J?c zqCy6CZxsovrQ&4vjR2I(a_)H$-aQ6$My=-P%F$!e1S(6@a|+gN#j@k0YR>W+xD#^m zhH=BN6+pLh~OqVWo3tc+Zt#lbyx6vi74v-~I-Fl}3`DNhoP7n3K))`Z& z*A@K$o1&5zvpC){c1?k#Mx-hd_%vR&uTb0yW+XljhSRS8tY*wrt(s!X(m0xp*Ed`a z0NnaJ3G*7szgDJyDzguoZnJcI9)$X+Cyy4$#60hi(8TB2+Y^8g%d@v9uhO8VLQ4OW zf;6T6MVBu1Z@P4)y9MM?8`%=h2E2TRUzJn#oVGE* zp1d5v?`Bz!_I1&!gb8#N)p;xbI-oN9ps6vdO3wp6@*%n;_=34WCi4c(wYzc3r2n`9 jD8iLR>KKbulQs-0brmDk%B)^x_KAka-V@>c*g5ncUk;py delta 17 YcmeBk@04S0V43=lZzC%cKU1k505gyU6951J diff --git a/prepare_data.py b/prepare_data.py index ae9b66dd..76d68253 100644 --- a/prepare_data.py +++ b/prepare_data.py @@ -1,12 +1,132 @@ import re import os +import json import pandas as pd +import snowflake.connector os.chdir('/Users/kellenblumberg/git/nft-deal-score') from solana_model import get_sales from scrape_sol_nfts import clean_name +def get_ctx(): + usr = os.getenv('SNOWFLAKE_USR') + pwd = os.getenv('SNOWFLAKE_PWD') + # with open('snowflake.pwd', 'r') as f: + # pwd = f.readlines()[0].strip() + # with open('snowflake.usr', 'r') as f: + # usr = f.readlines()[0].strip() + + ctx = snowflake.connector.connect( + user=usr, + password=pwd, + account='vna27887.us-east-1' + ) + return(ctx) + +def clean_colnames(df): + names = [ x.lower() for x in df.columns ] + df.columns = names + return(df) + +def overlap(): + query = ''' + WITH sales AS ( + SELECT l.label AS collection, SUM(sales_amount) AS volume, MIN(block_timestamp::date) AS first_sale_date + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON LOWER(l.address) = LOWER(s.mint) + WHERE block_timestamp >= CURRENT_DATE - 30 + GROUP BY 1 + ), base AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY volume DESC) AS volume_rank + FROM sales + ORDER BY volume DESC + LIMIT 50 + ), b2 AS ( + SELECT DISTINCT collection, first_sale_date, volume_rank, purchaser, mint + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON LOWER(l.address) = LOWER(s.mint) + JOIN base b ON b.collection = l.label + UNION + SELECT DISTINCT collection, first_sale_date, volume_rank, purchaser, mint + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON LOWER(l.address) = LOWER(m.mint) + JOIN base b ON b.collection = l.label + ) + SELECT DISTINCT INITCAP(collection) AS collection, first_sale_date, date_trunc('month', first_sale_date) AS first_sale_month, volume_rank, purchaser, mint + FROM b2 + ''' + ctx = get_ctx() + df = ctx.cursor().execute(query) + df = pd.DataFrame.from_records(iter(df), columns=[x[0] for x in df.description]) + df = clean_colnames(df) + df[df.collection == 'okay bears'] + len(df[df.collection == 'okay bears'].mint.unique()) + data = [] + list(df.collection.unique()).index(a) + list(df.collection.unique()).index(b) + cur = df[df.volume_rank <= 50] + for a in cur.collection.unique(): + print(a) + a1 = set(cur[cur.collection == a].purchaser.unique()) + ar = cur[cur.collection == a].volume_rank.values[0] + am = cur[cur.collection == a].first_sale_month.values[0] + # for b in cur[cur.collection > a].collection.unique(): + for b in cur.collection.unique(): + b1 = set(cur[cur.collection == b].purchaser.unique()) + br = cur[cur.collection == b].volume_rank.values[0] + bm = cur[cur.collection == b].first_sale_month.values[0] + data += [[ a, b, int(a < b), am, bm, ar, br, len(a1), len(b1), len(a1.intersection(b1)) ]] + cur = pd.DataFrame(data, columns=['col_1','col_2','include','am','bm','r_1','r_2','n_1','n_2','n_int']) + cur['pct'] = cur.apply(lambda x: x['n_int'] / min(x['n_1'], x['n_2']), 1 ) + cur = cur[cur.n_int.notnull()] + cur.to_csv('~/Downloads/overlap.csv', index=False) + cur.include.unique() + + +def add_back_metadata(): + query = ''' + SELECT * + FROM solana.dim_nft_metadata + WHERE LOWER(project_name) IN ( + 'degods' + , 'astrals' + , 'solstein' + , 'solgods' + , 'okay bears' + , 'meerkat millionaires' + , 'catalina whale mixer' + , 'citizens by solsteads' + , 'defi pirates' + ) + ''' + ctx = get_ctx() + mdf = ctx.cursor().execute(query) + mdf = pd.DataFrame.from_records(iter(mdf), columns=[x[0] for x in mdf.description]) + print('Loaded {} metadata'.format(len(mdf))) + mdf = clean_colnames(mdf) + mdf = mdf[[ 'contract_name','token_id','token_metadata' ]] + m = json.loads(mdf.token_metadata.values) + m = [json.loads(x) for x in mdf.token_metadata.values] + data = [] + collection = mdf.contract_name.values + token_id = mdf.token_id.values + for i in range(len(m)): + for k, v in m[i].items(): + data += [[ collection[i], token_id[i], k, v ]] + old = pd.read_csv('./data/metadata.csv') + metadata = pd.DataFrame(data, columns=['collection','token_id','feature_name','feature_value']) + del old['chain'] + old = old.append(metadata) + old['collection'] = old.collection.apply(lambda x: clean_name(x) ) + old = old.drop_duplicates(subset=['collection','token_id','feature_name'], keep='last') + old[old.collection == 'Cets On Creck'].feature_name.unique() + old[old.collection == 'Cets on Creck'].feature_name.unique() + tmp = old[['collection','feature_name']].drop_duplicates().groupby('collection').feature_name.count().reset_index() + tmp.to_csv('~/Downloads/tmp-1.csv', index=False) + old.to_csv('./data/metadata.csv', index=False) + def add_sf_metadata(): old = pd.read_csv('./data/metadata.csv') l0 = len(old) @@ -141,8 +261,25 @@ def add_att_count(): print('Adding {} rows'.format(l1 - l0)) m_df.to_csv('./data/metadata.csv', index=False) + +def tmp(): + m1 = pd.read_csv('./data/metadata.csv') + m2 = pd.read_csv('./data/metadata_2.csv') + t1 = pd.read_csv('./data/tokens.csv') + t2 = pd.read_csv('./data/tokens_2.csv') + m = m1.append(m2).drop_duplicates(keep='last') + t = t1.append(t2).drop_duplicates(keep='last') + t.to_csv('./data/tokens.csv', index=False) + m.to_csv('./data/metadata.csv', index=False) + def add_rarities(): + include = [ 'DeGods' ] m_df = pd.read_csv('./data/metadata.csv') + # m_df = m_df[-m_df.collection.isin([''])] + g0 = m_df.groupby('collection').token_id.count().reset_index() + + m_df['collection'] = m_df.collection.apply(lambda x: clean_name(x)) + # m_df = m_df[m_df.collection.isin(include)] # m_df['feature_name'] = m_df.feature_name.fillna(m_df.name) # m_df['feature_value'] = m_df.feature_value.fillna(m_df.value) for c in [ 'name','value','rarity' ]: @@ -164,6 +301,8 @@ def add_rarities(): # m_df[m_df.collection == 'BAYC'].feature_name.unique() tokens = pd.read_csv('./data/tokens.csv')[['collection','token_id','nft_rank']] + tokens['collection'] = tokens.collection.apply(lambda x: clean_name(x)) + # tokens = tokens[tokens.collection.isin(include)] tokens[((tokens.collection == 'Pesky Penguins')) & (tokens.token_id=='6437')] tokens[((tokens.collection == 'Pesky Penguins')) & (tokens.token_id==6437)] tokens[tokens.collection == 'SOLGods'] @@ -287,6 +426,10 @@ def add_rarities(): sorted(m_df.collection.unique()) l1 = len(m_df) + g1 = m_df.groupby('collection').token_id.count().reset_index() + g = g0.merge(g1, how='outer', on=['collection']) + g['dff'] = g.token_id_y - g.token_id_x + print(g[g.dff != 0].sort_values('dff', ascending=0)) print('Adding {} rows'.format(l1 - l0)) # m_df[m_df.collection == 'Galactic Angels'] # m_df[ (m_df.collection == 'Galactic Angels') & (m_df.token_id == '1') ] diff --git a/scrape_sol_nfts.py b/scrape_sol_nfts.py index 2a9ea4b8..bc9d72c4 100644 --- a/scrape_sol_nfts.py +++ b/scrape_sol_nfts.py @@ -23,7 +23,7 @@ import cloudscraper os.chdir('/Users/kellenblumberg/git/nft-deal-score') os.environ['PATH'] += os.pathsep + '/Users/kellenblumberg/shared/' -from utils import clean_token_id, merge, clean_name +from utils import clean_token_id, get_ctx, merge, clean_name # howrare.is api # https://api.howrare.is/v0.1/collections/smb/only_rarity @@ -34,66 +34,28 @@ from utils import clean_token_id, merge, clean_name # old = pd.read_csv('./data/tokens.csv') # metadata[(metadata.collection == 'Galactic Punks') & (metadata.feature_name=='attribute_count')].drop_duplicates(subset=['feature_value']).merge(old) +# url = 'https://api.solscan.io/collection/nft?sortBy=nameDec&collectionId=f046bec0889c9d431ce124a626237e2236bc2527051d32ed31f6b5e6dc230669&offset=0&limit=500' +# r = requests.get(url) +# j = r.json() +# j.keys() +# len(j['data']) +# j['data'][0] -def how_rare_is_api(): - url = 'https://api.howrare.is/v0.1/collections' - r = requests.get(url) - j = r.json() - j['result'].keys() - j['result']['data'][:10] - c_df = pd.DataFrame(j['result']['data']).sort_values('floor_marketcap', ascending=0) - c_df.head(16) - seen = [ 'smb','aurory','degenapes','thugbirdz','degods','okay_bears','catalinawhalemixer','cetsoncreck','stonedapecrew','solgods' ] - len(j['result']['data']) - t_data = [] - metadata = pd.DataFrame() - d = { - 'Degen Apes': 'degenapes' - , 'Pesky Penguins': 'peskypenguinclub' - , 'Aurory': 'aurory' - , 'Solana Monkey Business': 'smb' - , 'Thugbirdz': 'thugbirdz' - } - # for collection, url in d.items(): - # redo trippin ape tribe - for row in c_df.iterrows(): - row = row[1] - collection = row['name'] - url = row['url'][1:] - print('Working on collection {}, {}, {}'.format(collection, len(t_data), len(metadata))) - if url in seen or (len(metadata) and collection in metadata.collection.unique()): - print('Seen!') - continue - # collection = 'Cets on Creck' - # collection = 'SOLGods' - # collection = 'Meerkat Millionaires' - # collection = d['url'][1:] - # url = 'https://api.howrare.is/v0.1/collections'+d['url'] - # url = 'https://api.howrare.is/v0.1/collections/meerkatmillionaires' - url = 'https://api.howrare.is/v0.1/collections/'+url - r = requests.get(url) - j = r.json() - for i in j['result']['data']['items']: - token_id = int(i['id']) - nft_rank = int(i['rank']) - mint = i['mint'] - image = i['image'] - t_data += [[ collection, token_id, nft_rank, mint, image ]] - # m_data += [[ collection, token_id, nft_rank ]] - m = pd.DataFrame(i['attributes']) - m['token_id'] = token_id - m['collection'] = collection - # metadata = metadata.append(m) - metadata = pd.concat([metadata, m]) - old = pd.read_csv('./data/tokens.csv') + +def add_to_df(t_data): + old = pd.read_csv('./data/tokens_2.csv') sorted(old.collection.unique()) l0 = len(old) do_merge = False tokens = pd.DataFrame(t_data, columns=['collection','token_id','nft_rank','mint_address','image_url']) + len(tokens) + tokens[tokens.nft_rank.isnull()] tokens['collection'] = tokens.collection.apply(lambda x: 'Catalina Whale Mixer' if x == 'Catalina Whales' else x ) - metadata['collection'] = metadata.collection.apply(lambda x: 'Catalina Whale Mixer' if x == 'Catalina Whales' else x ) + rem = [ 'Jikan Studios','Fine Fillies' ] + print(tokens.groupby('collection').token_id.count()) tokens['clean_token_id'] = tokens.token_id tokens['chain'] = 'Solana' + tokens = tokens[-tokens.collection.isin(rem)] if do_merge: old['token_id'] = old.token_id.astype(str) tokens['token_id'] = tokens.token_id.astype(str) @@ -106,42 +68,230 @@ def how_rare_is_api(): old['clean_token_id'] = old.clean_token_id.fillna(old.token_id) old['chain'] = old.chain.fillna('Solana') else: - old = old.append(tokens) + # old = old.append(tokens) + old = pd.concat( [old, tokens] ) old['token_id'] = old.token_id.astype(str) old = old.drop_duplicates(subset=['collection','token_id'], keep='last') print('Adding {} rows'.format(len(old) - l0)) old[old.collection.isin(tokens.collection.unique())] old[(old.collection.isin(tokens.collection.unique())) & (old.token_id == '6437')] old[old.nft_rank.isnull()].groupby('collection').token_id.count() + old = old[-old.collection.isin(['Astrals','Dazedducks','Nyanheroes','Shadowysupercoder','Taiyorobotics'])] + old.to_csv('./data/tokens_2.csv', index=False) + # tokens.to_csv('./data/tokens_2.csv', index=False) + +def compile(): + ctx = get_ctx() + query = 'SELECT DISTINCT address FROM silver_CROSSCHAIN.ADDRESS_LABELS' + seen = ctx.cursor().execute(query) + seen = pd.DataFrame.from_records(iter(seen), columns=[x[0] for x in seen.description]) + seen = sorted(list(seen.ADDRESS.unique())) + + tokens = pd.read_csv('./data/tokens.csv') + tokens = tokens[tokens.chain == 'Solana'] + single_update_auth_labels = pd.read_csv('./data/single_update_auth_labels.csv') + mult_update_auth_labels = pd.read_csv('./data/mult_update_auth_labels.csv') + df = tokens.append(single_update_auth_labels).append(mult_update_auth_labels) + df = df[ (df.collection != 'Nan') & (df.collection != 'nan') & (df.collection.notnull()) ] + df = df[-(df.mint_address.isin(seen))] + df = df.drop_duplicates(subset=['mint_address'], keep='first') + # len(df) + # len(df.collection.unique()) + # df.head() + # df.mint_address.tail(11000).head(5) + # df[df.mint_address == '2GgPNKGyzAQL4mriuH4kBpntYCNVSM2pQfzdsu3p8du5'] + # df['seen'] = df.mint_address.isin(seen).astype(int) + # tmp = df[df.seen == 0].groupby('collection').mint_address.count().reset_index().sort_values('mint_address', ascending=0) + # tmp.head(40) + # tmp.mint_address.sum() + df[df.mint_address.isnull()] + df[['mint_address','collection']].to_csv('~/Downloads/solana-nft-labels-06-29.csv', index=False) + + +def add_to_df(t_data, metadata, exclude_new = False): + old = pd.read_csv('./data/tokens.csv') + g0 = old.groupby('collection').token_id.count().reset_index() + sorted(old.collection.unique()) + l0 = len(old) + do_merge = False + tokens = pd.DataFrame(t_data, columns=['collection','token_id','nft_rank','mint_address','image_url']) + len(tokens) + tokens[tokens.nft_rank.isnull()] + tokens['collection'] = tokens.collection.apply(lambda x: 'Catalina Whale Mixer' if x == 'Catalina Whales' else x ) + # rem = [ 'Jikan Studios','Fine Fillies' ] + # print(tokens.groupby('collection').token_id.count()) + metadata['collection'] = metadata.collection.apply(lambda x: 'Catalina Whale Mixer' if x == 'Catalina Whales' else x ) + tokens['clean_token_id'] = tokens.token_id + tokens['chain'] = 'Solana' + # tokens = tokens[-tokens.collection.isin(rem)] + # metadata = metadata[-metadata.collection.isin(rem)] + if do_merge: + old['token_id'] = old.token_id.astype(str) + tokens['token_id'] = tokens.token_id.astype(str) + old = old.merge(tokens, how='left', on=['collection','token_id']) + old[old.collection == 'Solana Monkey Business'] + for c in [ 'nft_rank','mint_address','image_url' ]: + old[c] = old[c+'_x'].fillna(old[c+'_y']) + del old[c+'_x'] + del old[c+'_y'] + old['clean_token_id'] = old.clean_token_id.fillna(old.token_id) + old['chain'] = old.chain.fillna('Solana') + else: + # old = old.append(tokens) + old['collection'] = old.collection.apply(lambda x: clean_name(x)) + tokens['collection'] = tokens.collection.apply(lambda x: clean_name(x)) + if exclude_new: + rem = tokens.collection.unique() + old = old[-(old.collection.isin(rem))] + old = pd.concat( [old, tokens] ) + old['token_id'] = old.token_id.astype(str) + old = old.drop_duplicates(subset=['collection','token_id'], keep='last') + g1 = old.groupby('collection').token_id.count().reset_index() + g = g0.merge(g1, how='outer', on=['collection']).fillna(0) + g['dff'] = g.token_id_y - g.token_id_x + print(g[g.dff != 0].sort_values('dff', ascending=0)) + g[g.dff != 0].sort_values('dff', ascending=0).to_csv('~/Downloads/tmp.csv', index=False) + print('Adding {} rows'.format(len(old) - l0)) + old = old[old.collection != 'Solanamonkeybusiness (Smb)'] + # old[old.collection.isin(tokens.collection.unique())] + # old[(old.collection.isin(tokens.collection.unique())) & (old.token_id == '6437')] + old[old.nft_rank.isnull()].groupby('collection').token_id.count() + # old = old[-old.collection.isin(['Astrals','Dazedducks','Nyanheroes','Shadowysupercoder','Taiyorobotics'])] old.to_csv('./data/tokens.csv', index=False) + # tokens.to_csv('./data/tokens_2.csv', index=False) old = pd.read_csv('./data/metadata.csv') - a = old[['collection','token_id']].drop_duplicates() - a['exclude'] = 0 - a['token_id'] = a.token_id.astype(str) - metadata['token_id'] = metadata.token_id.astype(str) - m = metadata.merge(a, how='left') - m = m[m.exclude.isnull()] - len(m[m.exclude.isnull()].token_id.unique()) - del m['exclude'] - # old = old[-(old.collection == 'Meerkat Millionaires Cc')] - print(sorted(old.collection.unique())) + g0 = old.groupby('collection').token_id.count().reset_index() l0 = len(old) - metadata.collection.unique() + old['collection'] = old.collection.apply(lambda x: clean_name(x)) + metadata['collection'] = metadata.collection.apply(lambda x: clean_name(x)) + if exclude_new: + rem = metadata.collection.unique() + old = old[-(old.collection.isin(rem))] + # old = old[-old.collection.isin(['Astrals','Dazedducks','Nyanheroes','Shadowysupercoder','Taiyorobotics'])] + # a = old[['collection','token_id']].drop_duplicates() + # a['exclude'] = 0 + # a['token_id'] = a.token_id.astype(str) + # metadata['token_id'] = metadata.token_id.astype(str) + # m = metadata.merge(a, how='left') + # m = m[m.exclude.isnull()] + # len(m[m.exclude.isnull()].token_id.unique()) + # del m['exclude'] + # old = old[-(old.collection == 'Meerkat Millionaires Cc')] + # print(sorted(old.collection.unique())) + # metadata.collection.unique() # metadata = pd.DataFrame(t_data, columns=['collection','token_id','nft_rank','mint_address','image_url']) # old = old.merge(tokens, how='left', on=['collection','token_id']) - old = old.append(m[['collection','token_id','name','value']].rename(columns={'name':'feature_name','value':'feature_value'}) ) + # old = old.append(m[['collection','token_id','name','value']].rename(columns={'name':'feature_name','value':'feature_value'}) ) + old = pd.concat( [old, metadata[['collection','token_id','name','value']].rename(columns={'name':'feature_name','value':'feature_value'})] ) old['token_id'] = old.token_id.astype(str) - old = old.drop_duplicates(subset=['collection','token_id','feature_name']) + old = old.drop_duplicates(subset=['collection','token_id','feature_name'], keep='last') # old['nft_rank'] = old.nft_rank_y.fillna(old.nft_rank_y) # del old['nft_rank_x'] + g1 = old.groupby('collection').token_id.count().reset_index() + g = g0.merge(g1, how='outer', on=['collection']).fillna(0) + g['dff'] = g.token_id_y - g.token_id_x + print(g[g.dff != 0].sort_values('dff', ascending=0)) # del old['nft_rank_y'] print('Adding {} rows'.format(len(old) - l0)) - print(old.groupby('collection').token_id.count()) - old[old.collection.isin(metadata.collection.unique())] - old[(old.collection == 'Catalina Whale Mixer') & (old.token_id == '1206')] + # print(old.groupby('collection').token_id.count()) + # old[old.collection.isin(metadata.collection.unique())] + # old[(old.collection == 'Catalina Whale Mixer') & (old.token_id == '1206')] old.to_csv('./data/metadata.csv', index=False) + # metadata.to_csv('./data/metadata_2.csv', index=False) +def how_rare_is_api(): + ctx = get_ctx() + query = ''' + SELECT DISTINCT LOWER(project_name) AS lower_collection + FROM solana.core.dim_nft_metadata + ''' + df = ctx.cursor().execute(query) + df = pd.DataFrame.from_records(iter(df), columns=[x[0] for x in df.description]) + + url = 'https://api.howrare.is/v0.1/collections' + r = requests.get(url) + j = r.json() + j['result'].keys() + j['result']['data'][:10] + c_df = pd.DataFrame(j['result']['data']).sort_values('floor_marketcap', ascending=0) + c_df['lower_collection'] = c_df.url.apply(lambda x: x.lower().strip() ) + seen = sorted(df.LOWER_COLLECTION.apply(lambda x: re.sub(' |_|\'', '', x) ).values) + # seen[:300] + # x = 590 + # seen[x:x+50] + c_df['seen_1'] = c_df.url.apply(lambda x: re.sub(' |_|\'', '', x[1:]).lower() in seen ).astype(int) + c_df['seen_2'] = c_df.name.apply(lambda x: re.sub(' |_|\'', '', x).lower() in seen ).astype(int) + c_df['seen'] = (c_df.seen_1 + c_df.seen_2 > 0).astype(int) + c_df.head() + c_df.seen.sum() + c_df[c_df.seen == 0].head(10) + # c_df.head(16) + seen = [ 'smb','aurory','degenapes','thugbirdz','degods','okay_bears','catalinawhalemixer','cetsoncreck','stonedapecrew','solgods' ] + c_df = c_df[-(c_df.url.isin([ '/'+x for x in seen]))] + # rem = [ 'kaiju','jikanstudios' ] + # c_df = c_df[-(c_df.url.isin([ '/'+x for x in rem]))] + # seen = list(pd.read_csv('./data/tokens.csv').collection.unique()) + # c_df = c_df[-(c_df.name.isin(seen))] + # len(j['result']['data']) + # c_df = c_df[c_df.url.isin(['/blocksmithlabs'])] + # c_df = c_df[c_df.url.isin(['/generousrobotsdao','/thestonedfrogs'])] + c_df = c_df[c_df.seen == 0] + sorted(c_df.url.unique()) + it = 0 + tot = len(c_df) + # c_df.head() + # c_df = c_df[c_df.url != '/midnightpanthers'] + t_data = [] + m_data = [] + # metadata = pd.DataFrame() + for row in c_df.iterrows(): + it += 1 + row = row[1] + collection = row['name'] + print('#{} / {}: {}'.format(it, tot, collection)) + url = row['url'][1:] + if it > 1: + assert(len(t_data)) + assert(len(m_data)) + print('Working on collection {}, {}, {}'.format(collection, len(t_data), len(m_data))) + # if url in seen or (len(metadata) and collection in metadata.collection.unique()): + # print('Seen!') + # continue + # collection = 'Cets on Creck' + # collection = 'SOLGods' + # collection = 'Meerkat Millionaires' + # collection = d['url'][1:] + # url = 'https://api.howrare.is/v0.1/collections'+d['url'] + # url = 'https://api.howrare.is/v0.1/collections/meerkatmillionaires' + # url = 'https://api.howrare.is/v0.1/collections/'+url+'/only_rarity' + url = 'https://api.howrare.is/v0.1/collections/'+url + r = requests.get(url) + j = r.json() + for i in j['result']['data']['items']: + try: + token_id = int(i['id']) + if True: + nft_rank = int(i['rank']) + mint = i['mint'] + image = i['image'] + t_data += [[ collection, token_id, nft_rank, mint, image ]] + if False: + for d in i['attributes']: + d['token_id'] = token_id + d['collection'] = collection + m_data += [ d ] + # metadata = metadata.append(m) + # metadata = pd.concat([metadata, m]) + except: + print('Error') + # add_to_df(t_data) + metadata = pd.DataFrame(m_data) + metadata + + add_to_df(t_data, metadata, True) + metadata.head() + metadata.value.unique() def convert_collection_names(): for c in [ 'pred_price', 'attributes', 'feature_values', 'model_sales', 'listings', 'coefsdf', 'tokens' ]: diff --git a/scratch.py b/scratch.py index ecf5278f..5ea832e8 100644 --- a/scratch.py +++ b/scratch.py @@ -1,6 +1,6 @@ import os import json -# import psycopg2 +import psycopg2 import pandas as pd import requests @@ -26,6 +26,49 @@ def thorchain(): def f(): conn = psycopg2.connect("dbname=suppliers user=postgres password=postgres") + conn = psycopg2.connect("dbname=suppliers user=postgres password=postgres") + conn = psycopg2.connect( + host="vic5o0tw1w-repl.twtim97jsb.tsdb.cloud.timescale.com", + user="tsdbadmin", + password="yP4wU5bL0tI0kP3k" + ) + +query = ''' + SELECT from_addr + , to_addr + , asset + , amount_e8 + , block_timestamp + , COUNT(1) AS n + FROM midgard.transfer_events + WHERE block_timestamp < 1650000000000000000 + AND block_timestamp >= 1640000000000000000 + GROUP BY 1, 2, 3, 4, 5 + HAVING COUNT(1) > 1 +''' +df = pd.read_sql_query(query, conn) +cur.execute(query) + +it = 0 +qs = [] +for i in range(1618000000000000000, 1657000000000000000, 3000000000000000): + print(i) + it += 1 + query = ''' + SELECT from_addr + , to_addr + , asset + , amount_e8 + , block_timestamp + , COUNT(1) AS n + FROM midgard.transfer_events + WHERE block_timestamp >= {} + AND block_timestamp < {} + GROUP BY 1, 2, 3, 4, 5 + HAVING COUNT(1) > 1 + '''.format(i, i + 3000000000000000) + with open('/Users/kellenblumberg/Downloads/query_{}.txt'.format(it), 'w') as f: + f.write(query) def read_tokenlist(): diff --git a/scratch.sql b/scratch.sql index 6dd79948..b9b9109b 100644 --- a/scratch.sql +++ b/scratch.sql @@ -1,49 +1,2279 @@ +1. Build in Public (Bravery) +2. Zero assumptions -rsync -azvv --progress -e "ssh -i /Users/kellenblumberg/git/props/aws/props-aws.pem" /Users/kellenblumberg/git/props/discord-pbt-bot/ ubuntu@34.204.49.162:~/ -rsync -azvv --progress -e "ssh -i /Users/kellenblumberg/git/props/aws/props-aws.pem" ~/Downloads/node-v16.15.0-linux-arm64.tar.xz ubuntu@34.204.49.162:~/ -rsync -azvv --progress -e "ssh -i /Users/kellenblumberg/git/props/aws/props-aws.pem" /Users/kellenblumberg/git/props/discord-pbt-bot/pm2.json ubuntu@34.204.49.162:~/pm2.json +When will you be out for? +Should we help out that 1st rock at all? +Thought leadership for NFTs as a utility +Why do we not have partners on Ethereum outside of Sushi? +Whats our sales pipeline look like? +Buillding stuff instead of analysis + +VGX*dup.umt!rum4bqx +SELECT * FROM information_schema.tables + +ProAd DLU +Chat + data for Michele to highlight the powers of the Flipside program for chains + analysts +Version 1.0 of THORChain ecosystem map +Chatted with Katana and built SDK demo for them +THORChain bounty GP reviewing +Launched flash bounty for ATOM launch +FYI ATOM launch has been a dud - they had to pause swaps for the last few days due to bug +Working with analytics to fix some data issues + +Can we send ideas to you (Eric)? +Is the focus more on L2s or on Ethereum itself? + +WITH sc1 AS ( + SELECT * + , date_trunc('month', block_timestamp) AS month + , rune_price + , ROW_NUMBER() OVER (PARTITION BY month ORDER BY rune_liquidity DESC) AS rn + FROM thorchain.daily_pool_stats + WHERE pool_name IN ('BNB.BUSD-BD1','ETH.DAI-0X6B175474E89094C44DA98B954EEDEAC495271D0F','ETH.USDC-0XA0B86991C6218B36C1D19D4A2E9EB0CE3606EB48') +) +SELECT * +FROM sc1 +WHERE rn = 1 +Is solana too dependent on Serum +They use their transaction flows +Planning stage of expanding into other chains + +Transactions for each month - report on network performance +From the start of 2022 - touch on the outages +Improvements in the network when they implemented quick +How many failed transactions from the same address (from top 10 addresses) +Work on skeleton of the report - do they want to collaborate on it +What caused the network outage +TPS / number of transactions +How to get the explorer involved +Detail + labeling in transaction logs (e.g. Program IDs are labeled) +We provide some of the charts, they provide context +"State of Solana" every quarter + + +WITH base AS ( + SELECT DATE_TRUNC('week', block_timestamp) AS date + , signers[0]::string AS address + , COUNT(1) AS n + FROM solana.fact_transactions + WHERE block_timestamp::date >= '2022-01-01' + AND succeeded = FALSE + GROUP BY 1, 2 +), b2 AS ( + SELECT * + , ROW_NUMBER() OVER (PARTITION BY date ORDER BY n DESC) AS rn + FROM base +), b3 AS ( + SELECT date + , CASE + WHEN rn <= 10 THEN 'Top 10' + WHEN rn <= 100 THEN 'Top 100' + ELSE 'Other' END AS address_type + , SUM(n) AS n + FROM b2 + WHERE rn <= 10 + GROUP BY 1, 2 +) +SELECT * +, n / (SUM(n) OVER ()) AS pct +FROM b3 + +SELECT * +FROM thorchain.swaps +WHERE pool_name LIKE 'GAIA.ATOM%' +ORDER BY block_timestamp +LIMIT 100 + +WITH base AS ( + SELECT native_to_address + , COUNT(1) AS n_swaps + , SUM(to_amount_usd) AS volume + FROM thorchain.swaps + WHERE to_asset LIKE 'ETH%' + GROUP BY 1 + ORDER BY 3 DESC + LIMIT 25 +) + +WITH base AS ( + SELECT signers[0]::string AS address + , COUNT(1) AS n + FROM solana.fact_transactions + WHERE block_timestamp::date >= '2022-01-01' + AND succeeded = FALSE + GROUP BY 1 + ORDER BY 2 DESC + LIMIT 1000 +), b2 AS ( + SELECT * + , ROW_NUMBER() OVER (PARTITION BY date ORDER BY n DESC) AS rn + FROM base +), b3 AS ( + SELECT date + , CASE + WHEN rn <= 10 THEN 'Top 10' + WHEN rn <= 100 THEN 'Top 100' + ELSE 'Other' END AS address_type + , SUM(n) AS n + FROM b2 + WHERE rn <= 10 + GROUP BY 1, 2 +) +SELECT * +, n / (SUM(n) OVER ()) AS pct +FROM b3 + + + + +SELECT s.block_timestamp::date AS date +, SUM(s.sales_amount) AS volume +FROM solana.core.fact_nft_sales s +JOIN solana.core.dim_nft_metadata m ON m.mint = s.mint AND m.collection = '${collection}' +WHERE s.block_timestamp::date < CURRENT_DATE::date +GROUP BY 1 +ORDER BY 1 + + +WITH base AS ( + SELECT day, date_trunc('month', day) AS month, SUM(asset_liquidity * asset_price_usd) AS non_rune_asset_value + GROUP BY 1 +), b2 AS ( + SELECT * + , ROW_NUMBER() OVER (PARTITION BY month ORDER BY day DESC) AS rn +), b3 AS ( + SELECT month + , non_rune_asset_value + FROM b2 + WHERE rn = 1 +) +SELECT * +FROM b3 + +FROM FLIPSIDE_PROD_DB.THORCHAIN.DAILY_POOL_STATS +WHERE day = '2022-07-08' +ORDER BY non_rune_asset_value desc + + + +SELECT * +, ARRAY_SIZE(t.instructions) AS arr_sz +FROM solana.core.fact_transactions t +WHERE block_timestamp >= '2022-07-03' +AND block_timestamp <= '2022-07-07' +AND tx_id = '3Gb17jSPuEpXDZjNLQ2VfoBEv7NCYz8dU9fjprouGCBzgKtqB7J4CQ5sX6UDxEd4rEBrDbWxyzHBcVJNp3uzwG2p' + +SELECT * +FROM solana.core.dim_labels +WHERE label = '3d sniping demons' + +WITH base AS ( + SELECT date_trunc('hour', block_timestamp) AS sacrifice_time + , SUM(ARRAY_SIZE(t.instructions) / 2) AS n_burns + FROM solana.core.fact_transactions t + JOIN solana.core.dim_labels l ON l.address = t.instructions[0]:parsed:info:mint::string + WHERE block_timestamp >= '2022-07-01' + AND block_timestamp <= '2022-07-10' + AND l.label = 'degentown' + AND instructions[0]:parsed:type = 'burn' + GROUP BY 1 +) +SELECT * +, SUM(n_burns) OVER (ORDER BY sacrifice_time) AS cumu_burns +FROM base + + + +with base as (select date_trunc('day', block_timestamp) as day, + pool_name, + CASE +WHEN CHARINDEX('-', pool_name) > 0 THEN LEFT(pool_name, CHARINDEX('-', pool_name)-1) ELSE pool_name END AS pool_names, + case + when pool_name ilike 'BTC.%' then 'BTC' + when pool_name ilike 'ETH.%' then 'ETH' + when pool_name ilike 'BNB.%' then 'BNB' + when pool_name ilike 'LTC.%' then 'LTC' + when pool_name ilike 'DOGE.%' then 'DOGE' + when pool_name ilike 'TERRA.%' then 'TERRA' + when pool_name ilike 'BCH.%' then 'BCH' + when pool_name ilike 'BNB.%' then 'BNB' + else pool_name + end as chain, + avg(asset_amount_usd) as asset_liq, + asset_liq * 2 as TVL, + case + when chain = 'TERRA' and day >= '2022-05-09' then 0 + else TVL + end as real_TVL +from thorchain.pool_block_balances +where asset_amount > 0 + and date_trunc('day', block_timestamp) >= '2022-01-01' +group by 1,2 +order by real_TVL desc), + +base4 as (select day, +sum(real_TVL) as real_TVLz +from base +group by 1), + +base5 as (select date_trunc('day', day) as day, +sum(total_value_bonded_usd) as total_value_bonded_usdz +from thorchain.daily_tvl +group by 1), + +base2 as (select a.day, +((real_TVLz / 2) + total_value_bonded_usdz) / 301000000 as deterministic_price +from base4 a +join base5 b +on a.day = b.day), + +base3 as (select date_trunc('day', block_timestamp) as day, +avg(rune_usd) as daily_rune_price +from thorchain.prices +group by 1) + +select a.day, +deterministic_price, +daily_rune_price, +daily_rune_price / deterministic_price as speculative_multiplier +from base2 a +join base3 b +on a.day = b.day + + + +with degen as (SELECT address, label +FROM solana.core.dim_labels +WHERE label = 'degentown'), + +discord_mods as (select purchaser, count(distinct mint) as number +from solana.core.fact_nft_mints mints +inner join degen on mints.mint = degen.address +group by purchaser +order by number desc), + + +mint_count as (select mint.purchaser as person, count(distinct mint) as mints +from solana.core.fact_nft_mints mint +inner join discord_mods on mint.purchaser = discord_mods.purchaser + where mint.block_timestamp > CURRENT_DATE - 30 +group by person +order by mints desc), + + +lisht as (select distinct purchaser as name +from solana.core.fact_nft_mints mints +inner join degen on mints.mint = degen.address) +, + +rishi as ( + select * + from solana.core.fact_nft_mints a + inner join lisht on a.purchaser = lisht.name + ), + + +too as (Select distinct purchaser,block_timestamp + From (SELECT RANK() OVER (PARTITION BY purchaser order by block_timestamp asc) + as RN,purchaser,block_TIMESTAMP +from rishi) as ST + Where st.rn = 1 +order by block_timestamp asc) +, first_tx AS ( + SELECT tx_to AS address, MIN(block_timestamp) AS mn + FROM solana.core.fact_transfers + GROUP BY 1 + UNION ALL + SELECT tx_from AS address, MIN(block_timestamp) AS mn + FROM solana.core.fact_transfers + GROUP BY 1 +), f2 AS ( + SELECT address, MIN(mn) AS first_tx_time + FROM first_tx + GROUP BY 1 +) +, base AS ( +select DISTINCT t.purchaser, CASE +WHEN COALESCE(f2.first_tx_time, t.block_timestamp)::date >= '2022-06-30' AND t.block_timestamp::date = '2022-07-01' THEN 'Burner' + WHEN t.block_timestamp::date = '2022-07-01' THEN 'New Minter' + ELSE 'Experienced Minter' END AS wallet_type + from too t + LEFT JOIN f2 ON f2.address = t.purchaser +) +select +case when COALESCE(m.mints, 0) <= 1 then '1 - Normie' +when m.mints < 10 then '2 - NFT Enthusiast' + when m.mints < 50 then '3 - Degen' +when m.mints < 100 then '4 - High Power Degen' +else '5 - GOD MODE DEGEN' end as power_level, count(*) as numba +FROM base b +LEFT JOIN mint_count m ON b.purchaser = m.person + where wallet_type = 'Experienced Minter' +group by power_level +order by numba desc + + + + + + + +with degen as (SELECT address, label +FROM solana.core.dim_labels +WHERE label = 'degentown'), + +lisht as (select distinct purchaser as name +from solana.core.fact_nft_mints mints +inner join degen on mints.mint = degen.address) +, + +rishi as ( + select * + from solana.core.fact_nft_mints a + inner join lisht on a.purchaser = lisht.name + ), + + +too as (Select distinct purchaser,block_timestamp + From (SELECT RANK() OVER (PARTITION BY purchaser order by block_timestamp asc) + as RN,purchaser,block_TIMESTAMP +from rishi) as ST + Where st.rn = 1 +order by block_timestamp asc) +, first_tx AS ( + SELECT tx_to AS address, MIN(block_timestamp) AS mn + FROM solana.core.fact_transfers + GROUP BY 1 + UNION ALL + SELECT tx_from AS address, MIN(block_timestamp) AS mn + FROM solana.core.fact_transfers + GROUP BY 1 +), f2 AS ( + SELECT address, MIN(mn) AS first_tx_time + FROM first_tx + GROUP BY 1 +) +, base AS ( +select CASE +WHEN COALESCE(f2.first_tx_time, t.block_timestamp)::date >= '2022-06-30' AND t.block_timestamp::date = '2022-07-01' THEN 'Burner' + WHEN t.block_timestamp::date = '2022-07-01' THEN 'New Minter' + ELSE 'Non-Burner' END AS wallet_type + , block_timestamp::date AS date + , count(1) AS n_wallets + from too t + LEFT JOIN f2 ON f2.address = t.purchaser +group by 1, 2 +order by 2 desc +) +SELECT * +, SUM(n_wallets) OVER () AS total_wallets +FROM base + + + + + + + + + + +[ + { + "parsed": { + "info": { + "account": "3XRvFfcjoqXeTT4t9DLnqJBgok7tepMcvgG6sNPXLpW9", + "amount": "1", + "authority": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "mint": "CKz4MabakdGnJ5icKXv9VdBAVjCcodWVA6D2Zop1ztz8" + }, + "type": "burn" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "3XRvFfcjoqXeTT4t9DLnqJBgok7tepMcvgG6sNPXLpW9", + "destination": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "multisigOwner": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "signers": [ + "burn68h9dS2tvZwtCFMt79SyaEgvqtcZZWJphizQxgt" + ] + }, + "type": "closeAccount" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "8YDvXGdN7jxEh5A3TsS3KgHpaxEkjhJgXrzRpRKUwTMt", + "amount": "1", + "authority": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "mint": "3XLdZCEpkZnrNpw3qv2AcRdQGgbk85T1QN2HBiRVXqUB" + }, + "type": "burn" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "8YDvXGdN7jxEh5A3TsS3KgHpaxEkjhJgXrzRpRKUwTMt", + "destination": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "multisigOwner": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "signers": [ + "burn68h9dS2tvZwtCFMt79SyaEgvqtcZZWJphizQxgt" + ] + }, + "type": "closeAccount" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "36DZPspQzx61sti7porCqvHkiUf6zZPgKgoV1XdCevQj", + "amount": "1", + "authority": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "mint": "7wShpK34PBwR8ys4oC4mR2zMYFTnB6ZhVQWxP2auo1dh" + }, + "type": "burn" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "36DZPspQzx61sti7porCqvHkiUf6zZPgKgoV1XdCevQj", + "destination": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "multisigOwner": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "signers": [ + "burn68h9dS2tvZwtCFMt79SyaEgvqtcZZWJphizQxgt" + ] + }, + "type": "closeAccount" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "59JVeBTm4vtkWwjvmB7raB2nyW3wKj6NbV9ZgXzUutQv", + "amount": "1", + "authority": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "mint": "EcaTM9LqBRqce9dLNu2kdWHm6kbW3cA4EjcZrWMeN1hm" + }, + "type": "burn" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "59JVeBTm4vtkWwjvmB7raB2nyW3wKj6NbV9ZgXzUutQv", + "destination": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "multisigOwner": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "signers": [ + "burn68h9dS2tvZwtCFMt79SyaEgvqtcZZWJphizQxgt" + ] + }, + "type": "closeAccount" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "9Nfso2ao2E57KDjeFSsWGXQmEtDBT7WUnRGNAAof8vbH", + "amount": "1", + "authority": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "mint": "6YAbarAamgwVggsKVfcK8VxaKfd33JB9sbnu5xrvXGyX" + }, + "type": "burn" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "9Nfso2ao2E57KDjeFSsWGXQmEtDBT7WUnRGNAAof8vbH", + "destination": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "multisigOwner": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "signers": [ + "burn68h9dS2tvZwtCFMt79SyaEgvqtcZZWJphizQxgt" + ] + }, + "type": "closeAccount" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "Nbp9GwLWoCaVGJfqfiMpMXbTWaiAZt42kPtkPVrmxEQ", + "amount": "1", + "authority": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "mint": "GtzGYEERyzAkXweXdzoSmUStMqhwS4y27q9DbHxDjfhV" + }, + "type": "burn" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "Nbp9GwLWoCaVGJfqfiMpMXbTWaiAZt42kPtkPVrmxEQ", + "destination": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "multisigOwner": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "signers": [ + "burn68h9dS2tvZwtCFMt79SyaEgvqtcZZWJphizQxgt" + ] + }, + "type": "closeAccount" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "12xmesVmUBWeE4VTBJEQf8tJmxxGrdZzwBozMSptqKqF", + "amount": "1", + "authority": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "mint": "91SAd85gQeCYNKHun94RDhYG8hMJmJzF2jAY6WisFHri" + }, + "type": "burn" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "12xmesVmUBWeE4VTBJEQf8tJmxxGrdZzwBozMSptqKqF", + "destination": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "multisigOwner": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "signers": [ + "burn68h9dS2tvZwtCFMt79SyaEgvqtcZZWJphizQxgt" + ] + }, + "type": "closeAccount" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "HpNxFhnAfo4xCw1Rhf4wSf6A77VP3ySPXKHJSe9DavDA", + "amount": "1", + "authority": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "mint": "GuFyccsRnc8JPhab5hyr3vv8k7WvoVShdDxGW9RHhJsh" + }, + "type": "burn" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "HpNxFhnAfo4xCw1Rhf4wSf6A77VP3ySPXKHJSe9DavDA", + "destination": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "multisigOwner": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "signers": [ + "burn68h9dS2tvZwtCFMt79SyaEgvqtcZZWJphizQxgt" + ] + }, + "type": "closeAccount" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "Es36Bvuroth81DntriTDnNkkGdVAGMTrHf1DwYWame1M", + "amount": "1", + "authority": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "mint": "9UcrGSbihtpMZQC1beAjM2WY6H9GC7PkkmDAxEsqeWqK" + }, + "type": "burn" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "Es36Bvuroth81DntriTDnNkkGdVAGMTrHf1DwYWame1M", + "destination": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "multisigOwner": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "signers": [ + "burn68h9dS2tvZwtCFMt79SyaEgvqtcZZWJphizQxgt" + ] + }, + "type": "closeAccount" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "4Cjt4voswT1bJ3FWjUcx1bpUdyafciG9QRxe7UD7oHnG", + "amount": "1", + "authority": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "mint": "5a8sxfwUUWkjZ6BgHWT6NZjZAjYQkiywFu2wYsu6YzVJ" + }, + "type": "burn" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "4Cjt4voswT1bJ3FWjUcx1bpUdyafciG9QRxe7UD7oHnG", + "destination": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "multisigOwner": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "signers": [ + "burn68h9dS2tvZwtCFMt79SyaEgvqtcZZWJphizQxgt" + ] + }, + "type": "closeAccount" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "Etah1rY8SkS43V15kSi1YaxgZ63REiLuZwARrrHp4YRv", + "amount": "1", + "authority": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "mint": "H4W2WozmMcLgCfoe5EERgS57wcKupEwmhjr4CDr1viqF" + }, + "type": "burn" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + }, + { + "parsed": { + "info": { + "account": "Etah1rY8SkS43V15kSi1YaxgZ63REiLuZwARrrHp4YRv", + "destination": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "multisigOwner": "835dotUpwzNnvWuHQW5DKpFjrvZp4zWJN2D2VmbPpz79", + "signers": [ + "burn68h9dS2tvZwtCFMt79SyaEgvqtcZZWJphizQxgt" + ] + }, + "type": "closeAccount" + }, + "program": "spl-token", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + } +] + +Some idears... +"Flipside API in Python or R." -> Carlos or me? +"EVM conceptual deep dive" -> Austin or Jim? +"ShroomDK deep dive." -> Jim or Chris or Don? + + + +WITH base AS ( + SELECT INITCAP(l.label) AS collection + , COUNT(DISTINCT l.address) AS n_tokens + , COUNT(DISTINCT m.mint) AS n_mints + , AVG(m.mint_price) AS avg_mint_price + , MIN(m.block_timestamp::date) AS mint_date + , MIN(date_trunc('month', m.block_timestamp)) AS mint_month + FROM solana.core.dim_labels l + LEFT JOIN solana.core.fact_nft_mints m ON m.mint = l.address AND m.mint_price > 0 AND m.mint_price < 10 + WHERE label_subtype = 'nf_token_contract' + GROUP BY 1 +), volume AS ( + SELECT INITCAP(l.label) AS collection + , COUNT(1) AS n_sales + , COUNT(DISTINCT block_timestamp::date) AS n_days + , SUM(sales_amount) AS volume + , volume / n_days AS daily_volume + FROM solana.core.fact_nft_sales s + JOIN solana.core.dim_labels l ON l.address = s.mint + GROUP BY 1 +) +SELECT b.* +, COALESCE(b.avg_mint_price, 0) * b.n_tokens AS mint_volume +, v.n_sales +, v.n_days +, v.volume +, v.daily_volume +FROM base b +JOIN volume v ON v.collection = b.collection +ORDER BY mint_month DESC, mint_volume DESC + +SELECT * +FROM solana.core.dim_labels +WHERE label ilike 'DegenTown' + +SELECT * +FROM solana.core.fact_transactions t +JOIN solana.core.dim_labels l ON l.address = t.instructions[8]:parsed:info:mint::string +WHERE block_timestamp >= '2022-07-01' +AND instructions[0]:parsed:type = 'burn' +AND tx_id = '4RtsBQJ8EnDL2kh7cLQfuJrY4sMQVBUagRkHjvivzs1mcWGvdrgmqJ8yKUcF3r911XqeSGq5uswzD5ouWRBJCjGz' + + + + + + +with data_sol as ( + select + instruction:accounts[0]::string as address + , MIN(block_timestamp) AS wormhole_time + from + solana.core.fact_events + where + block_timestamp::date >= CURRENT_DATE - 30 + and + program_id = 'wormDTUJ6AWPNvk59vGQbDvGJmqbDTdgWgAqcLBCgUb' + and + instruction:data = '4' -- retrieve from other chains + and + ( + inner_instruction:instructions[1]:parsed:type = 'mintTo' -- have careated associated token account + or + inner_instruction:instructions[3]:parsed:type = 'mintTo' -- not yet created associated token account + ) + and + succeeded = true + group by 1 +) +, data_user_programs as ( +select + b.address, + case when label is null then COALESCE(program_id, 'None') else label end as labeling, + ROW_NUMBER() OVER (PARTITION BY b.address ORDER BY block_timestamp) AS rn +from + data_sol b + LEFT JOIN + solana.core.fact_events a on a.block_timestamp > b.wormhole_time AND a.block_timestamp <= DATEADD('days', 4, b.wormhole_time) AND + b.address IN ( + instruction:accounts[0] + , instruction:accounts[1] + , instruction:accounts[2] + , instruction:accounts[3] + , instruction:accounts[4] + , instruction:accounts[5] + , instruction:accounts[6] + , instruction:accounts[7] + , instruction:accounts[8] + , instruction:accounts[9] + ) +left join solana.core.dim_labels c on a.program_id = c.address +where + a.block_timestamp::date >= CURRENT_DATE - 30 + and + succeeded = true + and + program_id not in ( + 'wormDTUJ6AWPNvk59vGQbDvGJmqbDTdgWgAqcLBCgUb', -- exclude wormhole + 'worm2ZoG2kUd4vFXhvjh93UUH596ayRfgQ2MgjNMTth', -- exclude wormhole + 'DeJBGdMFa1uynnnKiwrVioatTuHmNLpyFKnmB5kaFdzQ', -- Phantom wallet program id for trasnfer https://docs.phantom.app/resources/faq + '4MNPdKu9wFMvEeZBMt3Eipfs5ovVWTJb31pEXDJAAxX5' -- transfer token program + ) -- exclude wormhole program id + -- and + -- block_timestamp::date >= CURRENT_DATE - 90 + and labeling != 'solana' +-- group by 1, 2 +and ROW_NUMBER() OVER (PARTITION BY b.address ORDER BY block_timestamp) <= 3 +) +select + CASE + when labeling = 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K' then 'Magic Eden V2' + when labeling = '3Katmm9dhvLQijAvomteYMo6rfVbY5NaCRNq9ZBqBgr6' then 'Francium Lend Reward Program' + when labeling = 'QMNeHCGYnLVDn1icRAfQZpjPLBNkfGbSKRB83G5d8KB' then 'Quarry Protocol' + when labeling = 'VoLT1mJz1sbnxwq5Fv2SXjdVDgPXrb9tJyC8WpMDkSp' then 'Friktion Protocol' + when labeling = 'hausS13jsjafwWwGqZTUQRmWyvyxn9EQpqMwV1PBBmk' then 'Opensea' + when labeling = '2nAAsYdXF3eTQzaeUQS3fr4o782dDg8L28mX39Wr5j8N' then 'lyfRaydiumProgramID' + else labeling end as label, + count(*) as total + from data_user_programs + where label != 'solana' -- remove program id that dedicated to solana + and rn <=3 + + group by 1 order by 2 + desc +// limit 10 + + +https://twitter.com/pinehearst_/status/1542532946632265728 + +Updates / Accomplishments +Ran our first THORChain flash bounty this week, done by Pinehearst +THORChain analysis on Terra LPs +Added Famous Foxes and Primates to NFT Deal Score +Helped Jack Forlines with some queries for wormhole + Michael with some stuff for Metaplex report +Finished the Solana NFT Metadata update script +Added THORNames + slash_points tables and updates to switch events tables +Solana data sleuthing - worked with Jessica + Desmond to fix nft_mints + nft_sales tables +Problems Encountered +Wormhole query took a REALLY long time to run +Pretty clean week +Priorities +Get Solana NFT Metadata update script running on server +Chat with Eric about doing that cool wallet unique identifier thingy +Concerns +Nothing at the moment + + + + SELECT block_timestamp, SPLIT(to_asset, '-')[0] AS asset, from_address, to_amount AS usd_amount + FROM flipside_prod_db.thorchain.swaps + WHERE (to_asset ILIKE '%/BUSD%' OR to_asset = 'ETH/USDC-0XA0B86991C6218B36C1D19D4A2E9EB0CE3606EB48') AND + block_timestamp >= TO_DATE('2022-01-01') + + SELECT block_timestamp, SPLIT(to_asset, '-')[0] AS asset, from_address, to_amount AS usd_amount + FROM flipside_prod_db.thorchain.swaps + WHERE (to_asset ILIKE '%/BUSD%' OR to_asset = 'ETH/USDC-0XA0B86991C6218B36C1D19D4A2E9EB0CE3606EB48') AND + block_timestamp >= TO_DATE('2022-01-01') + + +SELECT SUM(to_e8) * POWER(10, -8) AS amt +FROM midgard.swap_events +WHERE to_asset ILIKE '%/BUSD%' + +SELECT SUM(to_e8) * POWER(10, -8) AS amt +FROM thorchain.swap_events +WHERE to_asset ILIKE '%/BUSD%' + +WITH base AS ( +SELECT from_address +, SUM(from_amount_usd) AS volume_usd +FROM thorchain.swaps +WHERE pool_name LIKE '%BUSD%' +AND block_timestamp >= '2022-04-01' +GROUP BY 1 +ORDER BY 2 DESC +LIMIT 10 +) +SELECT * +FROM base +ORDER BY 2 + +WITH base AS ( + SELECT SPLIT(pool_name, '-')[0] AS pool_name + , COUNT(1) AS n_swaps + , ROUND(SUM(from_amount_usd)) AS usd_volume + FROM thorchain.swaps + WHERE from_address = 'thor1yd6m7pek6h6zh7p32d5xt20xcegz96pml0n208' + OR native_to_address = 'thor1yd6m7pek6h6zh7p32d5xt20xcegz96pml0n208' + GROUP BY 1 +), b2 AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY usd_volume DESC) AS rn + FROM base +) +SELECT * +, CONCAT(CASE WHEN rn < 10 THEN '0', ELSE '' END, rn::varchar, '. ', pool_name) AS label +FROM b2 + + +SELECT DISTINCT date_trunc('hour', block_timestamp) AS hour +FROM thorchain.UPDATE_NODE_ACCOUNT_STATUS_EVENTS +ORDER BY 1 DESC + +SELECT COUNT(1) AS n, MAX(block_timestamp) +FROM thorchain.swaps + +189779216.728215 - 188233930.596686 +1,545,286 + +SELECT CASE WHEN ( + from_address IN ('thor1dhlfhswuqs64s0askzmkq3nqutaajunjj5n6wn','thor170e2xv7atjepzr8cjsjlpyq4fah6ykdagg6c2u','bnb1z3s7lqa96pw0netgzgge476xy8wp5eavg5vxv5') + OR native_to_address IN ('thor1dhlfhswuqs64s0askzmkq3nqutaajunjj5n6wn','thor170e2xv7atjepzr8cjsjlpyq4fah6ykdagg6c2u','bnb1z3s7lqa96pw0netgzgge476xy8wp5eavg5vxv5') +) THEN 'Synth Hoarders' ELSE 'Others' END AS swapper +, COUNT(1) AS n_swaps +, ROUND(SUM(from_amount_usd)) AS usd_volume +FROM thorchain.swaps +WHERE pool_name LIKE '%.BUSD%' +AND block_timestamp::date >= '2022-04-25' +GROUP BY 1 +ORDER BY 3 DESC + +SELECT REPLACE(SPLIT(pool_name, '-')[0], '/', '.') AS poolname +, SUM(CASE WHEN from_asset LIKE '%/%' THEN from_amount_usd ELSE 0 END) AS synth_swap_volume +, SUM(CASE WHEN from_asset LIKE '%/%' THEN 0 ELSE from_amount_usd END) AS non_synth_swap_volume +, SUM(CASE WHEN from_asset LIKE '%/%' THEN liq_fee_rune_usd + liq_fee_asset_usd ELSE 0 END) AS synth_swap_fees +, SUM(CASE WHEN from_asset LIKE '%/%' THEN 0 ELSE liq_fee_rune_usd + liq_fee_asset_usd END) AS non_synth_swap_fees +FROM thorchain.swaps +WHERE block_timestamp::date >= '2022-04-25' +GROUP BY 1 + +WITH base AS ( + SELECT CASE WHEN pool_name LIKE '%BUSD%' THEN 'BUSD' ELSE 'Others' AS pool + , SUM(CASE WHEN from_asset LIKE '%/%' THEN from_amount_usd ELSE 0 END) AS synth_swap_volume + , SUM(CASE WHEN from_asset LIKE '%/%' THEN 0 ELSE from_amount_usd END) AS non_synth_swap_volume + , SUM(CASE WHEN from_asset LIKE '%/%' THEN liq_fee_rune_usd + liq_fee_asset_usd ELSE 0 END) AS synth_swap_fees + , SUM(CASE WHEN from_asset LIKE '%/%' THEN 0 ELSE liq_fee_rune_usd + liq_fee_asset_usd END) AS non_synth_swap_fees + FROM thorchain.swaps + WHERE block_timestamp::date >= '2022-04-25' + GROUP BY 1 +) +SELECT * +, synth_swap_fees / non_synth_swap_volume AS synth_swap_fees_vs_non_synth_swap_volume_ratio +FROM base + + + +WITH base AS ( + SELECT CASE WHEN pool_name LIKE '%BUSD%' THEN 'BUSD' ELSE 'Others' END AS pool + , SUM(CASE WHEN from_asset LIKE '%/%' THEN from_amount_usd ELSE 0 END) AS synth_swap_volume + , SUM(CASE WHEN from_asset LIKE '%/%' THEN 0 ELSE from_amount_usd END) AS non_synth_swap_volume + , SUM(CASE WHEN from_asset LIKE '%/%' THEN liq_fee_rune_usd + liq_fee_asset_usd ELSE 0 END) AS synth_swap_fees + , SUM(CASE WHEN from_asset LIKE '%/%' THEN 0 ELSE liq_fee_rune_usd + liq_fee_asset_usd END) AS non_synth_swap_fees + FROM thorchain.swaps + WHERE block_timestamp::date >= '2022-04-25' + GROUP BY 1 +), b2 AS ( + SELECT * + , synth_swap_fees / non_synth_swap_volume AS synth_swap_fees_vs_non_synth_swap_volume_ratio + FROM base b +) +SELECT b2.* +, b3.synth_swap_fees_vs_non_synth_swap_volume_ratio * b2.synth_swap_volume AS expected_synth_swap_fees +, b2.synth_swap_fees - expected_synth_swap_fees AS difference +FROM b2 +JOIN b2 b3 ON b3.pool = 'Others' + + + + +WITH synths AS ( + SELECT CASE WHEN native_to_address = '' THEN from_address ELSE native_to_address END AS address + , SUM(to_amount) AS to_amount + , COUNT(1) AS n_synths + FROM thorchain.swaps + WHERE to_asset LIKE '%/BUSD%' + GROUP BY 1 + ORDER BY 2 DESC +), burns AS ( + SELECT from_address AS address + , SUM(from_amount) AS from_amount + , COUNT(1) AS n_burns + FROM thorchain.swaps + WHERE from_asset LIKE '%/BUSD%' + GROUP BY 1 + ORDER BY 2 DESC +), base AS ( + SELECT COALESCE(s.address, b.address) AS address + , COALESCE(s.to_amount, 0) AS synth_amount + , COALESCE(b.from_amount, 0) AS burn_amount + , COALESCE(s.n_synths, 0) AS n_synths + , COALESCE(b.n_burns, 0) AS n_burns + , synth_amount - burn_amount AS net_synth_amount + , ROW_NUMBER() OVER (ORDER BY net_synth_amount DESC) AS rn + FROM synths s + FULL OUTER JOIN burns b on b.address = s.address +) +SELECT * FROM base + +SELECT * +FROM thorchain.swaps +WHERE ( + from_address = 'thor163jhxz3tyf4qpgmsqjnmmdv80z40yr3hn6e6f9' + OR to_pool_address = 'thor163jhxz3tyf4qpgmsqjnmmdv80z40yr3hn6e6f9' + OR native_to_address = 'thor163jhxz3tyf4qpgmsqjnmmdv80z40yr3hn6e6f9' +) AND ( + from_asset LIKE '%/BUSD%' + OR to_asset LIKE '%/BUSD%' +) + + +SELECT SUM(to_amount) AS to_amount +FROM thorchain.swaps +WHERE to_asset LIKE '%/BUSD%' +GROUP BY 1 +ORDER BY 2 DESC + +SELECT SUM(from_amount) AS from_amount +FROM thorchain.swaps +WHERE from_asset LIKE '%/BUSD%' +GROUP BY 1 +ORDER BY 2 DESC + +), burns AS ( +SELECT from_address +, SUM(from_amount) AS from_amount +FROM thorchain.swaps +WHERE from_asset LIKE '%/BUSD%' +GROUP BY 1 +ORDER BY 2 DESC + + +WITH synths AS ( + SELECT DATE_TRUNC('week', block_timestamp) AS week + , SUM(to_amount) AS to_amount + FROM thorchain.swaps + WHERE to_asset LIKE '%/BUSD%' + AND from_address = 'thor1yd6m7pek6h6zh7p32d5xt20xcegz96pml0n208' + GROUP BY 1 + ORDER BY 2 DESC +), burns AS ( + SELECT DATE_TRUNC('week', block_timestamp) AS week + , SUM(from_amount) AS from_amount + FROM thorchain.swaps + WHERE from_asset LIKE '%/BUSD%' + AND from_address = 'thor1yd6m7pek6h6zh7p32d5xt20xcegz96pml0n208' + GROUP BY 1 + ORDER BY 2 DESC +) +SELECT COALESCE(s.week, b.week) AS week +, COALESCE(s.to_amount, 0) AS synth_amount +, COALESCE(b.from_amount, 0) AS burn_amount +, synth_amount - burn_amount AS net_synth_amount +, SUM(net_synth_amount) OVER (ORDER BY COALESCE(s.week, b.week)) AS cumu_net_synth_amount +FROM synths s +FULL OUTER JOIN burns b on b.week = s.week + + +SELECT from_address +, to_address +, from_asset +, to_asset +, COUNT(1) AS n_swaps +, SUM(from_amount_usd) AS swaps_volume +FROM thorchain.swaps +WHERE from_address = 'thor1yd6m7pek6h6zh7p32d5xt20xcegz96pml0n208' +OR to_address = 'thor1yd6m7pek6h6zh7p32d5xt20xcegz96pml0n208' +GROUP BY 1, 2, 3, 4 +ORDER BY 6 DESC + + + +WITH synths AS ( + SELECT from_address + , SUM(to_amount) AS to_amount + FROM thorchain.swaps + WHERE to_asset LIKE '%/BUSD%' + GROUP BY 1 + ORDER BY 2 DESC +), burns AS ( + SELECT from_address + , SUM(from_amount) AS from_amount + FROM thorchain.swaps + WHERE from_asset LIKE '%/BUSD%' + GROUP BY 1 + ORDER BY 2 DESC +), base AS ( +SELECT COALESCE(s.from_address, b.from_address) AS from_address +, COALESCE(s.to_amount, 0) AS synth_amount +, COALESCE(b.from_amount, 0) AS burn_amount +, synth_amount - burn_amount AS net_synth_amount +, ROW_NUMBER() OVER (ORDER BY net_synth_amount DESC) AS rn +FROM synths s +FULL OUTER JOIN burns b on b.from_address = s.from_address +WHERE net_synth_amount > 0 +ORDER BY net_synth_amount DESC +) +SELECT * +, CASE WHEN rn < 10 THEN CONCAT('0', rn::varchar, '. ', from_address) +ELSE '10. Others' END AS label +FROM base + + + + + + + +WITH to_synths AS ( + SELECT DATE_TRUNC({{timeframe}}, block_timestamp) AS time, SPLIT(to_asset, '-')[0] AS asset, SUM(to_amount) AS to_amount, SUM(to_amount_usd) AS to_amount_usd + FROM flipside_prod_db.thorchain.swaps + WHERE block_timestamp >= TO_DATE('2022-03-11') + AND (to_asset ILIKE '%/%') + GROUP BY 1, 2 +), +from_synths AS ( + SELECT DATE_TRUNC({{timeframe}}, block_timestamp) AS time, SPLIT(from_asset, '-')[0] AS asset, SUM(from_amount) AS from_amount, SUM(from_amount_usd) AS from_amount_usd + FROM flipside_prod_db.thorchain.swaps + WHERE block_timestamp >= TO_DATE('2022-03-11') + AND (from_asset ILIKE '%/%') + GROUP BY 1, 2 +), +base AS ( + SELECT COALESCE(t.asset, f.asset) AS asset + , COALESCE(t.time, f.time) AS time + , COALESCE(to_amount, 0) AS to_amount + , COALESCE(to_amount_usd, 0) AS to_amount_usd + , COALESCE(from_amount, 0) AS from_amount + , COALESCE(from_amount_usd, 0) AS from_amount_usd + FROM to_synths t + FULL OUTER JOIN from_synths f ON f.time = t.time AND f.asset = t.asset +) +SELECT * +, CASE WHEN from_amount = 0 THEN NULL ELSE to_amount * 1.0 / from_amount END AS amount_ratio +, to_amount_usd - from_amount_usd AS net_mint_amount_usd +, CASE WHEN from_amount_usd = 0 THEN NULL ELSE to_amount_usd * 1.0 / from_amount_usd END AS usd_ratio +, CASE WHEN to_amount = 0 THEN NULL ELSE from_amount * 1.0 / to_amount END AS amount_ratio_2 +, CASE WHEN to_amount_usd = 0 THEN NULL ELSE from_amount_usd * 1.0 / to_amount_usd END AS usd_ratio_2 +FROM base +WHERE (from_amount_usd + to_amount_usd) > 10000 + + + + +Lean into IBC +What is topical +Reach out directly on Twitter or on Discord +Getting retweets from big accounts +Use Reddit + Discord +Knowing what does well - adding NFT communities +Scanning mentions + +Do we skip over regulatory? +Do we use money? + +DraftKings + FanDuel were saying it's a skill game +Spin it up as a skill game? +VC funding comes first +From a legal standpoint that's huge +Shifting to real money is a high priority +Invite Manny to git + +https://linktr.ee/multichain + + + + +Baller gonna ball in any market. + +Holla at a Balla. + + +SELECT * +FROM information_schema.tables + +Got the most airdrops + +Have we run any bounties on the number of the number of unique program ids? +Interactions bby programId +Solana dApp Store +FTX is positioning itself as a way to pay in crypto by converting on their platform +The benefit of having a bunch of Qualcomm guys running the show +Stocks are coming to FTX +I think FTX might replace Robinhood for me +I think SBF might be one of my favorite people - had to slow down video +The Orca girl is incredible +Mobile focus is bullish for props + + +with +-- We are interested in knowing how many users are currently in the ecosystem. +-- In this case, how many are holding $RUNE in their wallet or have an open LP position. +-- What is this number currently, and how has it trended over time? + t1 as ( + SELECT +distinct from_address as user, + min(block_timestamp) as debut +from flipside_prod_db.thorchain.bond_events + group by 1 + ), +t2 as ( + SELECT + distinct from_address as user , + min(block_timestamp) as debut + from flipside_prod_db.thorchain.message_events + group by 1 + ), + t3 as ( + SELECT + distinct rune_address as user , + min(block_timestamp) as debut + from flipside_prod_db.thorchain.stake_events + group by 1 + ), + t4 as ( + SELECT + distinct from_address as user, + min(block_timestamp) as debut + from flipside_prod_db.thorchain.swap_events + group by 1 + ), +t5 as ( + SELECT + distinct from_address as user, + min(block_timestamp) as debut + from flipside_prod_db.thorchain.transfer_events + group by 1 + ), + final as ( + select * from t1 + UNION + select * from t2 + UNION + select * from t3 + UNION + select * from t4 + UNION + select * from t5 + ), + final2 as ( + SELECT + distinct user, + min(debut) as final_debut + from final + group by 1 + ), + total_users as ( +SELECT +trunc(final_debut,'day') as date, +count(distinct user) as new_users,--, +sum(new_users) over (order by date) as total_users +from final2 +group by 1 +order by 1 asc + ), + t6 as ( + SELECT + trunc(block_timestamp,'day') as date, + from_address as user, + sum(rune_amount) as rune_out, + sum(rune_out) over (order by date) as cum_rune_out +FROM flipside_prod_db.thorchain.transfers + group by 1,2 +), + t7 as ( +SELECT + trunc(block_timestamp,'day') as date, + to_address as user, + sum(rune_amount) as rune_in, + sum(rune_in) over (order by date) as cum_rune_in +FROM flipside_prod_db.thorchain.transfers + group by 1,2 + ), + final_rune_holders as ( + SELECT + t7.date, + t7.user, + cum_rune_in-cum_rune_out as net_rune + from t7 + left join t6 on t7.user=t6.user and t7.date=t6.date + ), + rune_holders as ( + select date, count(distinct user) as new_users, sum(new_users) over (order by date) as total_users from final_rune_holders where net_rune>0 group by 1 + ), +t8 as ( + SELECT + trunc(block_timestamp,'day') as date, + from_address as user, + pool_name, + sum(asset_amount) as staked, + sum(staked) over (partition by user,pool_name order by date) as cum_staked +FROM flipside_prod_db.thorchain.liquidity_actions + group by 1,2,3 +), +t9 as ( + SELECT + trunc(block_timestamp,'day') as date, + to_address as user, + pool_name, + sum(asset_amount) as unstaked, + sum(unstaked) over (partition by user,pool_name order by date) as cum_unstaked +FROM flipside_prod_db.thorchain.liquidity_actions + group by 1,2,3 +), + final_loopers as ( + SELECT + t8.date, + t8.user, + t8.pool_name, + cum_staked-cum_unstaked as net_staked + from t8 + left join t9 on t8.user=t9.user and t8.pool_name=t9.pool_name + ), + final_loopers2 as ( + SELECT + date,user,sum(net_staked) as total_staked + from final_loopers + group by 1,2 + ), + loopers as ( + select date, count(distinct user) as new_users, sum(new_users) over (order by date) as total_users from final_loopers2 where total_staked>0 group by 1 + ) +select 'RUNE holders' as type,* from rune_holders +UNION +select 'Loopers' as type,* from loopers + UNION + select 'All users' as type,* from total_users + + + + +SELECT project_name, COUNT(DISTINCT mint) AS n_mint +FROM solana.dim_nft_metadata +GROUP BY 1 +ORDER BY 1 + +WITH base AS ( + SELECT project_name AS collection + , COUNT(DISTINCT mint) AS n_mint + FROM solana.dim_nft_metadata + GROUP BY 1 +), b2 AS ( + SELECT CAST(((n_mint - 1) / 1000.0) AS INT) AS n_mint_grp + , COUNT(1) AS n + FROM base + GROUP BY 1 +) +SELECT CASE WHEN n_mint_grp >= 10 THEN '[10k, +)' +ELSE CONCAT('[',n_mint_grp,'k, ',n_mint_grp + 1,'k)') END AS n_mint_grp_2 +, SUM(n) AS n +FROM b2 +GROUP BY 1 +ORDER BY 1 + + + +WITH base AS ( + SELECT m.project_name AS collection + , SUM(sales_amount) AS sales_amount + FROM solana.fact_nft_sales s + JOIN solana.dim_nft_metadata m ON m.mint = s.mint + WHERE s.block_timestamp >= '2022-01-01' + GROUP BY 1 +), b2 AS ( + SELECT collection + , ROW_NUMBER() OVER (ORDER BY sales_amount DESC) AS rn + FROM base +), b3 AS ( + SELECT m.project_name AS collection + , date_trunc('month', s.block_timestamp) AS month + , SUM(sales_amount) AS sales_amount + FROM solana.fact_nft_sales s + JOIN solana.dim_nft_metadata m ON m.mint = s.mint + JOIN b2 ON b2.collection = m.project_name AND b2.rn <= 10 + WHERE s.block_timestamp >= '2022-01-01' + GROUP BY 1, 2 +) +SELECT * +, sales_amount / SUM(sales_amount) OVER (PARTITION BY month) AS pct +FROM b3 + + +SELECT * +FROM solana.dim_nft_metadata +WHERE LOWER(project_name) IN ( + 'cets on creck' + , 'astrals' + , 'solstein' + , 'solgods' + , 'okay bears' + , 'meerkat millionaires' + , 'catalina whale mixer' + , 'citizens by solsteads' + , 'defi pirates' +) + + +SELECT * +FROM solana.dim_nft_metadata +WHERE LOWER(project_name) IN ( + 'astrals' +) + + + +WITH mult_currency AS( + SELECT DISTINCT l.label + FROM solana_dev.silver.nft_mints m + JOIN solana.dim_labels l ON m.mint = l.address + WHERE m.mint_currency <> 'So11111111111111111111111111111111111111111' +), sales AS ( + SELECT l.label + , COUNT(1) AS n_sales + , SUM(s.sales_amount) AS volume + , COUNT(DISTINCT s.mint) AS n_mints + , MIN(block_timestamp::date) AS start_date + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l on s.mint = l.address + GROUP BY 1 +), mints AS ( + SELECT l.label + , CASE WHEN e.label IS NULL THEN 0 ELSE 1 END AS is_mult_currency + , COUNT(1) AS n_mints + , COUNT(DISTINCT m.mint) AS n_unique_mints + , SUM(m.mint_price) AS sol_mint_volume + FROM solana_dev.silver.nft_mints + JOIN solana.dim_labels l ON m.mint = l.address + LEFT JOIN mult_currency e ON e.label = l.label + GROUP BY 1, 2 +) +SELECT s.* +, m.is_mult_currency +, m.n_mints +, m.n_unique_mints +, m.sol_mint_volume +, CASE WHEN s.n_mints > COALESCE(m.n_unique_mints, 0) THEN 1 ELSE 0 END AS is_missing_mint_events +FROM sales s +LEFT JOIN mints m ON m.label = s.label +WHERE s.start_date >= '2022-01-01' +ORDER BY s.volume DESC + + + + +WITH mult_currency AS( + SELECT DISTINCT l.label + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON m.mint = l.address + WHERE m.mint_currency <> 'So11111111111111111111111111111111111111111' +), sales AS ( + SELECT l.label + , COUNT(1) AS n_sales + , SUM(s.sales_amount) AS volume + , COUNT(DISTINCT s.mint) AS n_mints + , MIN(block_timestamp::date) AS start_date + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l on s.mint = l.address + GROUP BY 1 +), mints AS ( + SELECT l.label + , CASE WHEN e.label IS NULL THEN 0 ELSE 1 END AS is_mult_currency + , COUNT(1) AS n_mints + , COUNT(DISTINCT m.mint) AS n_unique_mints + , SUM(m.mint_price) AS sol_mint_volume + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON m.mint = l.address + LEFT JOIN mult_currency e ON e.label = l.label + GROUP BY 1, 2 +) +SELECT s.* +, m.is_mult_currency +, m.n_mints +, m.n_unique_mints +, m.sol_mint_volume +, CASE WHEN s.n_mints > COALESCE(m.n_unique_mints, 0) THEN 1 ELSE 0 END AS is_missing_mint_events +FROM sales s +LEFT JOIN mints m ON m.label = s.label +WHERE s.start_date >= '2022-01-01' +ORDER BY s.volume DESC + +SELECT * +FROM solana.fact_nft_mints +WHERE block_timestamp >= '2022-04-24' +AND block_timestamp <= '2022-04-28' +AND mint = 'EJDtiPkuoRt9nNrQXhQDhtYuAGdV5JzQL9ZAVsRkEdJe' + +SELECT + +WITH sales AS ( + -- get all the sales + SELECT DISTINCT l.label + , s.mint + , s.tx_id + , s.block_timestamp::date AS date + , s.sales_amount + , MIN(date) OVER (PARTITION BY l.label) AS mn_date + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l on s.mint = l.address +), mints AS ( + -- get all the mints + SELECT DISTINCT l.label + , m.mint + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON m.mint = l.address +), base AS ( + -- check which sales do not have a corresponding mint + SELECT s.* + , ROW_NUMBER() OVER (PARTITION BY s.label ORDER BY s.sales_amount DESC) AS rn + FROM sales s + LEFT JOIN mints m ON m.mint = s.mint + WHERE m.mint IS NULL AND mn_date >= '2022-02-01' +), b2 AS ( + -- take the top 3 from each collection + SELECT * + FROM base + WHERE rn <= 3 +) +-- select only the top 500 sales missing a mint +SELECT * +FROM b2 +ORDER BY sales_amount DESC +LIMIT 500 + +@poolprops I think were close + +#1: $PROPS used to place bets +#2: Liquidity pools ensure ease of enter and exit +#3: 🤔 +#4: $PROPS stakers earn % of transaction fees +#5: Contributors rewarded with $PROPS based on DAO vote +#6: Protocol sustained by small tx fees +#7: https://kellen-blumberg.gitbook.io/props/ + +SELECT l.label +, COUNT(1) AS n +, COUNT(DISTINCT m.mint) AS n_mints +FROM solana.fact_nft_mints m +JOIN solana.dim_labels l ON l.address = m.mint +GROUP BY 1 + + +SELECT l.label +, m.mint +, COUNT(1) AS n_mints +, MAX(tx_id) AS mx_tx +, MIN(tx_id) AS mn_tx +FROM solana.fact_nft_mints m +JOIN solana.dim_labels l ON l.address = m.mint +GROUP BY 1, 2 +HAVING COUNT(1) > 1 +ORDER BY 3 DESC + + +WITH base AS ( + SELECT l.label + , m.mint + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON l.address = m.mint + GROUP BY 1, 2 + HAVING COUNT(1) > 1 +) +SELECT * +FROM solana.fact_nft_mints m +JOIN base b ON b.mint = m.mint +ORDER BY block_timestamp DESC, m.mint, tx_id + +WITH base AS ( + SELECT COALESCE(l.label, 'Unknownn') + , date_trunc('month') + , SUM(sales_amount) AS volume + +) + + +WITH non_sol_mint AS( + SELECT DISTINCT l.label + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON m.mint = l.address + WHERE m.mint_currency <> 'So11111111111111111111111111111111111111111' +), mult_mints AS ( + SELECT l.label, m.mint + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON m.mint = l.address + WHERE mint_price > 0 + GROUP BY 1, 2 + HAVING COUNT(1) > 1 +), times AS( + SELECT l.label + , m.mint + , m.mint_price + , m.block_timestamp + , m.mint_currency + , m.tx_id + , ROW_NUMBER() OVER (PARTITION BY l.label ORDER BY m.block_timestamp DESC) AS rn + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON m.mint = l.address + LEFT JOIN non_sol_mint n ON n.label = l.label + LEFT JOIN mult_mints mm ON mm.label = l.label + WHERE n.label IS NULL + AND mm.label IS NULL + AND mint_price > 0 +), rishi AS ( + SELECT label + , MIN(block_timestamp) as mn_date + , MAX(block_timestamp) as mx_date + , AVG(mint_price) as avg_price + , MIN(mint_price) as mn_price + , MAX(mint_price) as mx_price + , COUNT(DISTINCT ROUND(mint_price, 3)) as n_mint_prices + , COUNT(DISTINCT mint) as n_mints + FROM times + GROUP BY 1 +), sales AS ( + SELECT RANK() OVER (PARTITION BY l.label ORDER BY s.block_timestamp desc) AS rn + , l.label + , s.block_TIMESTAMP + , s.marketplace + , s.sales_amount + , s.mint + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l on s.mint = l.address +), s2 AS ( + SELECT * + , ROW_NUMBER() OVER (PARTITION BY label ORDER BY sales_amount ASC) AS rn2 + FROM sales + WHERE rn <= 20 +), cur_floor AS ( + SELECT label + , MIN(CASE WHEN rn2 = 4 THEN sales_amount ELSE NULL END) AS cur_floor + , MEDIAN(sales_amount) AS cur_floor_md + FROM s2 + GROUP BY 1 +), total_sales AS ( + SELECT label + , COUNT(1) AS n_sales + , SUM(sales_amount) AS volume + , COUNT(DISTINCT mint) as n_unique_sales + FROM sales + GROUP BY 1 +), base AS ( + SELECT t.label + , t.n_sales + , n_mints + , n_unique_sales + , t.volume + , f.cur_floor + , f.cur_floor_md + , r.mn_date + , r.mx_date + , DATEDIFF('days', mn_date, mx_date) AS dff + , avg_price + , mn_price + , mx_price + , mx_price - mn_price AS range + , n_mint_prices + , cur_floor - avg_price AS profit + , cur_floor / avg_price AS return + FROM total_sales t + JOIN cur_floor f ON f.label = t.label + JOIN rishi r ON r.label = t.label + WHERE dff <= 10 + AND n_sales >= 30 + AND n_mints > 50 + AND NOT t.label IN ('rithfu','degen games: dice game','solchicks','vs-y1-22') + AND mn_date >= '2022-01-01' + AND mn_date < '2022-06-01' + ORDER BY profit DESC +), b2 AS ( + SELECT t.label + , t.mint + , COUNT(1) AS n + , COUNT(DISTINCT t.tx_id) AS n_tx + FROM times t + JOIN base b ON b.label = t.label + GROUP BY 1, 2 + HAVING COUNT(1) > 1 + ORDER BY 3 DESC +) +SELECT t.label +, t.mint +, t.mint_price +, t.block_timestamp +, t.mint_currency +, t.tx_id +FROM times t +JOIN b2 ON b2.label = t.label AND b2.mint = t.mint +ORDER BY t.block_timestamp DESC, t.label, t.mint, t.mint_price, t.tx_id + + +WITH non_sol_mint AS( + SELECT DISTINCT l.label + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON m.mint = l.address + WHERE m.mint_currency <> 'So11111111111111111111111111111111111111111' +), times AS( + SELECT l.label + , m.mint + , m.mint_price + , m.block_timestamp + , m.mint_currency + , ROW_NUMBER() OVER (PARTITION BY l.label ORDER BY m.block_timestamp DESC) AS rn + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON m.mint = l.address + LEFT JOIN non_sol_mint e ON e.label = l.label + WHERE e.label IS NULL + AND mint_price > 0 +), rishi AS ( + SELECT label + , MIN(block_timestamp) as mn_date + , MAX(block_timestamp) as mx_date + , AVG(mint_price) as avg_price + , MIN(mint_price) as mn_price + , MAX(mint_price) as mx_price + , COUNT(DISTINCT ROUND(mint_price, 3)) as n_mint_prices + , COUNT(DISTINCT mint) as n_mints + FROM times + GROUP BY 1 +), sales AS ( + SELECT RANK() OVER (PARTITION BY l.label ORDER BY s.block_timestamp desc) AS rn + , l.label + , s.block_TIMESTAMP + , s.marketplace + , s.sales_amount + , s.mint + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l on s.mint = l.address +), s2 AS ( + SELECT * + , ROW_NUMBER() OVER (PARTITION BY label ORDER BY sales_amount ASC) AS rn2 + FROM sales + WHERE rn <= 20 +), cur_floor AS ( + SELECT label + , MIN(CASE WHEN rn2 = 4 THEN sales_amount ELSE NULL END) AS cur_floor + , MEDIAN(sales_amount) AS cur_floor_md + FROM s2 + GROUP BY 1 +), total_sales AS ( + SELECT label + , COUNT(1) AS n_sales + , SUM(sales_amount) AS volume + , COUNT(DISTINCT mint) as n_unique_sales + FROM sales + GROUP BY 1 +), base AS ( + SELECT t.label + , t.n_sales + , n_mints + , n_unique_sales + , t.volume + , f.cur_floor + , f.cur_floor_md + , r.mn_date + , r.mx_date + , DATEDIFF('days', mn_date, mx_date) AS dff + , avg_price + , mn_price + , mx_price + , mx_price - mn_price AS range + , n_mint_prices + , cur_floor - avg_price AS profit + , cur_floor / avg_price AS return + FROM total_sales t + JOIN cur_floor f ON f.label = t.label + JOIN rishi r ON r.label = t.label + WHERE dff <= 10 + AND n_sales >= 30 + AND n_mints > 50 + AND NOT t.label IN ('rithfu','degen games: dice game','solchicks') + AND mn_date >= '2022-01-01' + AND mn_date < '2022-06-01' + ORDER BY profit DESC +), b2 AS ( + SELECT CASE WHEN avg_price < 0.25 THEN '1: (0, 0.5) $SOL' + WHEN avg_price < 0.5 THEN '2: [0.5, 1.0) $SOL' + WHEN avg_price < 2 THEN '3: [1.0, 2.0) $SOL' + WHEN avg_price < 10 THEN '4: [2.0, 5.0) $SOL' + ELSE '5: 5+ $SOL' END AS mint_price + , CASE WHEN cur_floor < avg_price THEN '1: Unprofitable' + WHEN cur_floor - avg_price < 0.5 THEN '2: [0, 0.5) $SOL Profit' + WHEN cur_floor - avg_price < 2 THEN '3: [0.5, 2.0) $SOL Profit' + WHEN cur_floor - avg_price < 10 THEN '4: [2.0, 10.0) $SOL Profit' + ELSE '5: 10+ $SOL Profit' END AS profit_group + , COUNT(1) AS n_collections + FROM base + GROUP BY 1, 2 +) +SELECT * +, n_collections * 100 / SUM(n_collections) OVER (PARTITION BY mint_price) AS pct +FROM b2 +ORDER BY mint_price, profit_group + + + +SELECT m.* +FROM solana.fact_nft_mints m +JOIN solana.dim_labels l ON l.address = m.mint +WHERE l.label = 'jelly rascals' + + +secondtable AS ( + SELECT DISTINCT label + ,sales_amount + ,marketplace + ,block_timestamp + FROM (SELECT RANK() OVER (PARTITION BY label order by block_timestamp desc) + as RN,label,block_TIMESTAMP,marketplace,sales_amount //as date + from solana.core.fact_nft_sales join solana.core.dim_labels on solana.core.fact_nft_sales.mint = solana.core.dim_labels.address) as ST +Where ST.RN = 1), + +profit_table as(Select rishi.label,rishi.price_of_mint,secondtable.sales_amount, Case when rishi.price_of_mint = 0 then 99999 else secondtable.sales_amount/rishi.price_of_mint end as profit +from rishi +Inner join secondtable on rishi.label = secondtable.label +order by profit desc) + + + +SELECT CASE WHEN profit < 1 THEN '1: Less than 1x Returns' + WHEN profit >= 1 and profit < 2 THEN '2: 1-2x Returns' + WHEN profit >=2 and profit < 5 THEN '3: 2-5x Returns' + WHEN profit >= 5 and profit < 10 THEN '4: 5-10x Returns' + WHEN profit >= 10 and profit < 50 THEN '5: 10-50x Returns' + WHEN profit > 50 THEN '6: More than 50x Returns' + END AS profitability, count(label) as collections + From profit_table +GROUP by profitability +Order by profitability + + + + + + + +SELECT CASE WHEN profit < 1 THEN '1: Less than 1x Returns' + WHEN profit >= 1 and profit < 2 THEN '2: 1-2x Returns' + WHEN profit >=2 and profit < 5 THEN '3: 2-5x Returns' + WHEN profit >= 5 and profit < 10 THEN '4: 5-10x Returns' + WHEN profit >= 10 and profit < 50 THEN '5: 10-50x Returns' + WHEN profit > 50 THEN '6: More than 50x Returns' + END AS profitability, count(label) as collections + From profit_table +GROUP by profitability +Order by profitability + + + + + +https://howrare.is/vbagame +https://howrare.is/junglecats +https://howrare.is/meerkatmillionaires +https://howrare.is/nakedmeerkats +https://howrare.is/chroniclesnft +https://howrare.is/hellopantha +https://howrare.is/soldierxsolvivorsolvivor +https://howrare.is/gangstagators +https://howrare.is/foxyfennecsgang +https://howrare.is/dangervalleyducks + +https://twitter.com/cheesybobas +https://www.pixiv.net/en/users/36919471 + + +WITH base AS ( + SELECT f.asset + , f.asset_e8 + , f.pool_deduct + , f.block_timestamp + , f.tx + , p.cnt + , ROW_NUMBER() OVER (PARTITION BY f.asset, f.asset_e8, f.pool_deduct, f.block_timestamp, f.tx ORDER BY p.cnt) AS rn + FROM bronze_midgard_2_6_9_20220405.midgard_fee_events f + JOIN bronze_midgard_2_6_9_20220405.fee_events_pk_count p + ON p.asset = f.asset + AND p.asset_e8 = f.asset_e8 + AND p.pool_deduct = f.pool_deduct + AND p.block_timestamp = f.block_timestamp + AND p.tx = f.tx +) +SELECT asset +, asset_e8 +, pool_deduct +, block_timestamp +, tx +FROM base +WHERE rn <= cnt + +SELECT tx +, asset +, asset_e8 +, pool_deduct +, block_timestamp +, ROW_NUMBER() OVER (ORDER BY block_timestamp, tx) AS event_id +FROM midgard.fee_events curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash - - ssh -i "Users/kellenblumberg/git/props/aws/props-aws.pem" ubuntu@ec2-34-204-49-162.compute-1.amazonaws.com ssh -i "~/git/props/aws/props-aws.pem" ubuntu@34.204.49.162 +pm2 start npm -- start +sudo reboot + API Key: IioFYwv2t7KEsWjbh8xsEKd8d API Secret Key: 0650aF7Vo3gz2H7z1o10ws4ojXgjbl8UWk3N8kqVUtqvgIBv26 Bearer Token: AAAAAAAAAAAAAAAAAAAAAF%2BRdAEAAAAAQ2emwRpqU1B7Cj22LUxfJ%2Btrhck%3DpFpWm4G4Wd5dNLJjP2Ru5EuVVHEx6lroYfkiEAlPQZ0HQvqwsn -Hey just wanted introduce myself and let you know we really like your project. We're also a sports-related crypto project (actually giving away a Suite for one of our contests rn). +WITH base AS ( + SELECT * + , ROW_NUMBER() OVER (PARTITION BY project_name ORDER BY insert_date DESC) AS rn + FROM crosschain.address_labels + WHERE blockchain = 'solana' + AND label_subtype = 'nf_token_contract' +) +SELECT * +FROM base WHERE rn = 1 -I used to work in the analytics department for the Jacksonville Jaguars for 5 years and am now a Rust dev. Anyways just wanted to say hello and happy to keep the lines open if you guys ever wanted to talk about ideas or synergies between our projects. -The things that seemed to resonate most with Gaius were: -1. Team experience -2. We already had a live product -3. Our tokenomics +With secondtable as (Select DISTINCT label,marketplace,block_timestamp +From (SELECT RANK() OVER (PARTITION BY label order by block_timestamp) + as RN,label,block_TIMESTAMP,marketplace //as date + from solana.core.fact_nft_sales join solana.core.dim_labels on solana.core.fact_nft_sales.mint = solana.core.dim_labels.address) as ST +Where ST.RN = 1) -So maybe a few things to highlight would be +SELECT block_timestamp::date AS date +, COUNT(1) AS n_sales +, AVG( CASE WHEN l.label IS NULL THEN 0 ELSE 1 END ) AS pct_labeled +, SUM( CASE WHEN l.label IS NULL THEN 0 ELSE sales_amount END ) / SUM(sales_amount) AS pct_labeled_volume +FROM solana.core.fact_nft_sales s +LEFT JOIN solana.dim_labels l on s.mint = l.address +WHERE marketplace in ('magic eden v1', 'magic eden v2') +AND block_timestamp >= '2022-01-01' +GROUP BY 1 +ORDER BY 1 -The things that seemed to resonate most with Gaius were: -1. We have background working in sports analytics for an NFL team -2. We already have a live product live on mainnet -3. Our tokenomics -Updates -Made a cool thing to show off our THORChain awesomeness -Getting Rishi up to speed on stuff and saw him create some amazing dashboards -Working on Airdrop NTR for Optimism -Chatting with Yawww people to get Deal Score into their site -Problems Encountered -Errors on NFT Deal Score -Priorities -Fix errors on NFT Deal Score -NFT Loan Score -Concerns -Hopefully THORChain pays us more; it's been an up-and-down relationship +SELECT * +FROM bi_analytics.bronze.hevo_grades g +JOIN bi_analytics.bronze.hevo_submissions s ON s.id = g.submission_id +JOIN bi_analytics.bronze.hevo_users u ON u.id = s.created_by_id +LIMIT 10 + + + + +WITH base AS ( + SELECT from_address AS address + , date_trunc('month', block_timestamp) AS month + , SUM(-rune_amount) AS net_rune_amount + FROM thorchain.transfers + GROUP BY 1, 2 + UNION + SELECT to_address AS address + , date_trunc('month', block_timestamp) AS month + , SUM(rune_amount) AS net_rune_amount + FROM thorchain.transfers + GROUP BY 1, 2 +), b2 AS ( + SELECT address + , MIN(month) AS month + , SUM(net_rune_amount) AS net_rune_amount + FROM base + GROUP BY 1 +), l AS ( + SELECT + from_address AS address + , pool_name + , sum(CASE WHEN lp_action = 'add_liquidity' THEN stake_units ELSE -stake_units END) as net_stake_units + FROM thorchain.liquidity_actions + WHERE lp_action IN ('add_liquidity','remove_liquidity') + AND address LIKE 'thor%' + GROUP BY 1, 2 +), l2 AS ( + SELECT DISTINCT address + FROM l + WHERE net_stake_units > 0 +), f AS ( + SELECT DISTINCT address + FROM silver_crosschain.ntr + WHERE address LIKE 'thor%' +), tot_pool AS ( + SELECT pool_name + , SUM(net_stake_units) AS tot_stake_units + FROM l + WHERE net_stake_units > 0 + GROUP BY 1 +), pct_pool AS ( + SELECT address + , l.pool_name + , net_stake_units + , tot_stake_units + , net_stake_units / tot_stake_units AS pct_pool + FROM l + JOIN tot_pool t ON t.pool_name = l.pool_name + WHERE net_stake_units > 0 +), p AS ( + SELECT * + , ROW_NUMBER() OVER (PARTITION BY pool_name ORDER BY block_timestamp DESC) AS rn + FROM thorchain.pool_block_balances + WHERE block_timestamp >= CURRENT_DATE - 3 +), pool_val AS ( + SELECT pool_name + , rune_amount_usd + asset_amount_usd AS tvl + FROM p + WHERE rn = 1 +), user_val AS ( + SELECT pp.address + , pct_pool * tvl AS lp_val + FROM pct_pool pp + JOIN pool_val v ON v.pool_name = pp.pool_name +) +SELECT +-- date_trunc('year', month) AS month +CASE WHEN f.address IS NULL THEN 0 ELSE 1 END AS is_flipside +, COUNT(1) AS n +, AVG(CASE WHEN net_rune_amount > 0 OR l2.address IS NOT NULL THEN 1 ELSE 0 END) AS pct_hold +, AVG(CASE WHEN net_rune_amount > 0 AND l2.address IS NULL THEN 1 ELSE 0 END) AS pct_rune_only +, AVG(CASE WHEN net_rune_amount <= 0 AND l2.address IS NOT NULL THEN 1 ELSE 0 END) AS pct_lp_only +, AVG(CASE WHEN net_rune_amount > 0 AND l2.address IS NOT NULL THEN 1 ELSE 0 END) AS pct_lp_and_rune +, SUM( COALESCE(lp_val, 0)) AS lp_val +FROM b2 +LEFT JOIN l2 ON l2.address = b2.address +LEFT JOIN f ON f.address = b2.address +LEFT JOIN user_val u ON u.address = b2.address +GROUP BY 1 +SELECT v.collection, v.mint, v.token_id, v.deal_score_rank, v.rarity_rank, v.fair_market_price , m.image_url +FROM flipside_prod_db.crosschain.nft_fair_market_value v +LEFT JOIN solana.core.dim_nft_metadata m ON m.mint = v.mint +WHERE v.mint = '3hDW84qQdbh8TcCvJWwDQ1SXbqWmG6QxX84JCq7ZP4Dp' +ORDER BY CASE WHEN COALESCE(m.image_url, 'None') = 'None' THEN 1 ELSE 0 END +LIMIT 1 + + +When I run it the first time, it doesn't work + + +SELECT v.collection, v.mint, v.token_id, v.deal_score_rank, v.rarity_rank, v.fair_market_price, '' AS image_url +FROM flipside_prod_db.crosschain.nft_fair_market_value v +WHERE v.mint = 'FWPW8ddEZYbpaQ8b6qiU5yMbFan3Myg4z55oEN5qTpsE' +LIMIT 1 + + +SELECT + block_timestamp, + 'Deposit' as action, + 'SOL Covered Call' as vault, + '8vyTqVVPmJfqFexRcMBGDAHoSCyZ52RC5sRVhYzbfU4j' as vault_address, + inner_instruction:index as index, + tx_id, + inner_instruction:instructions [0] :parsed:info:authority as user_wallet, + inner_instruction:instructions [0] :parsed:info:amount / 1e9 as amount +FROM + solana.core.fact_events +WHERE + program_id = '1349iiGjWC7ZTbu6otFmJwztms122jEEnShKgpVnNewy' + and inner_instruction:instructions [0] :parsed:info:destination = '8vyTqVVPmJfqFexRcMBGDAHoSCyZ52RC5sRVhYzbfU4j' + and inner_instruction:instructions [0] :parsed:type = 'transfer' + and SUCCEEDED = 'TRUE' +UNION +SELECT + e.block_timestamp, + 'Withdraw' as action, + 'SOL Covered Call' as vault, + '8vyTqVVPmJfqFexRcMBGDAHoSCyZ52RC5sRVhYzbfU4j' as vault_address, + inner_instruction:index as index, + e.tx_id, + case when array_size(instruction:accounts) > 12 then + instruction:accounts [1] + when array_size(instruction:accounts) < 10 then + instruction:accounts [6] + else + instruction:accounts [9] + end as user_wallet, + case when inner_instruction:instructions [1] :parsed:type = 'transfer' then + inner_instruction:instructions [1] :parsed:info:amount / 1e9 + else + inner_instruction:instructions [0] :parsed:info:amount / 1e9 + end as amount +FROM + solana.core.fact_events e +WHERE + program_id = '1349iiGjWC7ZTbu6otFmJwztms122jEEnShKgpVnNewy' + AND ( + ( + inner_instruction:instructions [1] :parsed:info:source = '8vyTqVVPmJfqFexRcMBGDAHoSCyZ52RC5sRVhYzbfU4j' + AND inner_instruction:instructions [1] :parsed:type = 'transfer' + ) + OR ( + inner_instruction:instructions [0] :parsed:info:source = '8vyTqVVPmJfqFexRcMBGDAHoSCyZ52RC5sRVhYzbfU4j' + AND inner_instruction:instructions [0] :parsed:type = 'transfer' + ) + ) and SUCCEEDED = 'TRUE' + and user_wallet = 'BEmgLNjcUJx8ibdoryYxJDZVEovADSZnWCccs1HLgt37' +UNION +SELECT + block_timestamp, + 'Initiate Withdraw' as action, + 'SOL Covered Call' as vault, + 'DRLcUXwMcFf8itWJy7NdzKuWrZep1HceLaTgRRDs51SH' as vault_address, + inner_instruction:index as index, + tx_id, + inner_instruction:instructions [0] :parsed:info:authority as user_wallet, + inner_instruction:instructions [0] :parsed:info:amount / 1e9 as amount +FROM + solana.core.fact_events +WHERE + program_id = '1349iiGjWC7ZTbu6otFmJwztms122jEEnShKgpVnNewy' + and inner_instruction:instructions [0] :parsed:info:destination = 'DRLcUXwMcFf8itWJy7NdzKuWrZep1HceLaTgRRDs51SH' + and inner_instruction:instructions [0] :parsed:type = 'transfer' + and SUCCEEDED = 'TRUE' + and user_wallet = 'BEmgLNjcUJx8ibdoryYxJDZVEovADSZnWCccs1HLgt37' + + + + +Education on the site +Andy8052 +BetDex taking on risk + +SELECT * +FROM silver_crosschain.address_labels +WHERE address = '6z8T7je8BZfCbUaEpX3uNVGB1rsjXdQSRTashfArJjDP' + +SELECT * +FROM solana.dim_labels +WHERE mint = '6z8T7je8BZfCbUaEpX3uNVGB1rsjXdQSRTashfArJjDP' + +SELECT m.mint +, m.block_timestamp +, m.tx_id +, m.mint_price +FROM solana.fact_nft_mints m +WHERE tx_id = '4VzwY68wQ7shxyxFtxexWufe76ojZaPEpT1EYEKBQvZroVdEkSdx58m3sBpYAaJpFV7neCxRUoHCUx9XQTmwXCDv' + + +SELECT l.address +, m.block_timestamp +, m.tx_id +, m.mint_price +FROM flipside_prod_db.solana.dim_labels l +JOIN solana.fact_nft_mints m ON m.mint = l.address +WHERE label_type = 'nft' +AND label = 'bulldog doghouses' + + +SELECT label, address, COUNT(1) AS n +FROM solana.dim_labels +GROUP BY 1, 2 +HAVING COUNT(1) > 1 +ORDER BY 3 DESC + + +SELECT cp.project_name +, cp.title +, cp.slug +, b.title AS bounty_title +, b.difficulty +, u.discord_handle +, g.created_at AS grade_time +, grade_time::date AS grade_date +, total_score +, result_url +FROM bi_analytics.bronze.hevo_grades g +JOIN bi_analytics.bronze.hevo_submissions s ON s.id = g.submission_id +JOIN bi_analytics.bronze.hevo_users u ON u.id = s.created_by_id +JOIN bi_analytics.bronze.hevo_claims c ON c.id = s.claim_id +JOIN bi_analytics.bronze.hevo_bounties b ON b.id = c.bounty_id +JOIN bi_analytics.bronze.hevo_campaigns cp ON cp.id = b.campaign_id +WHERE total_score IS NOT NULL +AND cp.project_name ilike '%thor%' +ORDER BY grade_time DESC +LIMIT 1000 + + +WITH base AS ( + SELECT label + , MIN(m.block_timestamp::date) AS mn_date + , MAX(m.block_timestamp::date) AS mx_date + , MIN(m.mint_price) AS mn_price + , MAX(m.mint_price) AS mx_price + , COUNT(1) AS n_mints + FROM solana.core.fact_nft_mints m + JOIN solana.dim_labels l on m.mint = l.address + AND block_timestamp >= '2022-01-01' + GROUP BY 1 + ORDER BY 1 +), filter AS ( + SELECT DISTINCT label + FROM solana.core.fact_nft_sales s + LEFT JOIN solana.dim_labels l on s.mint = l.address + WHERE marketplace in ('magic eden v1', 'magic eden v2') + AND block_timestamp >= '2022-01-01' +) +SELECT b.* +, datediff('days', mn_date, mx_date) AS mint_days +FROM base b +JOIN filter f ON f.label = b.label +ORDER BY mint_days DESC + +WITH t AS ( + SELECT DISTINCT tx_id + FROM thorchain.swaps + WHERE block_timestamp >= CURRENT_DATE - 1 + LIMIT 10 +) +SELECT * +FROM thorchain.swaps s +JOIN t ON t.tx_id = s.tx_id +ORDER BY tx_id + + + +WITH t AS ( + SELECT tx_id, COUNT(1) AS n + FROM thorchain.swaps + WHERE block_timestamp >= CURRENT_DATE - 1 + GROUP BY 1 +) +SELECT s.*, CASE WHEN t.n = 1 THEN 0 ELSE 1 END AS two_sided_swap_via_rune +FROM thorchain.swaps s +JOIN t ON t.tx_id = s.tx_id +WHERE block_timestamp >= CURRENT_DATE - 1 + + +https://docs.google.com/presentation/d/1u1f41MsaEpTdCtBnR-lrX6yoIJeh2gOZDK6_54E70y0/edit#slide=id.g132b60abcc1_0_152 + + +SELECT ARRAY_SIZE(inner_instructions[0]:instructions) AS sz +, COUNT(1) AS n +FROM solana.fact_transactions +WHERE block_timestamp >= CURRENT_DATE - 30 +AND instructions[0]:programId = 'GrcZwT9hSByY1QrUTaRPp6zs5KxAA5QYuqEhjT1wihbm' +GROUP BY 1 + +With times as +( + Select solana.core.fact_nft_mints.BLOCK_TIMESTAMP, Solana.core.dim_labels.LABEL + From Solana.core.fact_nft_mints + INNER JOIN solana.core.dim_labels on solana.core.fact_nft_mints.MINT = solana.core.dim_labels.ADDRESS +), +//this is for the combination of mint and labels tabel where it connects the adress and gives the timestamp and english label +second_table as (Select DISTINCT label,block_timestamp +From (SELECT RANK() OVER (PARTITION BY label order by block_timestamp) + as RN,label,block_TIMESTAMP //as date + from times) as ST +Where ST.RN = 1), +//this is creating the min times for each collection +min_mint_table as( +Select label,block_timestamp as date +From second_table +), first_sale AS ( + SELECT l.label, MIN(block_timestamp) AS first_sale_date + FROM solana.fact_nft_sales s + INNER JOIN solana.dim_labels l on l.address = s.mint +), +//just renaming blocktimestamp as date to seprate in combined table +combined_mint_table as( +Select t.block_timestamp +,t.label +, m.date +, f.first_sale_date +,TIMESTAMPDIFF('day', t.block_timestamp, first_sale_date) AS sale_difference +,TIMESTAMPDIFF('day', m.date, t.block_timestamp) AS difference +from times t +JOIN first_sale f ON f.label = t.label +JOIN min_mint_table m ON m.label = t.label +), n_mints AS ( + SELECT label, COUNT(DISTINCT address) AS n + FROM solana.core.dim_labels + GROUP BY 1 +) +Select t.label, n_mints, MAX(difference) AS mx, AVG(difference) as avg +From combined_mint_table t +JOIN n_mints n ON n.label = t.label +GROUP BY 1, 2 +ORDER BY 3 DESC WITH base AS ( SELECT t.from AS address @@ -520,11 +2750,12 @@ FROM thorchain.liquidity_actions GROUP BY 1, 2 -SELECT to_address +SELECT block_timestamp::date AS date , COUNT(1) AS n +, SUM(RUNE_AMOUNT) AS RUNE_AMOUNT FROM thorchain.liquidity_actions -WHERE lp_action = 'add_liquidity' -AND to_address like 'thor%' +WHERE lp_action = 'remove_liquidity' +AND pool_name LIKE 'TERRA%' GROUP BY 1 @@ -677,6 +2908,11 @@ union select count(distinct(address)) - count(distinct(from_address)) as total_wallets_removed_liquidity, 'total_wallets_removed_liquidity' from earliest_total_add, earliest_still_add +SELECT record_content[0]:collection AS collection +, COUNT(1) AS n +FROM flipside_prod_db.bronze.prod_data_science_uploads_1748940988 +GROUP BY 1 +ORDER BY 2 DESC WITH base AS ( SELECT * @@ -726,8 +2962,83 @@ WITH base AS ( , old_sd * new_floor / old_floor AS cur_sd , old_fair_market_price + ((new_floor - old_floor) * lin_coef) + (( new_floor - old_floor) * log_coef * old_fair_market_price / old_floor) AS new_fair_market_price FROM base5 +), b7 AS ( + SELECT collection + , mint + , token_id + , deal_score_rank + , rarity_rank + , new_floor AS floor_price + , ROUND(CASE WHEN new_fair_market_price < floor_price THEN floor_price ELSE new_fair_market_price END, 2) AS fair_market_price + , ROUND(CASE WHEN new_fair_market_price - cur_sd < floor_price * 0.975 THEN floor_price * 0.975 ELSE new_fair_market_price - cur_sd END, 2) AS price_low + , ROUND(CASE WHEN new_fair_market_price + cur_sd < floor_price * 1.025 THEN floor_price * 1.025 ELSE new_fair_market_price + cur_sd END, 2) AS price_high + FROM base6 +) + + +WITH base AS ( + SELECT * + FROM flipside_prod_db.bronze.prod_data_science_uploads_1748940988 + WHERE record_content[0]:collection IS NOT NULL + AND record_metadata:key like '%nft-deal-score-rankings-%' +), base2 AS ( + SELECT t.value:collection::string AS collection + , t.value:cur_floor AS old_floor + , t.value:cur_sd AS old_sd + , t.value:deal_score_rank AS deal_score_rank + , t.value:fair_market_price AS old_fair_market_price + , t.value:lin_coef AS lin_coef + , t.value:log_coef AS log_coef + , t.value:rarity_rank AS rarity_rank + , t.value:token_id AS token_id + , ROW_NUMBER() OVER (PARTITION BY collection, token_id ORDER BY record_metadata:CreateTime DESC) AS rn + FROM base + , LATERAL FLATTEN( + input => record_content + ) t +), base3 AS ( + SELECT * + FROM flipside_prod_db.bronze.prod_data_science_uploads_1748940988 + WHERE record_content[0]:collection IS NOT NULL + AND record_metadata:key like '%nft-deal-score-floors-%' +), base4 AS ( + SELECT t.value:collection::string AS collection + , t.value:cur_floor AS new_floor + , b.* + , ROW_NUMBER() OVER (PARTITION BY collection ORDER BY record_metadata:CreateTime DESC) AS rn + FROM base3 b + , LATERAL FLATTEN( + input => record_content + ) t +), metadata AS ( + SELECT project_name AS collection + , token_id + , mint + , ROW_NUMBER() OVER (PARTITION BY collection, token_id ORDER BY created_at_timestamp DESC) AS rn + FROM solana.dim_nft_metadata + WHERE mint IS NOT NULL +), base5 AS ( + SELECT b2.* + , m.mint + , b4.new_floor + FROM base2 b2 + JOIN base4 b4 ON b2.collection = b4.collection AND b4.rn = 1 + LEFT JOIN metadata m ON m.token_id = b2.token_id AND m.collection = b2.collection AND m.rn = 1 + WHERE b2.rn = 1 +), base6 AS ( + SELECT * + , old_sd * new_floor / old_floor AS cur_sd + , old_fair_market_price + ((new_floor - old_floor) * lin_coef) + (( new_floor - old_floor) * log_coef * old_fair_market_price / old_floor) AS new_fair_market_price + FROM base5 +), b7 AS ( + SELECT mint + , COUNT(1) AS n + FROM base6 + GROUP BY 1 ) SELECT collection +, b.mint +, n , token_id , deal_score_rank , rarity_rank @@ -735,7 +3046,8 @@ SELECT collection , ROUND(CASE WHEN new_fair_market_price < floor_price THEN floor_price ELSE new_fair_market_price END, 2) AS fair_market_price , ROUND(CASE WHEN new_fair_market_price - cur_sd < floor_price * 0.975 THEN floor_price * 0.975 ELSE new_fair_market_price - cur_sd END, 2) AS price_low , ROUND(CASE WHEN new_fair_market_price + cur_sd < floor_price * 1.025 THEN floor_price * 1.025 ELSE new_fair_market_price + cur_sd END, 2) AS price_high -FROM base6 +FROM base6 b +JOIN b7 ON b7.mint = b.mint @@ -1350,8 +3662,16 @@ FROM thorchain.bond_events WHERE address = 'thor1xd4j3gk9frpxh8r22runntnqy34lwzrdkazldh' -SELECT COUNT(1) -FROM midgard.bond_events +SELECT COUNT(1), MAX(block_timestamp) +FROM bronze_midgard_2_6_9_20220405.midgard_fee_events + +SELECT COUNT(1), MAX(block_timestamp) +FROM midgard.bond_events +WHERE block_timestamp <= 1655955911000000000 +WHERE block_timestamp <= 1655986950000000000 + +SELECT COUNT(1), MAX(block_timestamp) +FROM thorchain.bond_events @@ -2023,6 +4343,9 @@ FROM ethereum.events_emitted WHERE contract_address = '0xa5f2211b9b8170f694421f2046281775e8468044' LIMIT 100 +106. [Hard] New User Onboarding +When a new wallet gets created, where are they first getting their $RUNE? Is it from a direct transfer? Or from a centralized exchange? From a $RUNE upgrade from BNB.RUNE or ETH.RUNE? Or from a swap from another chain's asset to $RUNE? If it's a swap, which chain / asset are they coming from? Are there any trends over time you can see? + conn = psycopg2.connect( host="vic5o0tw1w-repl.twtim97jsb.tsdb.cloud.timescale.com", @@ -2926,6 +5249,954 @@ ORDER BY pool_deduct SELECT * FROM midgard.fee_events LIMIT 100 +Chat with +Central wallets tranferring into new wallets + + +WITH base AS ( + SELECT INITCAP(l.label) AS collection, s.purchaser, SUM(s.sales_amount) AS volume + FROM solana.core.fact_nft_sales s + JOIN solana.core.dim_labels l ON l.address = s.mint + GROUP BY 1, 2 +), b2 AS ( + SELECT collection, purchaser, volume, SUM(volume) OVER (PARTITION BY collection) AS tot_volume + , ROW_NUMBER() OVER (PARTITION BY collection ORDER BY volume DESC) AS rn + FROM base +), b3 AS ( + SELECT collection, tot_volume, SUM(volume) AS top_vol + , ROW_NUMBER() OVER (ORDER BY tot_volume DESC) AS rn_2 + FROM b2 + WHERE rn <= 50 + GROUP BY 1, 2 +) +SELECT *, top_vol / tot_volume AS pct_top +FROM b3 +WHERE rn_2 <= 25 +ORDER BY pct_top DESC + + +SELECT COUNT(tx) +, COUNT(asset) +, COUNT(asset_e8) +, COUNT(pool_deduct) +, COUNT(block_timestamp) +, COUNT(1) AS n +FROM midgard.fee_events + +Using NFT artist from in-house +Build out the brand +Get onto Magic Eden launchpad +Hit things quick + + +Outfit_Astronaut Light +Outfit_Astronaut Orange +Head_Crown +Attribute count_1 7 +Edition_Alien +Outfit_Armor Light + + +SELECT block_timestamp::date AS date +, COUNT(1) AS n +FROM solana.core.fact_nft_sales +WHERE sales_amount = 0 +GROUP BY 1 +ORDER BY 1 + +SELECT * +FROM bronze_midgard_2_6_9_20220405.switch_events +LIMIT 100 + + +SELECT COUNT(1) AS n +FROM midgard.fee_events +WHERE block_timestamp <= 1656360639000000000 + AND block_timestamp < 1654041600000000000 + +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +, MAX(block_timestamp) AS mxx +FROM thorchain.fee_events + +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +, MAX(block_timestamp) AS mxx +FROM thorchain.bond_events + + + + + +WITH base AS ( + SELECT tx + , asset + , asset_e8 + , pool_deduct + , block_timestamp + , COUNT(1) AS n + FROM midgard.fee_events + GROUP BY 1, 2, 3, 4, 5 + HAVING COUNT(1) > 1 +) +SELECT MAX(block_timestamp) AS mx +, COUNT(1) AS n +, COUNT(1) AS nn +FROM base + +fee_events +bond_events +switch_events +transfer_events + + +WITH base AS ( + SELECT tx + , asset + , asset_e8 + , pool_deduct + , block_timestamp + , COUNT(1) AS n + FROM bronze_midgard_2_6_9_20220405.midgard_fee_events + GROUP BY 1, 2, 3, 4, 5 + HAVING COUNT(1) > 1 +) +SELECT MAX(block_timestamp) AS mx +, COUNT(1) AS n +, COUNT(1) AS nn +FROM base + +WITH base AS ( + SELECT tx_id + , asset + , asset_e8 + , pool_deduct + , block_timestamp + , COUNT(1) AS n + FROM thorchain.fee_events + GROUP BY 1, 2, 3, 4, 5 + HAVING COUNT(1) > 1 +) +SELECT MAX(block_timestamp) AS mx +, COUNT(1) AS n +, COUNT(1) AS nn +FROM base + + + +WITH base AS ( + SELECT tx_id + , asset + , asset_e8 + , pool_deduct + , block_timestamp + , COUNT(1) AS n + FROM thorchain.fee_events + GROUP BY 1, 2, 3, 4, 5 + HAVING COUNT(1) > 1 +) +SELECT MAX(block_timestamp) AS mx +, COUNT(1) AS n +, COUNT(1) AS nn +FROM base + +WITH base AS ( + SELECT from_addr + , to_addr + , asset + , amount_e8 + , block_timestamp + , COUNT(1) AS cnt + FROM midgard.transfer_events + WHERE block_timestamp >= 1650092800000000000 + AND block_timestamp < 1654041600000000000 + GROUP BY 1, 2, 3, 4, 5 + HAVING COUNT(1) > 1 +) +SELECT asset +, COUNT(1) AS n +FROM base +GROUP BY 1 + + + +SELECT tx +, chain +, from_addr +, to_addr +, asset +, asset_e8 +, memo +, bond_type +, e8 +, block_timestamp +, COUNT(1) AS cnt +FROM FLIPSIDE_PROD_DB.BRONZE_MIDGARD_2_6_9_20220405.BOND_EVENTS_PK_COUNT +GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 + + +SELECT CASE WHEN tx IS NULL THEN 'null' ELSE tx END AS clean_tx +, COUNT(1) AS cnt +FROM midgard.bond_events +WHERE tx IS NULL OR tx = '' +GROUP BY 1 + + +SELECT CASE WHEN chain IS NULL THEN 'null' ELSE chain END AS chain_tx +, COUNT(1) AS cnt +FROM midgard.bond_events +WHERE tx IS NULL OR chain = '' +GROUP BY 1 + +SELECT project_name +, mint +, token_id +, token_metadata:"Accessories"::string AS "Accessories" +, token_metadata:"Attribute Count"::int AS "Attribute Count" +, token_metadata:"Background"::string AS "Background" +, token_metadata:"Big Drip"::string AS "Body" +, token_metadata:"Coat"::string AS "Coat" +, token_metadata:"Eyes"::string AS "Eyes" +, token_metadata:"Head"::string AS "Head" +, token_metadata:"Mouth"::string AS "Mouth" +, token_metadata:"Shoe"::string AS "Shoe" +FROM solana.dim_nft_metadata +WHERE project_name = 'Bubblegoose Ballers' +LIMIT 10 + +SELECT DISTINCT mint, project_name AS collection +FROM solana.dim_nft_metadata m +LEFT JOIN solana.dim_labels l ON l.address = m.mint +WHERE l.address IS NULL + +WITH sales AS ( + SELECT l.label AS collection, SUM(sales_amount) AS volume + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON l.address = s.mint + GROUP BY 1 +), base AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY volume DESC) AS volume_rank + FROM sales + ORDER BY volume DESC + LIMIT 50 +), b2 AS ( + SELECT DISTINCT collection, purchaser, mint + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON l.address = s.mint + JOIN base b ON b.collection = l.label + UNION + SELECT DISTINCT collection, purchaser, mint + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON l.address = m.mint + JOIN base b ON b.collection = l.label +) +SELECT DISTINCT collection, purchaser, mint +FROM b2 + + +WITH a AS ( + SELECT DISTINCT mint + FROM solana.fact_nft_sales s +), b AS ( + SELECT l.label + , SUM(s.mint_price) AS mint_volume + , SUM(s.mint_price) / COUNT(DISTINCT s.mint) AS mint_price + FROM solana.fact_nft_mints s + JOIN solana.dim_labels l ON l.address = s.mint + WHERE mint_currency = 'So11111111111111111111111111111111111111111' + AND block_timestamp >= '2022-03-01' + GROUP BY 1 +), c AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY mint_volume DESC) AS mint_volume_rank + FROM b +), d AS ( + SELECT l.label + , SUM(s.sales_amount) AS sales_volume + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON l.address = s.mint + GROUP BY 1 +), e AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY sales_volume DESC) AS sales_volume_rank + FROM d +) +SELECT c.label AS collection +, c.mint_volume +, c.mint_volume_rank +, e.sales_volume +, e.sales_volume_rank +, c.mint_price +, COUNT(DISTINCT m.mint) AS n_mints +, COUNT(DISTINCT a.mint) AS n_sell +, 1 - (n_sell / n_mints) AS pct_hold +FROM solana.fact_nft_mints m +JOIN solana.dim_labels l ON l.address = m.mint +JOIN c ON c.label = l.label +JOIN e ON e.label = l.label +LEFT JOIN a ON a.mint = m.mint +GROUP BY 1, 2, 3, 4, 5, 6 + + +SELECT * +FROM solana.dim_labels +WHERE label = 'lizard' + + + +WITH sales AS ( + SELECT INITCAP(l.label) AS collection + , SUM(s.sales_amount) AS volume + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON l.address = s.mint + WHERE date_trunc('month', s.block_timestamp) = '2022-05-01' + GROUP BY 1 +), base AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY volume DESC) AS rn + FROM sales +), b2 AS ( + SELECT * + , CASE WHEN rn < 10 THEN CONCAT('0', rn::varchar, '. ', collection) ELSE '10. Other' END AS label + FROM base +), b3 AS ( + SELECT label + , SUM(volume) AS volume + FROM b2 + GROUP BY 1 + ORDER BY 1 +) +SELECT * +, volume / SUM(volume) OVER () AS pct +FROM b3 + + +WITH sales AS ( + SELECT INITCAP(l.label) AS collection + , date_trunc('month', s.block_timestamp) AS month + , SUM(s.sales_amount) AS volume + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON l.address = s.mint + WHERE date_trunc('month', s.block_timestamp) >= '2022-01-01' + GROUP BY 1, 2 +), base AS ( + SELECT * + , ROW_NUMBER() OVER (PARTITION BY month ORDER BY volume DESC) AS rn + FROM sales +), b2 AS ( + SELECT * + , CONCAT(LEFT(month::varchar, 2), '. ', TO_CHAR(month, 'month'), ' - ', collection) AS label + FROM base + WHERE rn = 1 +) +SELECT * +FROM b2 + +SELECT L1_LABEL, COUNT(1) AS n +FROM FLIPSIDE_PROD_DB.SILVER_CROSSCHAIN.ADDRESS_LABELS +GROUP BY 1 +ORDER BY 2 DESC + + +SELECT L1_LABEL, COUNT(1) AS n +FROM FLIPSIDE_PROD_DB.SILVER_CROSSCHAIN.ADDRESS_LABELS +//WHERE blockchain = 'solana' +//ORDER BY insert_date desc +LIMIT 100 + +WITH prices AS ( + SELECT recorded_at::date AS date + , AVG(price) AS sol_price + FROM silver.prices_v2 + WHERE asset_id = 'solana' + GROUP BY 1 +), mints AS ( + SELECT INITCAP(l.label) AS collection + , SUM(mint_price) AS sol_volume + , COUNT(DISTINCT m.mint) AS n_mints + , MIN(block_timestamp::date) AS mn_date + , MAX(block_timestamp::date) AS mx_date + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON l.address = m.mint + GROUP BY 1 +), base AS ( + SELECT m.* + , p.sol_price + , p.sol_price * sol_volume AS usd_volume + , sol_volume / n_mints AS mint_price + , ROW_NUMBER() OVER (ORDER BY usd_volume DESC) AS usd_volume_rank + FROM mints m + JOIN prices p ON p.date = m.mn_date + WHERE mint_price <= 10 + ORDER BY usd_volume DESC +) +SELECT * +FROM base +ORDER BY usd_volume DESC + +WITH a AS ( + SELECT DISTINCT mint + FROM solana.fact_nft_sales s +), b AS ( + SELECT CASE WHEN l.label = 'lizard' THEN 'Reptilian Renegade' ELSE l.label END AS label + , SUM(s.mint_price) AS mint_volume + , SUM(s.mint_price) / COUNT(DISTINCT s.mint) AS mint_price + FROM solana.fact_nft_mints s + JOIN solana.dim_labels l ON l.address = s.mint + WHERE mint_currency = 'So11111111111111111111111111111111111111111' + AND block_timestamp >= '2022-03-01' + -- data issue, need to look into this + AND NOT l.label IN ( 'degen trash pandas' ) + GROUP BY 1 +), c AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY mint_volume DESC) AS mint_volume_rank + FROM b +), d AS ( + SELECT l.label + , SUM(s.sales_amount) AS sales_volume + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON l.address = s.mint + WHERE block_timestamp >= '2022-03-01' + GROUP BY 1 +), e AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY sales_volume DESC) AS sales_volume_rank + FROM d +), base AS ( + SELECT INITCAP(c.label) AS collection + , c.mint_volume + , c.mint_volume_rank + , e.sales_volume + , e.sales_volume_rank + , c.mint_price + , COUNT(DISTINCT m.mint) AS n_mints + , COUNT(DISTINCT a.mint) AS n_sell + , 100 * (1 - (n_sell / n_mints)) AS pct_hold + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON l.address = m.mint + JOIN c ON c.label = l.label + JOIN e ON e.label = l.label + LEFT JOIN a ON a.mint = m.mint + GROUP BY 1, 2, 3, 4, 5, 6 + ORDER BY sales_volume_rank + LIMIT 35 +), b2 AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY pct_hold DESC) AS rn + FROM base +) +SELECT * +, CASE WHEN rn < 10 THEN CONCAT('0', rn::varchar) ELSE rn::varchar END AS clean_rn +, CONCAT(clean_rn, '. ', collection) AS label +FROM b2 +ORDER BY label + + +WITH base AS ( + SELECT token_metadata:Fur::string AS Fur + , AVG(sales_amount) AS avg_price + , MEDIAN(sales_amount) AS med_price + FROM solana.dim_nft_metadata m + JOIN solana.fact_nft_sales s ON s.mint = m.mint + WHERE project_name = 'Okay Bears' + GROUP BY 1 +), b2 AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY avg_price DESC) AS rn + FROM base +) +SELECT * +, CASE WHEN rn <= 10 THEN CONCAT('0', rn::varchar) ELSE rn END AS clean_rn +, CONCAT(clean_rn, '. ', Fur) AS value +FROM b2 +ORDER BY value + + +SELECT CASE WHEN s.block_timestamp::date <= '2022-05-04' THEN 'Apr 26 - May 4' ELSE 'After May 4' END AS sale_date +, CASE WHEN token_metadata:"Attribute Count"::string = '4' THEN '4 Att' ELSE 'Others' END AS bear_type +, CASE WHEN sale_date = 'Apr 26 - May 4' AND bear_type = '4 Att' THEN '1. Apr 26 - May 4: 4 Att' +WHEN sale_date = 'Apr 26 - May 4' THEN '2. Apr 26 - May 4: Others' +WHEN bear_type = '4 Att' THEN '3. After May 4: 4 Att' +ELSE '4. After May 4: Others' END AS sale_type +, AVG(s.sales_amount) AS avg_price +, MEDIAN(s.sales_amount) AS med_price +FROM solana.dim_nft_metadata m +JOIN solana.fact_nft_sales s ON s.mint = m.mint +WHERE project_name = 'Okay Bears' +GROUP BY 1, 2, 3 +ORDER BY 1, 2, 3 + + +SELECT CASE WHEN s.block_timestamp::date <= '2022-05-04' THEN 'Apr 26 - May 4' ELSE 'After May 4' END AS sale_date +, CASE WHEN token_metadata:"Attribute Count"::string = '4' THEN '4 Att' ELSE 'Others' END AS bear_type +, CASE WHEN sale_date = 'Apr 26 - May 4' AND bear_type = '4 Att' THEN '1. Apr 26 - May 4: 4 Att' +WHEN sale_date = 'Apr 26 - May 4' THEN '2. Apr 26 - May 4: Others' +WHEN bear_type = '4 Att' THEN '3. After May 4: 4 Att' +ELSE '4. After May 4: Others' END AS sale_type +, AVG(s.sales_amount) AS avg_price +, MEDIAN(s.sales_amount) AS med_price +FROM solana.dim_nft_metadata m +JOIN solana.fact_nft_sales s ON s.mint = m.mint +WHERE project_name = 'Okay Bears' +GROUP BY 1, 2, 3 +ORDER BY 1, 2, 3 + + +-- transfer_events_pk_count +SELECT from_addr +, to_addr +, asset +, amount_e8 +, block_timestamp +, COUNT(1) AS cnt +FROM midgard.transfer_events +WHERE block_timestamp >= 1655294400000000000 +GROUP BY 1, 2, 3, 4, 5 +HAVING COUNT(1) > 1 + + +WITH base AS ( + SELECT from_addr + , to_addr + , asset + , amount_e8 + , block_timestamp + , COUNT(1) AS cnt + FROM bronze_midgard_2_6_9_20220405.midgard_transfer_events + WHERE block_timestamp >= 1655294400000000000 + AND block_timestamp <= 1655906524979260126 + GROUP BY 1, 2, 3, 4, 5 + HAVING COUNT(1) > 1 +) +SELECT COUNT(1) AS n +, MAX(block_timestamp) +FROM base + + +SELECT from_addr +, to_addr +, asset +, amount_e8 +, block_timestamp +FROM midgard.transfer_events +WHERE from_addr IS NULL +OR to_addr IS NULL +OR asset IS NULL +OR amount_e8 IS NULL +OR block_timestamp IS NULL + + +WITH base AS ( + SELECT from_addr + , to_addr + , asset + , amount_e8 + , block_timestamp + , COUNT(1) AS cnt + FROM midgard.transfer_events + WHERE block_timestamp >= 1655294400000000000 + AND block_timestamp <= 1655906524979260126 + GROUP BY 1, 2, 3, 4, 5 + HAVING COUNT(1) > 1 +) +SELECT COUNT(1) AS n +, MAX(block_timestamp) +FROM base + + +WITH base AS ( +SELECT from_addr +, to_addr +, asset +, amount_e8 +, block_timestamp +, COUNT(1) AS cnt +FROM midgard.transfer_events +-- WHERE block_timestamp < 1620299757083512012 +WHERE block_timestamp < 1620299000000000000 +WHERE block_timestamp < 1640995200000000000 +WHERE block_timestamp < 1637301700000000000 +WHERE block_timestamp < 1633046400000000000 +WHERE block_timestamp >= 1640995200000000000 +AND block_timestamp < 1654041600000000000 +AND block_timestamp < 1655294400000000000 +GROUP BY 1, 2, 3, 4, 5 +HAVING COUNT(1) > 1 +) +SELECT COUNT(1) AS n +FROM base + +-- bond_events_pk_count +SELECT tx +, chain +, from_addr +, to_addr +, asset +, asset_e8 +, memo +, bond_type +, e8 +, block_timestamp +, COUNT(1) AS cnt +FROM midgard.bond_events +GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 + + +SELECT tx +, COALESCE(chain, '') AS chain +, COALESCE(from_addr, '') AS from_addr +, COALESCE(to_addr, '') AS to_addr +, COALESCE(asset, '') AS asset +, asset_e8 +, COALESCE(memo, '') AS memo +, bond_type +, e8 +, block_timestamp +, COUNT(1) AS cnt +FROM midgard.bond_events + + +SELECT COUNT(tx) AS tx +, COUNT(chain) AS cahin +, COUNT(from_addr) As from_addr +, COUNT(to_addr) as to_address +, COUNT(asset) as asset +, COUNT(asset_e8) as asset_e8 +, COUNT(memo) as memo +, COUNT(bond_type) as bond_type +, COUNT(e8) as 38 +, COUNT(block_timestamp) as block_timestamp +, COUNT(1) AS cnt +FROM midgard.bond_events + +SELECT COUNT(tx) AS tx +, COUNT(chain) AS cahin +, COUNT(from_addr) As from_addr +, COUNT(to_addr) as to_address +, COUNT(asset) as asset +, COUNT(asset_e8) as asset_e8 +, COUNT(memo) as memo +, COUNT(bond_type) as bond_type +, COUNT(e8) as 38 +, COUNT(block_timestamp) as block_timestamp +, COUNT(1) AS cnt +FROM bronze_midgard_2_6_9_20220405.midgard_bond_events + +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +FROM flipside_dev_db.thorchain.bond_actions + +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +FROM flipside_dev_db.bronze_midgard_2_6_9_20220405.bond_events + +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +FROM midgard.bond_events +WHERE block_timestamp <= 1655880505782611860 + + +-- fee_events +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +FROM flipside_dev_db.thorchain.fee_events + +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +FROM flipside_dev_db.bronze_midgard_2_6_9_20220405.midgard_fee_events + +SELECT INITCAP(l.label) AS collection +, SUM(s.sales_amount) AS volume +, AVG(s.sales_amount) AS avg_price +, MEDIAN(s.sales_amount) AS med_price +FROM solana.fact_nft_sales s +JOIN solana.dim_labels l ON l.address = s.mint +ORDER BY 2 DESC + +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +FROM midgard.fee_events +WHERE block_timestamp <= 1655330913000000000 +WHERE block_timestamp <= 1655885723960048067 + + +-- switch_events +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +FROM flipside_dev_db.thorchain.switch_events + +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +FROM flipside_dev_db.bronze_midgard_2_6_9_20220405.midgard_switch_events + +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +FROM midgard.switch_events +WHERE block_timestamp <= 1655884351385908318 + + +-- switch_events_pk_count +SELECT tx +, from_addr +, to_addr +, burn_asset +, burn_e8 +, mint_e8 +, block_timestamp +, COUNT(1) AS cnt +FROM midgard.switch_events +GROUP BY 1, 2, 3, 4, 5, 6, 7 + +-- transfer_events_pk_count +SELECT from_addr +, to_addr +, asset +, amount_e8 +, block_timestamp +, COUNT(1) AS cnt +FROM midgard.transfer_events +GROUP BY 1, 2, 3, 4, 5 + + + + +WITH base AS ( + SELECT from_addr + , to_addr + , asset + , amount_e8 + , block_timestamp + , COUNT(1) AS n + FROM midgard.transfer_events + WHERE block_timestamp < 16540416000000000000 + AND block_timestamp >= 16513632000000000000 + GROUP BY 1, 2, 3, 4, 5 + HAVING COUNT(1) > 1 +) +SELECT MAX(block_timestamp) AS mx +, COUNT(1) AS n +, COUNT(1) AS nn +FROM base + + +WITH base AS ( + SELECT from_addr + , to_addr + , asset + , amount_e8 + , block_timestamp + , COUNT(1) AS n + FROM bronze_midgard_2_6_9_20220405.midgard_transfer_events + WHERE block_timestamp >= 1653048000000000000 + GROUP BY 1, 2, 3, 4, 5 + HAVING COUNT(1) > 1 +) +SELECT MAX(block_timestamp) AS mx +, COUNT(1) AS n +, COUNT(1) AS nn +FROM base + + + +SELECT * +FROM bronze_midgard_2_6_9_20220405.midgard_fee_events +LIMIT 10 + + + +SELECT DISTINCT LOWER(project_name) AS lower_collection +FROM solana.dim_nft_metadata + + +Solana thread on NFT Metadata +THORChain bounty question + grading +Met w/ 9R to discuss expectations +Discussing future bounty qs w/ community in Discord + + +WITH base AS ( + SELECT s.block_timestamp::date AS date + , INITCAP(m.project_name) AS collection + , COUNT(1) AS n + , SUM(sales_amount) AS volume + FROM solana.fact_nft_sales s + JOIN solana.dim_nft_metadata m ON m.mint = s.mint + WHERE s.block_timestamp::date >= CURRENT_DATE - 7 + GROUP BY 1, 2 + ORDER BY 1, 2 +), b2 AS ( + SELECT * + , ROW_NUMBER() OVER (PARTITION BY date ORDER BY volume DESC) AS rn + FROM base +) +SELECT * +FROM b2 +WHERE rn <= 10 + +WITH base AS ( + SELECT s.block_timestamp::date AS date + , INITCAP(m.project_name) AS collection + , COUNT(1) AS n + , SUM(sales_amount) AS volume + FROM solana.fact_nft_sales s + JOIN solana.dim_nft_metadata m ON m.mint = s.mint + WHERE s.block_timestamp::date >= CURRENT_DATE - 20 + GROUP BY 1, 2 + ORDER BY 1, 2 +), b2 AS ( + SELECT * + , ROW_NUMBER() OVER (PARTITION BY date ORDER BY volume DESC) AS rn + FROM base +) +SELECT * +FROM b2 +WHERE rn <= 1 + +WITH base AS ( + SELECT s.mint, s.purchaser, s.block_timestamp AS date, s.sales_amount AS price + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON l.address = s.mint + WHERE l.label = 'okay bears' AND s.mint = '2E9NM1zugqH253SKiUDN22xnaVgH5vZz44Pv1e4V3PZq' + UNION + SELECT s.mint, s.purchaser, s.block_timestamp AS date, s.mint_price AS price + FROM solana.fact_nft_mints s + JOIN solana.dim_labels l ON l.address = s.mint + WHERE l.label = 'okay bears' AND s.mint = '2E9NM1zugqH253SKiUDN22xnaVgH5vZz44Pv1e4V3PZq' +) +SELECT * FROM base ORDER BY date +SELECT s.mint, s.purchaser, s.block_timestamp AS date, s.sales_amount AS price +FROM solana.fact_nft_sales s +JOIN solana.dim_labels l ON l.address = s.mint +WHERE l.label = 'okay bears' AND s.mint = '2E9NM1zugqH253SKiUDN22xnaVgH5vZz44Pv1e4V3PZq' +UNION +SELECT s.mint, s.purchaser, s.block_timestamp AS date, s.mint_price AS price +FROM solana.fact_nft_mints s +JOIN solana.dim_labels l ON l.address = s.mint +WHERE l.label = 'okay bears' AND s.mint = '2E9NM1zugqH253SKiUDN22xnaVgH5vZz44Pv1e4V3PZq' + +WITH base AS ( + SELECT s.mint, s.purchaser + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON l.address = s.mint + WHERE l.label = 'solana monkey Business' + UNION + SELECT s.mint, s.purchaser + FROM solana.fact_nft_mints s + JOIN solana.dim_labels l ON l.address = s.mint + WHERE l.label = 'solana monkey Business' +), b2 AS ( + SELECT mint, COUNT(DISTINCT purchaser) AS n_owners + FROM base + GROUP BY 1 +) +SELECT n_owners +, COUNT (1) AS n_mints +FROM b2 +GROUP BY 1 + + + +WITH base AS ( + SELECT INITCAP(l.label) AS collection + , date_trunc('week', s.block_timestamp) AS month + , SUM(sales_amount) AS volume + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON l.address = s.mint + WHERE s.block_timestamp >= '2022-01-01' + GROUP BY 1, 2 +), b2 AS ( + SELECT collection + , COUNT(1) AS n_weeks + FROM base + WHERE volume >= 2000 + GROUP BY 1 + ORDER BY 2 DESC + LIMIT 10 +), b3 AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY n_weeks DESC, collection) AS rn + FROM b2 +) +SELECT * +, CONCAT(CASE WHEN rn < 10 THEN CONCAT('0', rn::varchar) ELSE rn::varchar END, '. ', collection) AS label +, CASE WHEN collection = 'Famous Fox Federation' THEN 'Famous Fox Federation' ELSE 'Others' END AS is_fff +FROM b3 +ORDER BY label + + + + + +WITH base AS ( + SELECT INITCAP(l.label) AS collection + , date_trunc('week', s.block_timestamp) AS month + , SUM(sales_amount) AS volume + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON m.address = s.mint + WHERE m.project_name ilike 'Famous Fox Federation' + AND s.block_timestamp >= '2021-12-01' + GROUP BY 1, 2 +) +SELECT collection +, COUNT(1) AS n_weeks +FROM base +WHERE volume >= 1000 +GROUP BY 1 +ORDER BY 2 DESC + +WITH base AS ( + SELECT INITCAP(l.label) AS collection + , date_trunc('week', s.block_timestamp) AS month + , SUM(sales_amount) AS volume + FROM solana.fact_nft_sales s + JOIN solana.dim_nft_metadata m ON m.address = s.mint + WHERE s.block_timestamp >= '2021-12-01' + GROUP BY 1, 2 +) +SELECT collection +, COUNT(1) AS n_weeks +FROM base +WHERE volume >= 1000 +GROUP BY 1 +ORDER BY 2 DESC + +In terms of stuff that's been done before I think Meerkat Millionaires is closest to what we're going for, something friendly but with some swagger (and ours would be more like jocks than rich people). For other examples, Okay Bears + Catalina Whales also in this category. +A little different, but also loved the art from the Cyber Sam Gen 2 collection. +I'd also be open to something a little different, like what VBA game did. This pic that Michael found would be very intriguing. + +Something like Meerkat Millionaires / Okay Bears / Catalina Whales + +GP Review - how to get to Cantina +How to surface top analyst + +Put into snowflake +Documentation over time +Weighting by difficulty +Breaking it into tiers +Check with Sam + Dan +Variable based on score - What does the data say about what + +100,000 +10,000 / day +User queries to compute user portfolio info in vaults + + +BI for blockchains. + +Understand the nuances of the data - the impact of synths on liquidity providers +The impact of adding a new chain to the protocol + + + + + + + + +VBA Game + data-science @@ -2933,7 +6204,40 @@ DKiZEzE-CZ+o59}L boatpartydotbiz sudo cp /rstudio-data/algorand_console_data.RData ~/ -sudo cp /rstudio-data/nft_deal_score_sales.csv ~/ +sudo cp ~/ntr_data.RData /rstudio-data/ +sudo cp ~/nft_deal_score_data.RData /rstudio-data/ +sudo cp ~/nft_deal_score_listings_data.RData /rstudio-data/ +sudo cp ~/nft_deal_score_sales_data.RData /rstudio-data/ +sudo cp ~/nft_deal_score_sales.csv /rstudio-data/ + +Coat_Black Jag +Big Drip_Spaced Out +Big Drip_Midnight + +Head_King +Head_Brainfreeze +Head_Sahara Dropout +Mouth_Big Smoke +Mouth_Diamond Tasting + +Body_Porcelain +Shoe_Froggies +Head_Smoothie Dropout +Accessories_Gold Bgb Chain + +Solana NFT showcase + +1. Banbannard +2. Zook +3. Multipartite +4. hernandezngronk +5. adriaparcerisas + +SELECT COUNT(1) AS n +FROM midgard.slash_amounts + +SELECT COUNT(1) AS n +FROM bronze_midgard_2_6_9_20220405.midgard_slash_amounts SELECT pool , asset @@ -2944,6 +6248,16 @@ SELECT pool FROM bronze_midgard_2_6_9_20220405.midgard_slash_amounts GROUP BY 1, 2, 3, 4 +SELECT +node_address +, slash_points +, reason +, block_timestamp +, COUNT(1) AS n +FROM midgard.slash_points +GROUP BY 1, 2, 3, 4 +ORDER BY 5 DESC + SELECT pool , asset , asset_e8 @@ -2953,6 +6267,17 @@ SELECT pool FROM midgard.slash_amounts GROUP BY 1, 2, 3, 4 + +SELECT +COUNT(1) AS n +FROM thorchain.slash_amounts + + +SELECT +COUNT(1) AS n +, COUNT(1) AS n2 +FROM midgard.slash_amounts + sudo chmod 774 nft_deal_score_data.RData ls -l @@ -4038,39 +7363,6 @@ GROUP BY 1 -21:33:53 Failure in test dbt_utils_unique_combination_of_columns_thorchain__bond_events_TX_ID__BLOCK_ID (models/thorchain/thorchain__bond_events.yml) -21:33:53 Got 6806 results, configured to fail if != 0 -21:33:53 -21:33:53 compiled SQL at target/compiled/sql_models/models/thorchain/thorchain__bond_events.yml/dbt_utils_unique_combination_o_4c33313a0a68966742499a57f9c413f2.sql -21:33:53 -21:33:53 Failure in test dbt_utils_unique_combination_of_columns_thorchain__daily_pool_stats_DAY__POOL_NAME (models/thorchain/thorchain__daily_pool_stats.yml) -21:33:53 Got 103 results, configured to fail if != 0 -21:33:53 -21:33:53 compiled SQL at target/compiled/sql_models/models/thorchain/thorchain__daily_pool_stats.yml/dbt_utils_unique_combination_o_091de1e000a8d4666a014c0688a50b0f.sql -21:33:53 -21:33:53 Failure in test dbt_utils_unique_combination_of_columns_thorchain__pool_balance_change_events_BLOCK_ID__ASSET__REASON (models/thorchain/thorchain__pool_balance_change_events.yml) -21:33:53 Got 870 results, configured to fail if != 0 -21:33:53 -21:33:53 compiled SQL at target/compiled/sql_models/models/thorchain/thorchain__pool_balance_change_events.yml/dbt_utils_unique_combination_o_309a465be5fc79461b0d64ac3c9308c9.sql -21:33:53 -21:33:53 Failure in test dbt_utils_unique_combination_of_columns_thorchain__pool_block_statistics_DAY__ASSET (models/thorchain/thorchain__pool_block_statistics.yml) -21:33:53 Got 103 results, configured to fail if != 0 -21:33:53 -21:33:53 compiled SQL at target/compiled/sql_models/models/thorchain/thorchain__pool_block_statistics.yml/dbt_utils_unique_combination_o_e0e0f1f5a63053051ee1a1a182029461.sql -21:33:53 -21:33:53 Failure in test dbt_utils_unique_combination_of_columns_thorchain__refund_events_TX_ID__BLOCK_ID__TX_ID__TO_ADDRESS__FROM_ADDRESS__ASSET__ASSET_E8__ASSET_2ND (models/thorchain/thorchain__refund_events.yml) -21:33:53 Got 1 result, configured to fail if != 0 -21:33:53 -21:33:53 compiled SQL at target/compiled/sql_models/models/thorchain/thorchain__refund_events.yml/dbt_utils_unique_combination_o_1602ea813e0daecea06daa5bf0ce53c0.sql -21:33:53 - -thorchain__bond_events: dupes come from tx = "0000000000000000000000000000000000000000000000000000000000000000". did not implement any fix -thorchain__daily_pool_stats: dupes come from pool_block_statistics dupes, which I implemented a fix for -thorchain__pool_balance_change_events: have incorrect test, which I removed -thorchain__pool_block_statistics: implemented fix -thorchain__refund_events: not sure what the issue is there. need to do more digging - - WITH base AS ( SELECT * , synth_amount / asset_amount AS ratio @@ -4091,14 +7383,1498 @@ FROM thorchain.liquidity_actions WHERE lp_action = 'remove_liquidity' GROUP BY 1 + +WITH f AS ( + SELECT l.label + , MIN(block_timestamp) AS start_time + , MIN(block_timestamp::date) AS start_date + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON l.address = s.mint + WHERE marketplace LIKE 'magic%' + AND block_timestamp >= '2022-05-01' + GROUP BY 1 +) +SELECT start_time::date AS date +, l.label +, COUNT(1) AS n_sales +, SUM(sales_amount) AS volume +FROM solana.fact_nft_sales s +JOIN solana.dim_labels l ON l.address = s.mint +JOIN f ON DATEADD('minute', 60 * 12, start_time) >= block_timestamp AND l.label = f.label +JOIN m ON m.start_date = f.start_date AND l.label = m.label +WHERE date >= '2022-05-10' +GROUP BY 1, 2 +ORDER BY 3 DESC + + +SELECT l.label +, (block_timestamp::date) AS start_date +, COUNT(1) AS n +FROM solana.fact_nft_mints s +JOIN solana.dim_labels l ON l.address = s.mint +WHERE block_timestamp >= '2022-05-01' +AND l.label = 'fine fillies' +GROUP BY 1, 2 + +Blocksmith Labs +Cets on Creck +Citizens by Solsteads +Dazed Duck +Degen Trash Pandas +Famous Foxes +Galactic Gecko +Generous Robots +Ghostface Gen 2 +Marinade Chefs +Portals +Psyker Hideouts +Smoke Heads +Solstein +The Orcs +The Remnants +The Tower +The Vaultx DAO +Visionary Studios + +SELECT COALESCE(tx) as tx +, COUNT(from_addr) as from_addr +, COUNT(to_addr) as to_addr +, COUNT(burn_asset) AS burn_asset +, COUNT(burn_e8) AS burn_e8 +, COUNT(mint_e8) AS mint_e8 +, COUNT(block_timestamp) AS block_timestamp +, COUNT(1) AS cnt +FROM midgard.switch_events + +-- check upgrades to make sure it has mint amount +-- check switch events to make sure number of rows is right + + +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +, MAX(block_timestamp) AS mxx +FROM flipside_dev_db.thorchain.upgrades + +SELECT COUNT(1) AS n +, MAX(block_timestamp) AS mx +, MAX(block_timestamp) AS mxx +FROM midgard.switch_events +WHERE block_timestamp <= 1656483014000000000 + + goes as profile.yml in ~/.dbt/ dbt deps to install dependencies dbt test dbt run --full-refresh -s models/thorchain +dbt run --full-refresh -s models/crosschain +dbt run --full-refresh -s models/thorchain/dbt/thorchain_dbt__fee_events_pk_count.sql dbt test -s models/thorchain dbt run --full-refresh -s models/crosschain dbt test -s models/crosschain +WITH first_sale AS ( + SELECT l.label + , MIN(s.block_timestamp) AS mn_date + FROM solana.core.fact_nft_sales s + JOIN solana.core.dim_labels l ON l.address = s.mint + HAVING mn_date >= CURRENT_DATE - 7 +) +SELECT INITCAP(l.label) AS collection +, SUM(s.sales_amount) AS volume +, AVG(s.sales_amount) AS avg_price +FROM solana.core.fact_nft_sales s +JOIN solana.core.dim_labels l ON l.address = s.mint +JOIN first_sale f ON f.label = l.label +WHERE s.sales_amount > 0 +ORDER BY 2 DESC + + +WITH first_sale AS ( + SELECT l.label + , MIN(s.block_timestamp) AS mn_date + FROM solana.core.fact_nft_sales s + JOIN solana.core.dim_labels l ON l.address = s.mint + GROUP BY 1 + HAVING mn_date::date = '2022-06-28' +) +SELECT INITCAP(l.label) AS collection +, SUM(s.sales_amount) AS volume +, AVG(s.sales_amount) AS avg_price +FROM solana.core.fact_nft_sales s +JOIN solana.core.dim_labels l ON l.address = s.mint +JOIN first_sale f ON f.label = l.label +WHERE s.sales_amount > 0 +GROUP BY 1 +ORDER BY 2 DESC + +WITH base AS ( + SELECT node_address + , slash_points + , reason + , block_timestamp + , COUNT(1) AS n + FROM midgard.slash_points + GROUP BY 1, 2, 3, 4 + HAVING COUNT(1) > 1 +) +SELECT COUNT(1) +FROM base + + + +with data_token_transfer as ( + select + substring(input_data, 202, 1), tx_hash + from + ethereum.core.fact_transactions + where + to_address = '0x3ee18b2214aff97000d974cf647e7c347e8fa585' + and + origin_function_signature = '0x0f5287b0' -- tokenTransfer + and + status = 'SUCCESS' + and + substring(input_data, 202, 1)::string = '1' -- solana +) +, data_token as ( + select + block_timestamp::date as date, + symbol, + sum(amount_usd) as total, + sum(amount) as total_tokens + from + ethereum.core.ez_token_transfers + where + tx_hash in (select tx_hash from data_token_transfer) + and + to_address = '0x3ee18b2214aff97000d974cf647e7c347e8fa585' + and + symbol is not null + and + amount_usd > 0 + group by 1, 2 +) +, eth_price as ( + select + hour, + price + from + ethereum.core.fact_hourly_token_prices + where + token_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' +) +, data_eth as ( + select + date_trunc('hour', block_timestamp) as hour, + sum(eth_value) as total_eth + from + ethereum.core.fact_transactions + where + substring(input_data, 74, 1)::string = '1' -- solana + and + to_address = '0x3ee18b2214aff97000d974cf647e7c347e8fa585' + and + status = 'SUCCESS' + and + origin_function_signature = '0x9981509f' -- wrapAndTransfer + group by 1 +) +, data_token_eth as ( + select hour::date as date, 'ETH' as symbol, sum(total) as total, sum(total_eth) as total_tokens from ( + select + a.hour, total_eth*price as total, total_eth + from data_eth a join eth_price b on a.hour = b.hour + ) dd group by 1 +) +, data_tokens as ( + select * from ( + select + date_trunc('month', date) as months, + symbol, + sum(total) as total_volumes, + sum(total_tokens) as total_volumes_tokens, + row_number() over (partition by months order by total_volumes desc) as nomor + from ( + select * from data_token + union all + select * from data_token_eth ) ee group by 1, 2 ) ff where nomor = 1 +) +select * from data_tokens + + + + + + +with data_token_transfer as ( + select + substring(input_data, 202, 1), tx_hash + from + ethereum.core.fact_transactions + where + to_address = '0x3ee18b2214aff97000d974cf647e7c347e8fa585' + and + origin_function_signature = '0x0f5287b0' -- tokenTransfer + and + status = 'SUCCESS' + and + substring(input_data, 202, 1)::string = '1' -- solana +) +, data_token as ( + select + block_timestamp::date as date, + symbol, + sum(amount_usd) as total, + sum(amount) as total_tokens + from + ethereum.core.ez_token_transfers + where + tx_hash in (select tx_hash from data_token_transfer) + and + to_address = '0x3ee18b2214aff97000d974cf647e7c347e8fa585' + and + symbol is not null + and + amount_usd > 0 + group by 1, 2 +) +, eth_price as ( + select + hour, + price + from + ethereum.core.fact_hourly_token_prices + where + token_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' +) +, data_eth as ( + select + date_trunc('hour', block_timestamp) as hour, + sum(eth_value) as total_eth + from + ethereum.core.fact_transactions + where + substring(input_data, 74, 1)::string = '1' -- solana + and + to_address = '0x3ee18b2214aff97000d974cf647e7c347e8fa585' + and + status = 'SUCCESS' + and + origin_function_signature = '0x9981509f' -- wrapAndTransfer + group by 1 +) +, data_token_eth as ( + select hour::date as date, 'ETH' as symbol, sum(total) as total, sum(total_eth) as total_tokens from ( + select + a.hour, total_eth*price as total, total_eth + from data_eth a join eth_price b on a.hour = b.hour + ) dd group by 1 +) +, data_tokens as ( + select * from ( + select + date_trunc('month', date) as months, + symbol, + sum(total) as total_volumes, + sum(total_tokens) as total_volumes_tokens, + row_number() over (partition by months order by total_volumes desc) as nomor + from ( + select * from data_token + union all + select * from data_token_eth ) ee group by 1, 2 ) ff where nomor = 1 +) +select * from data_tokens + + + +SELECT node_address +, slash_points +, reason +, block_timestamp +, COUNT(1) AS n +FROM midgard.slash_points +GROUP BY 1, 2, 3, 4 +HAVING n > 1 + +SELECT chain, COUNT(1) AS n +, MIN(block_timestamp) as mn +FROM BRONZE_MIDGARD_2_6_9_20220405.MIDGARD_ADD_EVENTS +GROUP BY 1 + + + +SELECT COUNT(1) AS N +, MAX(block_timestamp) as mx +, MAX(block_timestamp) as mx2 +FROM THORCHAIN.THORNAME_CHANGE_EVENTS + + +SELECT * +FROM solana.core.dim_nft_metadata +WHERE mint = '6XxjKYFbcndh2gDcsUrmZgVEsoDxXMnfsaGY6fpTJzNr' + + + +SELECT * +FROM solana.core.fact_nft_sales +WHERE tx_id IN ( + 'vsqqEKqud1LEC5zX7FuXwHmaqpza6tC6vtig8iWyuaJkfAUUFtpgvmdMVenrBJKyc7c9QdbpMs1WKDJNPtELfcV' + , '52MBuWHddd23EaokX4LgbQvor15HfDv3pBPvrTL8txWTewRdh6oZmZZyaVHCLbE4fvSTBJZWza7z4sp1kKMvJw5F' +) + +SELECT marketplace +, seller +, COUNT(1) AS n +FROM solana.core.fact_nft_sales +WHERE marketplace = 'solport' +GROUP BY 1, 2 +ORDER BY 3 DESC +LIMIT 100 + + +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 + +git remote set-url origin https://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@github.com/kblumberg/props-program.git + +9zQABKCHpQaHhbA + + +SELECT * +FROM solana.core.fact_nft_sales +WHERE seller = 'CCWxahZcw47EsH8pavCeNKeGbpGNEs5JEKQ17fjP4ehA' +ORDER BY block_timestamp DESC + + + +SELECT marketplace +, seller +, COUNT(1) AS n +FROM solana.core.fact_nft_sales +GROUP BY 1, 2 +ORDER BY 3 DESC +LIMIT 100 + +SELECT s.block_timestamp +, s.block_timestamp::date AS date +, s.marketplace +, s.tx_id +, s.mint +, s.sales_amount +, m.token_metadata:"Attribute Count"::string AS "Attribute Count" +, m.token_metadata:"Background"::string AS "Background" +, m.token_metadata:"Body"::string AS "Body" +, m.token_metadata:"Clothing"::string AS "Clothing" +, m.token_metadata:"Eyes"::string AS "Eyes" +, m.token_metadata:"Glasses"::string AS "Glasses" +, m.token_metadata:"Hair Hole"::string AS "Hair Hole" +, m.token_metadata:"Hat"::string AS "Hat" +, m.token_metadata:"Helmet"::string AS "Helmet" +, m.token_metadata:"Mouth"::string AS "Mouth" +, m.token_metadata:"Ski Mask"::string AS "Ski Mask" +FROM solana.core.fact_nft_sales s +JOIN solana.core.dim_nft_metadata m ON m.mint = s.mint +WHERE m.project_name ilike 'catalina whale%' +AND sales_amount > 0 + + + +SELECT s.block_timestamp +, s.block_timestamp::date AS date +, s.marketplace +, s.tx_id +, s.mint +, s.sales_amount +, m.token_metadata:"Accessories"::string AS "Accessories" +, m.token_metadata:"Attribute Count"::string AS "Attribute Count" +, m.token_metadata:"Background"::string AS "Background" +, m.token_metadata:"Big Drip"::string AS "Big Drip" +, m.token_metadata:"Body"::string AS "Body" +, m.token_metadata:"Coat"::string AS "Coat" +, m.token_metadata:"Eyes"::string AS "Eyes" +, m.token_metadata:"Head"::string AS "Head" +, m.token_metadata:"Mouth"::string AS "Mouth" +, m.token_metadata:"Shoe"::string AS "Shoe" +FROM solana.core.fact_nft_sales s +JOIN solana.core.dim_nft_metadata m ON m.mint = s.mint +WHERE m.project_name = 'Bubblegoose Ballers' +AND sales_amount > 0 +ORDER BY block_timestamp DESC + + +With labeled_table as +(Select distinct m.mint, m.purchaser,Solana.core.dim_labels.LABEL, + m.mint_price,m.BLOCK_TIMESTAMP,m.mint_currency +From Solana.core.fact_nft_mints m +INNER JOIN solana.core.dim_labels on m.MINT = solana.core.dim_labels.ADDRESS +where m.mint_currency = 'So11111111111111111111111111111111111111111' +and m.mint_price > 0.1 ), + +person as(Select * +from solana.core.fact_nft_mints +order by purchaser), + +firstsale as( +Select Distinct mint,sales_amount,block_timestamp +From (SELECT RANK() OVER (PARTITION BY mint order by block_timestamp) + as RN,mint,sales_amount,block_TIMESTAMP //as date + from solana.core.fact_nft_sales WHERE sales_amount > 0) as ST +Where ST.RN = 1 +), + + + +purchaser_table as(Select person.purchaser +,sum(cast(firstsale.sales_amount as decimal(12,3)) - cast(person.mint_price as decimal(12,3))) as profit_sol +from person +Inner join firstsale on person.mint = firstsale.mint +group by purchaser +order by profit_sol desc +limit 100), + +ayo as (Select moon.purchaser as purchaser, count(distinct moon.label) as collection_number +from labeled_table moon +group by purchaser +order by collection_number desc) + + + +select ayo.purchaser,ayo.collection_number, purchaser_table.profit_sol +from ayo +inner join purchaser_table on ayo.purchaser = purchaser_table.purchaser +order by profit_sol desc + + +SELECT * +FROM solana.core.fact_transfers +WHERE tx_to_address = '9zky78suDwh8t8mPbBCP4nr2Ew9pH2e1kLJMvMGDFT6L' +OR tx_from_address = '9zky78suDwh8t8mPbBCP4nr2Ew9pH2e1kLJMvMGDFT6L' + + +WITH base AS ( + SELECT instructions[0]:parsed:info:source::string AS source + , instructions[0]:parsed:info:destination::string AS destination + , instructions[0]:parsed:info:lamports AS lamports + FROM solana.core.fact_transactions + WHERE tx_id = '5R1cRC8KUgPPDnZz19bbNBJvw6Me1THPTEkDVYLYQRPN97ptrtKVnne1SgFRccH6Go4EKD9bdkobjgYbRkdgSkwZ' + AND instructions[0]:programId = '11111111111111111111111111111111' + AND instructions[0]:parsed:type = 'transfer' + AND ARRAY_SIZE(instructions) = 1 +) + + + + + +With person as(Select * +from solana.core.fact_nft_mints +where mint_price > 0.1 +order by purchaser), +--gets a specific person and all thier transactions from the mint table + + + +firstsale as( +Select mint,sales_amount,block_timestamp +From (SELECT RANK() OVER (PARTITION BY mint order by block_timestamp) + as RN,mint,sales_amount,block_TIMESTAMP //as date + from solana.core.fact_nft_sales) as ST +Where ST.RN = 1 +) +--need to find the first transaction for each mint address + + + +--this gets the person and than the first sales of the nft by mint and than finds it based on the mint adress of purchaser +Select person.purchaser +,sum(cast(firstsale.sales_amount as decimal(12,3)) - cast(person.mint_price as decimal(12,3))) as profit_sol +from person +Inner join firstsale on person.mint = firstsale.mint +group by purchaser +order by profit_sol desc + + + +SELECT ROUND(mint_price, 3) AS mp, COUNT(1) AS n +, COUNT(1) AS nn +FROM solana.dim_labels +JOIN solana.fact_nft_mints m ON LOWER(m.mint) = LOWER(l.address) +WHERE project_name = 'taiyo oil' +GROUP BY 1 + + + +WITH a AS ( + SELECT DISTINCT LOWER(mint) AS mint + FROM solana.fact_nft_sales s +), b AS ( + SELECT l.label + , SUM(s.mint_price) AS mint_volume + , SUM(s.mint_price) / COUNT(DISTINCT s.mint) AS mint_price + FROM solana.fact_nft_mints s + JOIN solana.dim_labels l ON LOWER(l.address) = LOWER(s.mint) + WHERE mint_currency = 'So11111111111111111111111111111111111111111' + AND block_timestamp >= '2022-03-01' + -- data issue, need to look into this + AND NOT l.label IN ( 'degen trash pandas','mindfolk owls','bored ape solana club' ) + GROUP BY 1 +), c AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY mint_volume DESC) AS mint_volume_rank + FROM b +), d AS ( + SELECT l.label + , SUM(s.sales_amount) AS sales_volume + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON LOWER(l.address) = LOWER(s.mint) + WHERE block_timestamp >= '2022-03-01' + GROUP BY 1 +), e AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY sales_volume DESC) AS sales_volume_rank + FROM d +), base AS ( + SELECT CASE WHEN c.label = 'taiyo oil' AND ROUND(m.mint_price, 1) <> 1.0 THEN 'Taiyo Infants' ELSE INITCAP(c.label) END AS collection + , c.mint_volume + , c.mint_volume_rank + , e.sales_volume + , e.sales_volume_rank + , c.mint_price + , COUNT(DISTINCT m.mint) AS n_mints + , COUNT(DISTINCT a.mint) AS n_sell + , 100 * (1 - (n_sell / n_mints)) AS pct_hold + FROM solana.fact_nft_mints m + JOIN solana.dim_labels l ON LOWER(l.address) = LOWER(m.mint) + JOIN c ON c.label = l.label + JOIN e ON e.label = l.label + LEFT JOIN a ON LOWER(a.mint) = LOWER(m.mint) + GROUP BY 1, 2, 3, 4, 5, 6 + ORDER BY sales_volume_rank + LIMIT 70 +), b2 AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY pct_hold DESC) AS rn + FROM base +) +SELECT * +, CASE WHEN rn < 10 THEN CONCAT('0', rn::varchar) ELSE rn::varchar END AS clean_rn +, CONCAT(clean_rn, '. ', collection) AS label +, CASE WHEN mint_price < 0.1 THEN 'Airdrop' ELSE 'Mint' END AS owner_type +FROM b2 +-- WHERE sales_volume_rank <= 70 +ORDER BY label + + +Updates / Accomplishments +Now have 98% of ME sales labeled! +Twitter thread showing off how awesome our Solana NFT data is +Have all the THORChain tables except for transfers de-duped! +I ordered a Saga Phone! +Problems Encountered +Nothing major. +Priorities +Automate Solana NFT data pipeline +Get those collections added to NFT deal score +Concerns + + + +SELECT INITCAP(l.label) AS collection +, SUM(s.sales_volume) AS volume +, MIN(s.block_timestamp::date) AS date +FROM solana.core.fact_nft_sales s +JOIN solana.core.dim_labels l ON l.addrss = s.mint +GROUP BY 1 +ORDER BY 2 DESC + + + +WITH a AS ( + SELECT DISTINCT LOWER(mint) AS mint + FROM solana.fact_nft_sales s +), b AS ( + SELECT l.label + , SUM(s.mint_price) AS mint_volume + , SUM(s.mint_price) / COUNT(DISTINCT s.mint) AS mint_price + FROM solana.fact_nft_mints s + JOIN solana.dim_labels l ON LOWER(l.address) = LOWER(s.mint) + WHERE mint_currency = 'So11111111111111111111111111111111111111111' + AND block_timestamp >= '2022-03-01' + -- data issue, need to look into this + AND NOT l.label IN ( 'degen trash pandas','mindfolk owls','bored ape solana club' ) + GROUP BY 1 +), c AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY mint_volume DESC) AS mint_volume_rank + FROM b +), d AS ( + SELECT l.label + , SUM(s.sales_amount) AS sales_volume + FROM solana.fact_nft_sales s + JOIN solana.dim_labels l ON LOWER(l.address) = LOWER(s.mint) + WHERE block_timestamp >= '2022-03-01' + GROUP BY 1 +), e AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY sales_volume DESC) AS sales_volume_rank + FROM d +), base AS ( + SELECT CASE WHEN c.label = 'taiyo oil' AND ROUND(m.mint_price, 1) <> 1.0 THEN 'Taiyo Infants' ELSE INITCAP(c.label) END AS collection + , m.mint + FROM solana.core.fact_nft_mints m + JOIN solana.core.dim_labels l ON l.address = m.mint +) +SELECT marketplace +, SUM(s.sales_amount) AS volume +FROM base b +JOIN solana.core.fact_nft_sales s ON s.mint = b.mint +WHERE b.collection = 'Taiyo Oil' +GROUP BY 1 + + + + +WITH base AS ( + SELECT INITCAP(l.label) AS collection + , SUM(COALESCE(s.mint_price, 0)) AS volume + , MIN(s.block_timestamp::date) AS date + , COUNT(DISTINCT l.address) AS n_tokens + , COUNT(DISTINCT s.block_timestamp) AS n_mints + , MAX(CASE WHEN s.mint IS NULL THEN l.address ELSE '' END) AS mint_example_1 + , MIN(CASE WHEN s.mint IS NULL THEN l.address ELSE '' END) AS mint_example_2 + FROM solana.core.dim_labels l + LEFT JOIN solana.core.fact_nft_mints s ON LOWER(l.address) = LOWER(s.mint) + WHERE label_type = 'nft' + AND l.address IS NOT NULL + GROUP BY 1 + HAVING n_tokens > 100 + ORDER BY 2 DESC +), b2 AS ( + SELECT INITCAP(l.label) AS collection + , SUM(COALESCE(s.sales_amount, 0)) AS sales_volume + FROM solana.core.fact_nft_sales s + JOIN solana.core.dim_labels l on LOWER(l.address) = LOWER(s.mint) + GROUP BY 1 + HAVING sales_volume > 1000 + AND MIN(block_timestamp) >= '2022-02-01' +) +SELECT b.* +, b2.sales_volume +, n_mints / n_tokens AS pct +FROM base b +JOIN b2 ON b2.collection = b.collection +ORDER BY pct, sales_volume DESC + + +SELECT * +FROM solana.core.dim_labels l +-- LEFT JOIN solana.core.fact_nft_mints s ON LOWER(l.address) = LOWER(s.mint) +WHERE label_type = 'nft' +AND l.label ilike 'okay bears' + +SELECT DISTINCT address +FROM solana.core.dim_labels l +-- LEFT JOIN solana.core.fact_nft_mints s ON LOWER(l.address) = LOWER(s.mint) +WHERE label_type = 'nft' +AND l.label ilike 'okay bears' + + +SELECT CASE + WHEN tx_id = '2cL9Qj5Xc3FteQimae7qXHzARX1vkZgYavLbutuyt8xUz6Ebep4yfM3r891BrMrm8MEDKggXzyS4kzLWSowWagDP' THEN 'Cyber Samurai Gen 2' + WHEN tx_id = '2MqHqoWUTRePzicvLiHJmcyPspFXH6nxzZJYRvSURjLjYXJoDKSYB8zgHSj5C2DbXws7fJCpc4HCg5LDh9z8QDxs' THEN 'Astrals' + WHEN tx_id = '4XAZbakhNWVcxPB8rrCHD5YQat8oNq3n8H92oLjDzeEcDQJp8GCZfWcNJ2pGN5MySATVvoK2PTeHqc7SZYJsopHN' THEN 'Blocksmith Labs' + WHEN tx_id = '55e6y11uYPso53U2sVct5TCmGZhD7GD3YsueMwspL9TgzndctH2ZdCh5ytZyzcm3XUx172VhiFWa82Lo7hggmQYQ' THEN 'Crypto Cowboy Country' + WHEN tx_id = '668nNaC5u9zmyqYoyFQJNriaFdD79Tt65XJArwpadPGvWwDNZDZMPGBFLjkY3KTrtp2YBeT9K1b3c4bY6y41T2eC' THEN 'Ghostface Gen 2' + WHEN tx_id = 'GxXD7S1Qyp2fzBjJ4tKAvrbSKn4bnV9hfcWvLD3xQkFDa7hQ22ov15ePJGScNQz8pd5wAmG23tfPnjBz71f4ieA' THEN 'Tombstoned High Society' + WHEN tx_id = 'MxnMeosgLe2PvzCc387gK5tLwzBcBjjb8ENvyfuoo6eaQSziuJEhqJCmLNWjcEDdMLDnBwAdeyRyszNTJfqU527' THEN 'Shin Sengoku' +ELSE 'Other' END AS collection +, * +FROM solana.core.transactions +WHERE block_timestamp >= '2022-02-01' +AND tx_id IN ( + '2cL9Qj5Xc3FteQimae7qXHzARX1vkZgYavLbutuyt8xUz6Ebep4yfM3r891BrMrm8MEDKggXzyS4kzLWSowWagDP', + , '2MqHqoWUTRePzicvLiHJmcyPspFXH6nxzZJYRvSURjLjYXJoDKSYB8zgHSj5C2DbXws7fJCpc4HCg5LDh9z8QDxs', + , '4XAZbakhNWVcxPB8rrCHD5YQat8oNq3n8H92oLjDzeEcDQJp8GCZfWcNJ2pGN5MySATVvoK2PTeHqc7SZYJsopHN', + , '55e6y11uYPso53U2sVct5TCmGZhD7GD3YsueMwspL9TgzndctH2ZdCh5ytZyzcm3XUx172VhiFWa82Lo7hggmQYQ', + , '668nNaC5u9zmyqYoyFQJNriaFdD79Tt65XJArwpadPGvWwDNZDZMPGBFLjkY3KTrtp2YBeT9K1b3c4bY6y41T2eC', + , 'GxXD7S1Qyp2fzBjJ4tKAvrbSKn4bnV9hfcWvLD3xQkFDa7hQ22ov15ePJGScNQz8pd5wAmG23tfPnjBz71f4ieA', + , 'MxnMeosgLe2PvzCc387gK5tLwzBcBjjb8ENvyfuoo6eaQSziuJEhqJCmLNWjcEDdMLDnBwAdeyRyszNTJfqU527' +) + + + +-- 8,456 +WITH base AS ( + SELECT pool_name + , from_address + , SUM(CASE WHEN lp_action = 'add_liquidity' THEN stake_units ELSE -stake_units END) AS net_stake_units + FROM thorchain.liquidity_actions + GROUP BY 1, 2 + HAVING net_stake_units > 0 +) +SELECT COUNT(DISTINCT from_address) AS n +FROM base + +WITH lp1 AS ( + SELECT pool_name + , from_address + , block_timestamp::date AS date + , SUM(CASE WHEN lp_action = 'add_liquidity' THEN stake_units ELSE -stake_units END) AS net_stake_units + FROM thorchain.liquidity_actions + GROUP BY 1, 2, 3 +), lp2 AS ( + SELECT * + , SUM(net_stake_units) OVER (PARTITION BY pool_name, from_address ORDER BY date) AS cumu_stake_units + FROM lp1 +), lp3 AS ( + SELECT * + , CASE WHEN net_stake_units > 0 AND cumu_stake_units > 0 AND cumu_stake_units - net_stake_units <= 0 THEN 1 + WHEN net_stake_units < 0 AND cumu_stake_units <= 0 AND cumu_stake_units - net_stake_units > 0 THEN -1 + ELSE 0 END AS net_user_stake_change + FROM lp2 +), lp4 AS ( + SELECT pool_name + , from_address + , SUM(net_user_stake_change) AS net_user_stake_change + FROM lp3 + GROUP BY 1, 2 +) +SELECT COUNT(DISTINCT from_address) AS n +, COUNT(1) AS n_lp +FROM lp4 +WHERE net_user_stake_change > 0 + + +WITH lp1 AS ( + SELECT pool_name + , from_address + , block_timestamp::date AS date + , SUM(CASE WHEN lp_action = 'add_liquidity' THEN stake_units ELSE -stake_units END) AS net_stake_units + FROM thorchain.liquidity_actions + GROUP BY 1, 2, 3 +), lp2 AS ( + SELECT * + , SUM(net_stake_units) OVER (PARTITION BY pool_name, from_address ORDER BY date) AS cumu_stake_units + FROM lp1 +), lp3 AS ( + SELECT * + , CASE WHEN net_stake_units > 0 AND cumu_stake_units > 0 AND cumu_stake_units - net_stake_units <= 0 THEN 1 + WHEN net_stake_units < 0 AND cumu_stake_units <= 0 AND cumu_stake_units - net_stake_units > 0 THEN -1 + ELSE 0 END AS net_user_stake_change + FROM lp2 +), lp4 AS ( + SELECT from_address + , date + , SUM(net_user_stake_change) AS daily_net_user_stake_change + FROM lp3 + GROUP BY 1, 2 + HAVING daily_net_user_stake_change <> 0 +), lp5 AS ( + SELECT * + , SUM(daily_net_user_stake_change) OVER (PARTITION BY from_address ORDER BY date) AS cumu_net_user_stake_change + FROM lp4 +) lp6 AS ( + SELECT date + , from_address + , CASE WHEN daily_net_user_stake_change > 0 AND cumu_net_user_stake_change > 0 AND cumu_net_user_stake_change - daily_net_user_stake_change <= 0 THEN 1 + WHEN daily_net_user_stake_change < 0 AND cumu_net_user_stake_change <= 0 AND cumu_net_user_stake_change - daily_net_user_stake_change > 0 THEN -1 + ELSE 0 END AS net_user_stake_change + FROM lp5 +), lp7 AS ( + SELECT date + , SUM(net_user_stake_change) AS daily_n_lp_change + FROM lp6 + GROUP BY 1 +) +SELECT * +, SUM(daily_n_lp_change) OVER (ORDER BY date) AS cumu_n_lp +FROM lp7 + +WITH r1 AS ( + SELECT from_address AS address, + , block_timestamp::date AS date + , SUM(CASE WHEN lp_action = 'add_liquidity' THEN stake_units ELSE -stake_units END) AS net_stake_units + FROM thorchain.liquidity_actions + GROUP BY 1, 2, 3 +), lp2 AS ( + SELECT * + , SUM(net_stake_units) OVER (PARTITION BY pool_name, from_address ORDER BY date) AS cumu_stake_units + FROM lp1 +), lp3 AS ( + SELECT * + , CASE WHEN net_stake_units > 0 AND cumu_stake_units > 0 AND cumu_stake_units - net_stake_units <= 0 THEN 1 + WHEN net_stake_units < 0 AND cumu_stake_units <= 0 AND cumu_stake_units - net_stake_units > 0 THEN -1 + ELSE 0 END AS net_user_stake_change + FROM lp2 +), lp4 AS ( + SELECT from_address + , date + , SUM(net_user_stake_change) AS daily_net_user_stake_change + FROM lp3 + GROUP BY 1, 2 + HAVING daily_net_user_stake_change <> 0 +), lp5 AS ( + SELECT * + , SUM(daily_net_user_stake_change) OVER (PARTITION BY from_address ORDER BY date) AS cumu_net_user_stake_change + FROM lp4 +) lp6 AS ( + SELECT date + , from_address + , CASE WHEN daily_net_user_stake_change > 0 AND cumu_net_user_stake_change > 0 AND cumu_net_user_stake_change - daily_net_user_stake_change <= 0 THEN 1 + WHEN daily_net_user_stake_change < 0 AND cumu_net_user_stake_change <= 0 AND cumu_net_user_stake_change - daily_net_user_stake_change > 0 THEN -1 + ELSE 0 END AS net_user_stake_change + FROM lp5 +), lp7 AS ( + SELECT date + , SUM(net_user_stake_change) AS daily_n_lp_change + FROM lp6 + GROUP BY 1 +) +SELECT * +, SUM(daily_n_lp_change) OVER (ORDER BY date) AS cumu_n_lp +FROM lp7 + + +SELECT * +FROM crosschain.labels +WHERE blockchain ilike 'thorchain' + + +SELECT date_trunc('hour', block_timestamp) AS hour +, COUNT(1) AS n +FROM thorchain.swaps +WHERE block_timestamp >= '2022-05-07' +AND block_timestamp >= '2022-05-13' +AND pool_name like 'TERRA%' +GROUP BY 1 + + + + +with data_sol as ( + select + instruction:accounts[0]::string as address + , MIN(block_timestamp) AS wormhole_time + from + solana.core.fact_events + where + block_timestamp::date >= CURRENT_DATE - 30 + and + program_id = 'wormDTUJ6AWPNvk59vGQbDvGJmqbDTdgWgAqcLBCgUb' + and + instruction:data = '4' -- retrieve from other chains + and + ( + inner_instruction:instructions[1]:parsed:type = 'mintTo' -- have careated associated token account + or + inner_instruction:instructions[3]:parsed:type = 'mintTo' -- not yet created associated token account + ) + and + succeeded = true + group by 1 +) +, data_user_programs as ( +select + b.address, + case when label is null then COALESCE(program_id, 'None') else label end as labeling, + ROW_NUMBER() OVER (PARTITION BY b.address ORDER BY block_timestamp) AS rn +from + data_sol b + LEFT JOIN + solana.core.fact_events a on a.block_timestamp > b.wormhole_time AND + b.address IN ( + instruction:accounts[0] + , instruction:accounts[1] + , instruction:accounts[2] + , instruction:accounts[3] + , instruction:accounts[4] + , instruction:accounts[5] + , instruction:accounts[6] + , instruction:accounts[7] + , instruction:accounts[8] + , instruction:accounts[9] + ) +left join solana.core.dim_labels c on a.program_id = c.address +where + a.block_timestamp::date >= CURRENT_DATE - 30 + and + succeeded = true + and + program_id not in ( + 'wormDTUJ6AWPNvk59vGQbDvGJmqbDTdgWgAqcLBCgUb', -- exclude wormhole + 'worm2ZoG2kUd4vFXhvjh93UUH596ayRfgQ2MgjNMTth', -- exclude wormhole + 'DeJBGdMFa1uynnnKiwrVioatTuHmNLpyFKnmB5kaFdzQ', -- Phantom wallet program id for trasnfer https://docs.phantom.app/resources/faq + '4MNPdKu9wFMvEeZBMt3Eipfs5ovVWTJb31pEXDJAAxX5' -- transfer token program + ) -- exclude wormhole program id + -- and + -- block_timestamp::date >= CURRENT_DATE - 90 + and labeling != 'solana' +-- group by 1, 2 +) +select + CASE + when labeling = 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K' then 'Magic Eden V2' + when labeling = '3Katmm9dhvLQijAvomteYMo6rfVbY5NaCRNq9ZBqBgr6' then 'Francium Lend Reward Program' + when labeling = 'QMNeHCGYnLVDn1icRAfQZpjPLBNkfGbSKRB83G5d8KB' then 'Quarry Protocol' + when labeling = 'VoLT1mJz1sbnxwq5Fv2SXjdVDgPXrb9tJyC8WpMDkSp' then 'Friktion Protocol' + when labeling = 'hausS13jsjafwWwGqZTUQRmWyvyxn9EQpqMwV1PBBmk' then 'Opensea' + when labeling = '2nAAsYdXF3eTQzaeUQS3fr4o782dDg8L28mX39Wr5j8N' then 'lyfRaydiumProgramID' + else labeling end as label, + count(*) as total + from data_user_programs + where label != 'solana' -- remove program id that dedicated to solana + and rn = 1 + + group by 1 order by 2 + desc limit 10 + + +WITH lp1 AS ( + SELECT pool_name + , from_address + , block_timestamp::date AS date + , SUM(CASE WHEN lp_action = 'add_liquidity' THEN stake_units ELSE -stake_units END) AS net_stake_units + FROM thorchain.liquidity_actions + WHERE pool_name ilike 'terra%' + GROUP BY 1, 2, 3 +), lp2 AS ( + SELECT * + , SUM(net_stake_units) OVER (PARTITION BY pool_name, from_address ORDER BY date) AS cumu_stake_units + FROM lp1 +), lp3 AS ( + SELECT * + , CASE WHEN net_stake_units > 0 AND cumu_stake_units > 0 AND cumu_stake_units - net_stake_units <= 0 THEN 1 + WHEN net_stake_units < 0 AND cumu_stake_units <= 0 AND cumu_stake_units - net_stake_units > 0 THEN -1 + ELSE 0 END AS net_user_stake_change + FROM lp2 +), lp4 AS ( + SELECT from_address + , date + , SUM(net_user_stake_change) AS daily_net_user_stake_change + FROM lp3 + GROUP BY 1, 2 + HAVING daily_net_user_stake_change <> 0 +), lp5 AS ( + SELECT * + , SUM(daily_net_user_stake_change) OVER (PARTITION BY from_address ORDER BY date) AS cumu_net_user_stake_change + FROM lp4 +), lp6 AS ( + SELECT date + , from_address + , CASE WHEN daily_net_user_stake_change > 0 AND cumu_net_user_stake_change > 0 AND cumu_net_user_stake_change - daily_net_user_stake_change <= 0 THEN 1 + WHEN daily_net_user_stake_change < 0 AND cumu_net_user_stake_change <= 0 AND cumu_net_user_stake_change - daily_net_user_stake_change > 0 THEN -1 + ELSE 0 END AS net_user_stake_change + FROM lp5 +), lp7 AS ( + SELECT date + , SUM(net_user_stake_change) AS daily_n_lp_change + FROM lp6 + GROUP BY 1 +) +SELECT * +, SUM(daily_n_lp_change) OVER (ORDER BY date) AS cumu_n_lp +FROM lp7 + + + + +SELECT pool_name, (block_timestamp::date) AS date, COUNT(1) AS n +FROM thorchain.swaps +WHERE pool_name LIKE 'TERRA%' +GROUP BY 1, 2 +ORDER BY 2, 1 + +WITH terra_lplers AS ( + SELECT from_address AS address, sum(rune_amount) AS removed_rune_amount, min(block_timestamp) AS first_removal_time + FROM flipside_prod_db.thorchain.liquidity_actions + WHERE (pool_name = 'TERRA.LUNA' OR pool_name = 'TERRA.UST') AND + lp_action = 'remove_liquidity' AND + block_timestamp > TO_DATE('2022-05-15') + GROUP BY address + HAVING sum(rune_amount) > 0 +), lp_addtions AS ( + SELECT from_address, sum(rune_amount) AS added_rune_amount + FROM flipside_prod_db.thorchain.liquidity_actions + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE lp_action = 'add_liquidity' + GROUP BY from_address +), +swaps AS ( + SELECT from_address, sum(from_amount) AS swapped_rune_amount + FROM flipside_prod_db.thorchain.swaps + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE from_asset = 'THOR.RUNE' + GROUP BY from_address +), +swappers_and_lplers AS ( + SELECT DISTINCT address + FROM terra_lplers + LEFT JOIN lp_addtions ON lp_addtions.from_address = address + LEFT JOIN swaps ON swaps.from_address = address + WHERE added_rune_amount > 0 OR swapped_rune_amount > 0 +), +tranfers AS ( + SELECT t.from_address, SUM(rune_amount) AS transferred_rune_amount + FROM flipside_prod_db.thorchain.transfers t + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + LEFT JOIN crosschain.address_labels l ON l.address = t.to_address AND blockchain = 'thorchain' + WHERE asset = 'THOR.RUNE' AND COALESCE(l.label_type, '') NOT IN ('defi','dex') + GROUP BY 1 +), +base AS ( + SELECT address + , removed_rune_amount + , COALESCE(la.added_rune_amount, 0) AS added_rune_amount + , COALESCE(s.swapped_rune_amount, 0) AS swapped_rune_amount + , COALESCE(t.transferred_rune_amount, 0) AS transferred_rune_amount + FROM terra_lplers tl + LEFT JOIN lp_addtions la ON la.from_address = tl.address + LEFT JOIN swaps s ON s.from_address = tl.address + LEFT JOIN tranfers t ON t.from_address = tl.address +), b2 AS ( + SELECT address + , removed_rune_amount + , GREATEST(0, removed_rune_amount - added_rune_amount - swapped_rune_amount - transferred_rune_amount) AS kept_amount_2 + , LEAST(b.added_rune_amount, b.removed_rune_amount) AS added_rune_amount_2 + , LEAST(b.removed_rune_amount - added_rune_amount_2, b.swapped_rune_amount) AS swapped_rune_amount_2 + , LEAST(b.removed_rune_amount - added_rune_amount_2 - swapped_rune_amount_2, b.transferred_rune_amount) AS transferred_rune_amount_2 + , CASE + WHEN kept_amount_2 >= GREATEST(kept_amount_2, added_rune_amount_2, swapped_rune_amount_2, transferred_rune_amount_2) THEN 'Keeping it in their wallet' + WHEN added_rune_amount_2 >= GREATEST(added_rune_amount_2, swapped_rune_amount_2, transferred_rune_amount_2) THEN 'Added to other pools' + WHEN swapped_rune_amount_2 >= GREATEST(swapped_rune_amount_2, transferred_rune_amount_2) THEN 'Swapped for other assets' + ELSE 'Transferred to other addresses' END AS action + FROM base b +) +SELECT 'Keeping it in their wallet' AS action, SUM(kept_amount_2) AS rune_amount FROM b2 GROUP BY 1 +UNION +SELECT 'Added to other pools' AS action, SUM(added_rune_amount_2) AS rune_amount FROM b2 GROUP BY 1 +UNION +SELECT 'Swapped for other assets' AS action, SUM(swapped_rune_amount_2) AS rune_amount FROM b2 GROUP BY 1 +UNION +SELECT 'Transferred to other addresses' AS action, SUM(transferred_rune_amount_2) AS rune_amount FROM b2 GROUP BY 1 + +, COUNT(DISTINCT address) AS n2 +FROM b2 +GROUP BY 1 +ORDER BY 2 DESC + + + + +WITH terra_lplers AS ( + SELECT from_address AS address, sum(rune_amount) AS removed_rune_amount, min(block_timestamp) AS first_removal_time + FROM flipside_prod_db.thorchain.liquidity_actions + WHERE (pool_name = 'TERRA.LUNA' OR pool_name = 'TERRA.UST') AND + lp_action = 'remove_liquidity' AND + block_timestamp > TO_DATE('2022-05-15') + GROUP BY address + HAVING sum(rune_amount) > 0 +), +lp_addtions AS ( + SELECT from_address, sum(rune_amount) AS added_rune_amount + FROM flipside_prod_db.thorchain.liquidity_actions + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE lp_action = 'add_liquidity' + GROUP BY from_address +), +swaps AS ( + SELECT from_address, sum(from_amount) AS swapped_rune_amount + FROM flipside_prod_db.thorchain.swaps + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE from_asset = 'THOR.RUNE' + GROUP BY from_address +), +swappers_and_lplers AS ( + SELECT DISTINCT address + FROM terra_lplers + LEFT JOIN lp_addtions ON lp_addtions.from_address = address + LEFT JOIN swaps ON swaps.from_address = address + WHERE added_rune_amount > 0 OR swapped_rune_amount > 0 +), +tranfers AS ( + SELECT transfers.*, rune_amount AS transferred_rune_amount + FROM flipside_prod_db.thorchain.transfers + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE asset = 'THOR.RUNE' AND from_address NOT IN (SELECT address FROM swappers_and_lplers) AND to_address != 'thor1dheycdevq39qlkxs2a6wuuzyn4aqxhve4qxtxt' --Reserve +), +movements AS ( + SELECT 'Swapped for other assets' AS action, sum(LEAST(removed_rune_amount, swapped_rune_amount)) AS rune_amount + FROM terra_lplers + JOIN swaps ON address = from_address + UNION + SELECT 'Added to other pools' AS action, sum(LEAST(removed_rune_amount, added_rune_amount)) AS rune_amount + FROM terra_lplers + JOIN lp_addtions ON address = from_address + UNION + SELECT 'Transferred to other addresses' AS action, sum(LEAST(removed_rune_amount, transferred_rune_amount)) AS rune_amount + FROM terra_lplers + JOIN tranfers ON address = from_address +), +movements1 AS ( + SELECT * FROM movements + UNION + SELECT 'Keeping it in their wallet' AS action, + ((SELECT sum(removed_rune_amount) AS rune_amount FROM terra_lplers) - (SELECT sum(rune_amount) FROM movements)) AS rune_amount +) + +SELECT * FROM movements1 + + + +WITH pool_start_date AS ( + SELECT SPLIT(pool_name, '-')[0] AS pool_name, MIN(block_timestamp::date) AS pool_start_date + FROM thorchain.swaps + GROUP BY 1 + ORDER BY 2 +), terra_lplers AS ( + SELECT from_address AS address, sum(rune_amount) AS removed_rune_amount, min(block_timestamp) AS first_removal_time + FROM flipside_prod_db.thorchain.liquidity_actions + WHERE (pool_name = 'TERRA.LUNA' OR pool_name = 'TERRA.UST') AND + lp_action = 'remove_liquidity' AND + block_timestamp > TO_DATE('2022-05-15') + GROUP BY address + HAVING sum(rune_amount) > 0 +), +lp_addtions AS ( + SELECT from_address, pool_name, sum(rune_amount) AS added_rune_amount + FROM flipside_prod_db.thorchain.liquidity_actions + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE lp_action = 'add_liquidity' + GROUP BY from_address, pool_name +), +swaps AS ( + SELECT from_address, pool_name, sum(from_amount) AS swapped_rune_amount + FROM flipside_prod_db.thorchain.swaps + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE from_asset = 'THOR.RUNE' + GROUP BY from_address, pool_name +), +swappers_and_lplers AS ( + SELECT DISTINCT address + FROM terra_lplers + LEFT JOIN lp_addtions ON lp_addtions.from_address = address + LEFT JOIN swaps ON swaps.from_address = address + WHERE added_rune_amount > 0 OR swapped_rune_amount > 0 +), +tranfers AS ( + SELECT transfers.*, rune_amount AS transferred_rune_amount + FROM flipside_prod_db.thorchain.transfers + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE asset = 'THOR.RUNE' AND from_address NOT IN (SELECT address FROM swappers_and_lplers) AND to_address != 'thor1dheycdevq39qlkxs2a6wuuzyn4aqxhve4qxtxt' --Reserve +) +, base AS ( +SELECT l.address, l.removed_rune_amount, SPLIT(a.pool_name, '-')[0] AS pool_name, a.block_timestamp, a.rune_amount +, SUM(a.rune_amount) OVER (PARTITION BY address ORDER BY a.block_timestamp) AS cumu_rune_amount +FROM terra_lplers l +JOIN flipside_prod_db.thorchain.liquidity_actions a + ON a.from_address = l.address + AND a.block_timestamp > first_removal_time + AND a.lp_action = 'add_liquidity' +), b2 AS ( + SELECT * + , CASE WHEN cumu_rune_amount < removed_rune_amount THEN rune_amount + WHEN cumu_rune_amount - rune_amount >= removed_rune_amount THEN 0 + ELSE removed_rune_amount - (cumu_rune_amount - rune_amount) END AS lp_rune_amount + FROM base +), b3 AS ( + SELECT pool_name + , SUM(lp_rune_amount) AS rune_amount + FROM b2 + GROUP BY 1 +), b4 AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY rune_amount DESC) AS rn + FROM b3 +) +SELECT b4.* +, CASE WHEN rn < 10 THEN CONCAT('0', rn::varchar, '. ', b4.pool_name) ELSE CONCAT(rn::varchar, '. ', b4.pool_name) END AS label +, p.pool_start_date +FROM b4 +JOIN pool_start_date p ON p.pool_name = b4.pool_name +ORDER BY label + + + +WITH pool_start_date AS ( + SELECT SPLIT(pool_name, '-')[0] AS pool_name, MIN(block_timestamp::date) AS pool_start_date + FROM thorchain.swaps + GROUP BY 1 + ORDER BY 2 +), terra_lplers AS ( + SELECT from_address AS address, sum(rune_amount) AS removed_rune_amount, min(block_timestamp) AS first_removal_time + FROM flipside_prod_db.thorchain.liquidity_actions + WHERE (pool_name = 'TERRA.LUNA' OR pool_name = 'TERRA.UST') AND + lp_action = 'remove_liquidity' AND + block_timestamp > TO_DATE('2022-05-15') + GROUP BY address + HAVING sum(rune_amount) > 0 +), +lp_addtions AS ( + SELECT from_address, pool_name, sum(rune_amount) AS added_rune_amount + FROM flipside_prod_db.thorchain.liquidity_actions + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE lp_action = 'add_liquidity' + GROUP BY from_address, pool_name +), +swaps AS ( + SELECT from_address, pool_name, sum(from_amount) AS swapped_rune_amount + FROM flipside_prod_db.thorchain.swaps + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE from_asset = 'THOR.RUNE' + GROUP BY from_address, pool_name +), +swappers_and_lplers AS ( + SELECT DISTINCT address + FROM terra_lplers + LEFT JOIN lp_addtions ON lp_addtions.from_address = address + LEFT JOIN swaps ON swaps.from_address = address + WHERE added_rune_amount > 0 OR swapped_rune_amount > 0 +), +tranfers AS ( + SELECT transfers.*, rune_amount AS transferred_rune_amount + FROM flipside_prod_db.thorchain.transfers + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE asset = 'THOR.RUNE' AND from_address NOT IN (SELECT address FROM swappers_and_lplers) AND to_address != 'thor1dheycdevq39qlkxs2a6wuuzyn4aqxhve4qxtxt' --Reserve +) +, base AS ( +SELECT sum(LEAST(removed_rune_amount, added_rune_amount)) AS rune_amount, SPLIT(pool_name, '-')[0] AS pool_name +FROM terra_lplers +JOIN lp_addtions ON address = from_address +GROUP BY pool_name +), b2 AS ( + SELECT * + , ROW_NUMBER() OVER (ORDER BY rune_amount DESC) AS rn + FROM base +) +SELECT b2.* +, CASE WHEN rn < 10 THEN CONCAT('0', rn::varchar, '. ', b2.pool_name) ELSE CONCAT(rn::varchar, '. ', b2.pool_name) END AS label +, p.pool_start_date +FROM b2 +JOIN pool_start_date p ON p.pool_name = b2.pool_name +ORDER BY label + + + +WITH terra_lplers AS ( + SELECT from_address AS address, sum(rune_amount) AS removed_rune_amount, min(block_timestamp) AS first_removal_time + FROM flipside_prod_db.thorchain.liquidity_actions + WHERE (pool_name = 'TERRA.LUNA' OR pool_name = 'TERRA.UST') AND + lp_action = 'remove_liquidity' AND + block_timestamp > TO_DATE('2022-05-15') + GROUP BY address + HAVING sum(rune_amount) > 0 +), +lp_addtions AS ( + SELECT from_address, sum(rune_amount) AS added_rune_amount + FROM flipside_prod_db.thorchain.liquidity_actions + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE lp_action = 'add_liquidity' + GROUP BY from_address +), +swaps AS ( + SELECT from_address, sum(from_amount) AS swapped_rune_amount + FROM flipside_prod_db.thorchain.swaps + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE from_asset = 'THOR.RUNE' + GROUP BY from_address +), +swappers_and_lplers AS ( + SELECT DISTINCT address + FROM terra_lplers + LEFT JOIN lp_addtions ON lp_addtions.from_address = address + LEFT JOIN swaps ON swaps.from_address = address + WHERE added_rune_amount > 0 OR swapped_rune_amount > 0 +), +tranfers AS ( + SELECT transfers.*, rune_amount AS transferred_rune_amount + FROM flipside_prod_db.thorchain.transfers + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE asset = 'THOR.RUNE' AND from_address NOT IN (SELECT address FROM swappers_and_lplers) AND to_address != 'thor1dheycdevq39qlkxs2a6wuuzyn4aqxhve4qxtxt' --Reserve +), +movements AS ( + SELECT 'Swapped for other assets' AS action, sum(LEAST(removed_rune_amount, swapped_rune_amount)) AS rune_amount + FROM terra_lplers + JOIN swaps ON address = from_address + UNION + SELECT 'Added to other pools' AS action, sum(LEAST(removed_rune_amount, added_rune_amount)) AS rune_amount + FROM terra_lplers + JOIN lp_addtions ON address = from_address + UNION + SELECT 'Transferred to other addresses' AS action, sum(LEAST(removed_rune_amount, transferred_rune_amount)) AS rune_amount + FROM terra_lplers + JOIN tranfers ON address = from_address +), +movements1 AS ( + SELECT * FROM movements + UNION + SELECT 'Keeping it in their wallet' AS action, + ((SELECT sum(removed_rune_amount) AS rune_amount FROM terra_lplers) - (SELECT sum(rune_amount) FROM movements)) AS rune_amount +) + +SELECT * FROM movements1 + + + +WITH terra_lplers AS ( + SELECT from_address AS address, sum(rune_amount) AS removed_rune_amount, min(block_timestamp) AS first_removal_time + FROM flipside_prod_db.thorchain.liquidity_actions + WHERE (pool_name = 'TERRA.LUNA' OR pool_name = 'TERRA.UST') AND + lp_action = 'remove_liquidity' AND + block_timestamp > TO_DATE('2022-05-15') + GROUP BY address + HAVING sum(rune_amount) > 0 +), +lp_addtions AS ( + SELECT from_address, pool_name, sum(rune_amount) AS added_rune_amount + FROM flipside_prod_db.thorchain.liquidity_actions + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE lp_action = 'add_liquidity' + GROUP BY from_address, pool_name +), +swaps AS ( + SELECT from_address, pool_name, sum(from_amount) AS swapped_rune_amount + FROM flipside_prod_db.thorchain.swaps + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE from_asset = 'THOR.RUNE' + GROUP BY from_address, pool_name +), +swappers_and_lplers AS ( + SELECT DISTINCT address + FROM terra_lplers + LEFT JOIN lp_addtions ON lp_addtions.from_address = address + LEFT JOIN swaps ON swaps.from_address = address + WHERE added_rune_amount > 0 OR swapped_rune_amount > 0 +), +tranfers AS ( + SELECT transfers.*, rune_amount AS transferred_rune_amount + FROM flipside_prod_db.thorchain.transfers + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE asset = 'THOR.RUNE' AND from_address NOT IN (SELECT address FROM swappers_and_lplers) AND to_address != 'thor1dheycdevq39qlkxs2a6wuuzyn4aqxhve4qxtxt' --Reserve +) +, base AS ( +SELECT sum(LEAST(removed_rune_amount, added_rune_amount)) AS rune_amount, SPLIT(pool_name, '-')[0] AS pool_name +FROM terra_lplers +JOIN lp_addtions ON address = from_address +GROUP BY pool_name +), b2 AS ( + SELECT * + , ROW_NUMBER() OVER ORDER BY rune_amount DESC AS rn + FROM base +) +SELECT * +, CASE WHEN rn < 10 THEN CONCAT('0', rn::varchar, '. ', pool_name) ELSE CONCAT(rn::varchar, '. ', pool_name) END AS label +FROM b2 +ORDER BY label + + +SELECT * +, ROW_NUMBER() +FROM movements1 + + +WITH terra_lplers AS ( + SELECT from_address AS address, sum(rune_amount) AS removed_rune_amount, min(block_timestamp) AS first_removal_time + FROM flipside_prod_db.thorchain.liquidity_actions + WHERE (pool_name = 'TERRA.LUNA' OR pool_name = 'TERRA.UST') AND + lp_action = 'remove_liquidity' AND + block_timestamp > TO_DATE('2022-05-15') + GROUP BY address + HAVING sum(rune_amount) > 0 +), +lp_addtions AS ( + SELECT from_address, pool_name, sum(rune_amount) AS added_rune_amount + FROM flipside_prod_db.thorchain.liquidity_actions + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE lp_action = 'add_liquidity' + GROUP BY from_address, pool_name +), +swaps AS ( + SELECT from_address, pool_name, sum(from_amount) AS swapped_rune_amount + FROM flipside_prod_db.thorchain.swaps + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE from_asset = 'THOR.RUNE' + GROUP BY from_address, pool_name +), +swappers_and_lplers AS ( + SELECT DISTINCT address + FROM terra_lplers + LEFT JOIN lp_addtions ON lp_addtions.from_address = address + LEFT JOIN swaps ON swaps.from_address = address + WHERE added_rune_amount > 0 OR swapped_rune_amount > 0 +), +tranfers AS ( + SELECT transfers.*, rune_amount AS transferred_rune_amount + FROM flipside_prod_db.thorchain.transfers + JOIN terra_lplers ON address = from_address AND block_timestamp > first_removal_time + WHERE asset = 'THOR.RUNE' AND from_address NOT IN (SELECT address FROM swappers_and_lplers) AND to_address != 'thor1dheycdevq39qlkxs2a6wuuzyn4aqxhve4qxtxt' --Reserve +) + +SELECT sum(LEAST(removed_rune_amount, added_rune_amount)) AS rune_amount, pool_name +FROM terra_lplers +JOIN lp_addtions ON address = from_address +GROUP BY pool_name + + + + + + + +SELECT * +FROM solana.core.fact_events e +JOIN solana.core.dim_labels l on e.program_id = l.address +WHERE e.block_timestamp >= CURRENT_DATE - 2 +AND succeeded +AND l.label ilike 'jup%' +LIMIT 1000 + + + +SELECT * FROM movements1 + + + +WITH lp1 AS ( + SELECT pool_name + , from_address + , block_timestamp::date AS date + , SUM(CASE WHEN lp_action = 'add_liquidity' THEN stake_units ELSE -stake_units END) AS net_stake_units + FROM thorchain.liquidity_actions + GROUP BY 1, 2, 3 +), lp2 AS ( + SELECT * + , SUM(net_stake_units) OVER (PARTITION BY pool_name, from_address ORDER BY date) AS cumu_stake_units + FROM lp1 +), lp3 AS ( + SELECT * + , CASE WHEN net_stake_units > 0 AND cumu_stake_units > 0 AND cumu_stake_units - net_stake_units <= 0 THEN 1 + WHEN net_stake_units < 0 AND cumu_stake_units <= 0 AND cumu_stake_units - net_stake_units > 0 THEN -1 + ELSE 0 END AS net_user_stake_change + FROM lp2 +), lp4 AS ( + SELECT from_address + , date + , SUM(net_user_stake_change) AS net_user_stake_change + FROM lp3 + GROUP BY 1, 2 +), lp5 AS ( + SELECT * + , SUM(net_user_stake_change) OVER (PARTITION BY from_address ORDER BY date) AS cumu_net_user_stake_change + FROM lp4 +), lp6 AS ( + SELECT date + , from_address + , CASE WHEN net_user_stake_change > 0 AND cumu_net_user_stake_change > 0 AND cumu_net_user_stake_change - net_user_stake_change <= 0 THEN 1 + WHEN net_user_stake_change < 0 AND cumu_net_user_stake_change <= 0 AND cumu_net_user_stake_change - net_user_stake_change > 0 THEN -1 + ELSE 0 END AS net_user_stake_change + FROM lp5 +) +SELECT date +, SUM(net_user_stake_change) OVER (ORDER BY date) AS net_user_lps +FROM lp6 + + + + WITH base AS ( SELECT project_name AS collection , token_id @@ -4149,6 +8925,11 @@ SELECT COUNT(1) AS n , COUNT(1) AS nn FROM thorchain.pool_block_statistics +SELECT day +, rune_price_usd +FROM thorchain.daily_pool_stats +WHERE pool_name LIKE 'BNB.BUSD%' + SELECT pool_name , COUNT(1) AS n , MAX(block_timestamp) AS mx diff --git a/solana_model.py b/solana_model.py index f174c3ca..fe4853ae 100644 --- a/solana_model.py +++ b/solana_model.py @@ -177,9 +177,9 @@ def train_model(check_exclude=False, supplement_with_listings=True, use_saved_pa , ( 'BAYC', 3562, 430 ) ] s_df = get_sales(check_exclude, exclude) - s_df.groupby('collection').block_timestamp.max() - s_df[s_df.collection == 'BAYC'].sort_values('block_timestamp', ascending=0).head()[['token_id','block_timestamp','price']] - s_df[s_df.collection == 'MAYC'].sort_values('price', ascending=0).head() + # s_df.groupby('collection').block_timestamp.max() + # s_df[s_df.collection == 'BAYC'].sort_values('block_timestamp', ascending=0).head()[['token_id','block_timestamp','price']] + # s_df[s_df.collection == 'MAYC'].sort_values('price', ascending=0).head() # s_df[s_df.collection == 'Solana Monkey Business'].to_csv('./tableau/data/smb_sales.csv', index=False) # s_df = s_df[-s_df.collection.isin(['BAYC','MAYC'])] # s_df[s_df.collection.isnull()] @@ -204,6 +204,15 @@ def train_model(check_exclude=False, supplement_with_listings=True, use_saved_pa # Load Metadata # ######################### m_df = pd.read_csv('./data/metadata.csv') + # sorted(m_df.collection.unique()) + # sorted(s_df.collection.unique()) + tmp = m_df[['collection','feature_name']].drop_duplicates().groupby('collection').feature_name.count().reset_index().sort_values('feature_name') + tmp.to_csv('~/Downloads/tmp-2.csv', index=False) + cs = sorted(s_df.collection.unique()) + m_df.head() + cs + m_df = m_df[m_df.collection.isin(cs)] + sorted(m_df.collection.unique()) m_df['feature_name'] = m_df.feature_name.apply(lambda x: 'Clothes' if x == 'Clother' else x ) # m_df[m_df.collection == 'DeGods'][['feature_name']].drop_duplicates() # sorted(m_df.collection.unique()) @@ -229,8 +238,8 @@ def train_model(check_exclude=False, supplement_with_listings=True, use_saved_pa m_df = merge(m_df, tokens[['collection','token_id','clean_token_id']].dropna().drop_duplicates() , how='left', ensure=True, on=['collection','token_id'], message='m_df x tokens') m_df['token_id'] = m_df.clean_token_id.fillna(m_df.token_id).astype(float).astype(int).astype(str) s_df = merge(s_df, tokens[['collection','token_id','clean_token_id']].drop_duplicates(), how='left', ensure=True, on=['collection','token_id'], message='s_df x tokens') - s_df[s_df.token_id.isnull()] - sorted(s_df.collection.unique()) + # s_df[s_df.token_id.isnull()] + # sorted(s_df.collection.unique()) # np.isinf(s_df).values.sum() # s_df['clean_token_id'] = s_df.clean_token_id.fillna(s_df.token_id) # s_df['token_id'] = (s_df.clean_token_id).apply(lambda x: re.sub('"', '', str(x))).astype(float).astype(int).astype(str) @@ -378,603 +387,636 @@ def train_model(check_exclude=False, supplement_with_listings=True, use_saved_pa s_df[s_df.collection.isin(['Okay Bears','Catalina Whale Mixer'])] collections = ['Okay Bears'] collections = ['Cets on Creck'] - collections = list(s_df[['collection']].drop_duplicates().merge(m_df[['collection']].drop_duplicates()).collection.unique()) collections = [ c for c in collections if not c in [ 'Galactic Punks','LunaBulls','Galactic Angels','Levana Dragon Eggs','BAKC','BAYC','Astrals','MAYC' ] ] collections = [ c for c in collections if not c in [ 'Okay Bears','Stoned Ape Crew','Cets on Creck' ] ] collections = ['SOLGods'] collections = ['Cets on Creck','Pesky Penguins'] + collections = ['Just Ape.','Bubblegoose Ballers'] + collections = ['Bubblegoose Ballers'] print(sorted(collections)) + salesdf.token_id.values[:3] + salesdf['token_id'] = salesdf.token_id.astype(int) + listings['token_id'] = listings.token_id.astype(int) + sorted(m_df.collection.unique()) + sorted(pred_price.collection.unique()) + rem = m_df[['collection','feature_name']].drop_duplicates().groupby('collection').feature_name.count().reset_index().sort_values('feature_name') + rem = rem[rem.feature_name <= 5].collection.unique() + collections = list(s_df[['collection']].drop_duplicates().merge(m_df[['collection']].drop_duplicates()).collection.unique()) + collections = [ c for c in collections if not c in rem ] + + # DeGods + m_df[m_df.collection=='Astrals'] + rem = sorted(pred_price.collection.unique()) + collections = [ c for c in collections if not c in rem ] + rem = [ 'Levana Dragon Eggs','LunaBulls' ] + collections = [ c for c in collections if not c in rem ] # collections = [ 'Catalina Whale Mixer', 'Okay Bears', 'Pesky Penguins' ] - for collection in collections: - # if collection in ['Astrals','Bakc','BAKC']+[ 'Catalina Whale Mixer', 'Okay Bears', 'Pesky Penguins' ]: - if collection in ['Astrals','Bakc','BAKC']: - continue - if not collection in saved_params.keys(): - saved_params[collection] = {} - coefsdf = coefsdf[coefsdf.collection != collection] - salesdf = salesdf[salesdf.collection != collection] - attributes = attributes[attributes.collection != collection] - pred_price = pred_price[pred_price.collection != collection] - feature_values = feature_values[feature_values.collection != collection] - print('\nWorking on collection {}'.format(collection)) - sales = s_df[ s_df.collection == collection ] - sales[sales.sim==0].block_timestamp.max() - metadata = m_df[ m_df.collection == collection ].drop_duplicates(subset=['token_id','feature_name'], keep='last') - metadata = metadata[metadata.feature_name != 'Genesis Role?'] - metadata[metadata.token_id=='1'] - metadata[metadata.feature_name=='Genesis Role?'].feature_value.unique() - sorted(metadata.feature_name.unique()) - # metadata.groupby(['feature_name','feature_value']).token_id.count().reset_index().to_csv('~/Downloads/tmp.csv', index=False) - # metadata[metadata.token_id == '1'] - metadata['feature_name'] = metadata.feature_name.apply(lambda x: x.strip() ) - metadata[metadata.token_id == '1'] - metadata[metadata.feature_name == 'rank'] - metadata = metadata[-metadata.feature_name.isin(['rank','pct','Pct','ipfs_image'])] - metadata.feature_name.unique() - metadata[(metadata.token_id=='1') & (metadata.collection == 'Solana Monkey Business')] + for i in range(2): + for collection in collections[1:]: + # try: + # if collection in ['Astrals','Bakc','BAKC']+[ 'Catalina Whale Mixer', 'Okay Bears', 'Pesky Penguins' ]: + if collection in ['Bakc','BAKC','Levana Dragon Eggs','LunaBulls']: + continue + # if collection in ['DeGods']: + # continue + if not collection in saved_params.keys(): + saved_params[collection] = {} + coefsdf = coefsdf[coefsdf.collection != collection] + salesdf = salesdf[salesdf.collection != collection] + attributes = attributes[attributes.collection != collection] + pred_price = pred_price[pred_price.collection != collection] + feature_values = feature_values[feature_values.collection != collection] + print('\nWorking on collection {}'.format(collection)) + sales = s_df[ s_df.collection == collection ] + sales[sales.sim==0].block_timestamp.max() + metadata = m_df[ m_df.collection == collection ].drop_duplicates(subset=['token_id','feature_name'], keep='last') + metadata.feature_name.unique() + metadata = metadata[metadata.feature_name != 'Genesis Role?'] + if collection == 'Degen Dojo': + metadata = metadata[metadata.feature_name != 'Encrypted Traits'] + metadata[metadata.token_id=='1'] + metadata[metadata.feature_name=='Genesis Role?'].feature_value.unique() + sorted(metadata.feature_name.unique()) + # metadata.groupby(['feature_name','feature_value']).token_id.count().reset_index().to_csv('~/Downloads/tmp.csv', index=False) + # metadata[metadata.token_id == '1'] + metadata['feature_name'] = metadata.feature_name.apply(lambda x: x.strip() ) + metadata[metadata.token_id == '1'] + metadata[metadata.feature_name == 'rank'] + metadata = metadata[-metadata.feature_name.isin(['rank','pct','Pct','ipfs_image'])] + metadata.feature_name.unique() + metadata[(metadata.token_id=='1') & (metadata.collection == 'Solana Monkey Business')] - # categorize columns - all_names = sorted(metadata.feature_name.unique()) - model_exclude = MODEL_EXCLUDE_COLS[collection] if collection in MODEL_EXCLUDE_COLS.keys() else [] - num_features = sorted((NUMERIC_COLS[collection] if collection in NUMERIC_COLS.keys() else []) + ALL_NUMERIC_COLS) - num_features = [ x for x in num_features if x in metadata.feature_name.unique() ] - num_metadata = metadata[metadata.feature_name.isin(num_features)] - num_metadata[num_metadata.feature_name == 'nft_rank'] - cat_features = sorted([ x for x in all_names if not x in (model_exclude + num_features) ]) - cat_metadata = metadata[metadata.feature_name.isin(cat_features)] + # categorize columns + all_names = sorted(metadata.feature_name.unique()) + model_exclude = MODEL_EXCLUDE_COLS[collection] if collection in MODEL_EXCLUDE_COLS.keys() else [] + num_features = sorted((NUMERIC_COLS[collection] if collection in NUMERIC_COLS.keys() else []) + ALL_NUMERIC_COLS) + num_features = [ x for x in num_features if x in metadata.feature_name.unique() ] + num_metadata = metadata[metadata.feature_name.isin(num_features)] + num_metadata[num_metadata.feature_name == 'nft_rank'] + cat_features = sorted([ x for x in all_names if not x in (model_exclude + num_features) ]) + cat_metadata = metadata[metadata.feature_name.isin(cat_features)] - # create dummies for binary variables - num_metadata = num_metadata.pivot( ['collection','token_id'], ['feature_name'], ['feature_value'] ).reset_index() - num_metadata.columns = [ 'collection','token_id' ] + num_features + # create dummies for binary variables + num_metadata = num_metadata.pivot( ['collection','token_id'], ['feature_name'], ['feature_value'] ).reset_index() + num_metadata.columns = [ 'collection','token_id' ] + num_features - # create dummies for binary variables - cat_metadata = cat_metadata.pivot( ['collection','token_id'], ['feature_name'], ['feature_value'] ).reset_index() - cat_metadata.columns = [ 'collection','token_id' ] + cat_features - # cat_metadata = calculate_percentages( cat_metadata, cat_features ) - dummies = pd.get_dummies(cat_metadata[cat_features]) - # dummies.head(1).to_csv('~/Downloads/tmp2.csv', index=False) - if collection == 'Solana Monkey Business': - # dummies['matching_white'] = ((dummies['Clothes_Beige Smoking'] == 1) & ((dummies['Hat_White Fedora 1'] + dummies['Hat_White Fedora 2']) == 1)).astype(int) - # dummies['matching_black'] = ((dummies['Clothes_Black Smoking'] == 1) & ((dummies['Hat_Black Fedora 1'] + dummies['Hat_Black Fedora 2'] + dummies['Hat_Black Top Hat']) == 1)).astype(int) - # dummies['matching_top'] = ((dummies['matching_black'] == 1) | (dummies['matching_white']== 1)).astype(int) - # dummies['matching_cop'] = ((dummies['Clothes_Cop Vest'] == 1) & ((dummies['Hat_Cop Hat']==1))).astype(int) - # dummies['matching_green'] = ((dummies['Clothes_Green Smoking'] == 1) & ((dummies['Hat_Green Top Hat']) == 1)).astype(int) - dummies['naked_1_att'] = ((dummies['Attribute Count_1'] == 1) & (dummies['Clothes_None'] == 1)).astype(int) - # dummies['naked_1_att_hat'] = ((dummies['Attribute Count_1'] == 1) & (dummies['Hat_None'] == 0)).astype(int) - dummies['fedora'] = (dummies['Hat_Black Fedora 1'] + dummies['Hat_Black Fedora 2'] + dummies['Hat_White Fedora 1'] + dummies['Hat_White Fedora 2'] + dummies['Hat_White Fedora 2'] >= 1 ).astype(int) - dummies['backwards_cap'] = (dummies['Hat_Black Backwards Cap'] + dummies['Hat_Blue Backwards Cap'] + dummies['Hat_Green Backwards Cap'] + dummies['Hat_Orange Backwards Cap'] + dummies['Hat_Purple Backwards Cap'] + dummies['Hat_Solana Backwards Cap'] >= 1 ).astype(int) - # del dummies['matching_white'] - # del dummies['matching_black'] + # create dummies for binary variables + cat_metadata = cat_metadata.pivot( ['collection','token_id'], ['feature_name'], ['feature_value'] ).reset_index() + cat_metadata.columns = [ 'collection','token_id' ] + cat_features + # cat_metadata = calculate_percentages( cat_metadata, cat_features ) + dummies = pd.get_dummies(cat_metadata[cat_features]) + # dummies.head(1).to_csv('~/Downloads/tmp2.csv', index=False) + if collection == 'Solana Monkey Business': + # dummies['matching_white'] = ((dummies['Clothes_Beige Smoking'] == 1) & ((dummies['Hat_White Fedora 1'] + dummies['Hat_White Fedora 2']) == 1)).astype(int) + # dummies['matching_black'] = ((dummies['Clothes_Black Smoking'] == 1) & ((dummies['Hat_Black Fedora 1'] + dummies['Hat_Black Fedora 2'] + dummies['Hat_Black Top Hat']) == 1)).astype(int) + # dummies['matching_top'] = ((dummies['matching_black'] == 1) | (dummies['matching_white']== 1)).astype(int) + # dummies['matching_cop'] = ((dummies['Clothes_Cop Vest'] == 1) & ((dummies['Hat_Cop Hat']==1))).astype(int) + # dummies['matching_green'] = ((dummies['Clothes_Green Smoking'] == 1) & ((dummies['Hat_Green Top Hat']) == 1)).astype(int) + dummies['naked_1_att'] = ((dummies['Attribute Count_1'] == 1) & (dummies['Clothes_None'] == 1)).astype(int) + # dummies['naked_1_att_hat'] = ((dummies['Attribute Count_1'] == 1) & (dummies['Hat_None'] == 0)).astype(int) + dummies['fedora'] = (dummies['Hat_Black Fedora 1'] + dummies['Hat_Black Fedora 2'] + dummies['Hat_White Fedora 1'] + dummies['Hat_White Fedora 2'] + dummies['Hat_White Fedora 2'] >= 1 ).astype(int) + dummies['backwards_cap'] = (dummies['Hat_Black Backwards Cap'] + dummies['Hat_Blue Backwards Cap'] + dummies['Hat_Green Backwards Cap'] + dummies['Hat_Orange Backwards Cap'] + dummies['Hat_Purple Backwards Cap'] + dummies['Hat_Solana Backwards Cap'] >= 1 ).astype(int) + # del dummies['matching_white'] + # del dummies['matching_black'] - cat_metadata = pd.concat([ cat_metadata.reset_index(drop=True), dummies.reset_index(drop=True) ], axis=1) - # del cat_metadata['pct'] + cat_metadata = pd.concat([ cat_metadata.reset_index(drop=True), dummies.reset_index(drop=True) ], axis=1) + # del cat_metadata['pct'] - for c in model_exclude: - if c in dummies.columns: - del dummies[c] - pred_cols = num_features + list(dummies.columns) - pred_cols = [ c for c in pred_cols if not c in model_exclude+['Matching_No'] ] + for c in model_exclude: + if c in dummies.columns: + del dummies[c] + pred_cols = num_features + list(dummies.columns) + pred_cols = [ c for c in pred_cols if not c in model_exclude+['Matching_No'] ] - if len(sales) < 1000: - pred_cols = [ x for x in pred_cols if 'rank' in x or 'is_top_' in x ] + if len(sales) < 1000: + pred_cols = [ x for x in pred_cols if 'rank' in x or 'is_top_' in x ] - # create training df - sales['token_id'] = sales.token_id.astype(str) - num_metadata['token_id'] = num_metadata.token_id.astype(str) - df = merge(sales, num_metadata, ['collection','token_id'], ensure=False) - df = merge(df, cat_metadata, ['collection','token_id'], ensure=False) - assert(len(df.columns) < 1000) + # create training df + sales['token_id'] = sales.token_id.astype(str) + num_metadata['token_id'] = num_metadata.token_id.astype(str) + df = merge(sales, num_metadata, ['collection','token_id'], ensure=False) + df = merge(df, cat_metadata, ['collection','token_id'], ensure=False) + assert(len(df.columns) < 1000) + list(df.columns)[100:] + list(df.columns)[-150:] - # test dataFrame - ensure = not collection in ['Aurory','Stoned Ape Crew'] - test = merge(num_metadata, cat_metadata, ['collection','token_id'], ensure=False) + # test dataFrame + ensure = not collection in ['Aurory','Stoned Ape Crew'] + test = merge(num_metadata, cat_metadata, ['collection','token_id'], ensure=False) - # if collection == 'Solana Monkey Business': - # hat = metadata[ metadata.feature_name == 'Hat' ] - # hat['color'] = hat.feature_value.apply(lambda x: re.split(' ', x)[0] ) - # clothes = metadata[ metadata.feature_name == 'Clothes' ] - # clothes['color'] = clothes.feature_value.apply(lambda x: re.split(' ', x)[0] ) - # matching = hat[['token_id','color']].merge(clothes[['token_id','color']]) - # app = cat_metadata[ (dummies.matching_top == 1) | (dummies.matching_cop == 1) ][['token_id']] - # matching = matching[['token_id']].append(app[['token_id']]).drop_duplicates() - # matching['matching'] = 1 - # del dummies['matching_cop'] - # del dummies['matching_top'] - # # dummies = merge(dummies, matching, on=['token_id'], how='left').fillna(0) - # df = merge(df, matching, on=['token_id'], how='left').fillna(0) - # test = merge(test, matching, on=['token_id'], how='left').fillna(0) - # pred_cols.append('matching') + # if collection == 'Solana Monkey Business': + # hat = metadata[ metadata.feature_name == 'Hat' ] + # hat['color'] = hat.feature_value.apply(lambda x: re.split(' ', x)[0] ) + # clothes = metadata[ metadata.feature_name == 'Clothes' ] + # clothes['color'] = clothes.feature_value.apply(lambda x: re.split(' ', x)[0] ) + # matching = hat[['token_id','color']].merge(clothes[['token_id','color']]) + # app = cat_metadata[ (dummies.matching_top == 1) | (dummies.matching_cop == 1) ][['token_id']] + # matching = matching[['token_id']].append(app[['token_id']]).drop_duplicates() + # matching['matching'] = 1 + # del dummies['matching_cop'] + # del dummies['matching_top'] + # # dummies = merge(dummies, matching, on=['token_id'], how='left').fillna(0) + # df = merge(df, matching, on=['token_id'], how='left').fillna(0) + # test = merge(test, matching, on=['token_id'], how='left').fillna(0) + # pred_cols.append('matching') - for c in num_features: - df[c] = df[c].apply(lambda x: just_float(x)) - test[c] = test[c].apply(lambda x: just_float(x) ) + for c in num_features: + df[c] = df[c].apply(lambda x: just_float(x)) + test[c] = test[c].apply(lambda x: just_float(x) ) - ################################# - # Create Test DataFrame # - ################################# - tail = df.sort_values('timestamp').tail(1) - if collection == 'Solana Monkey Business': - test.loc[ test.token_id == '903', 'nft_rank' ] = 18 - for c in [ 'std_timestamp','mn_20','log_mn_20' ]: - if c in tail.columns: - test[c] = tail[c].values[0] + ################################# + # Create Test DataFrame # + ################################# + tail = df.sort_values('timestamp').tail(1) + if collection == 'Solana Monkey Business': + test.loc[ test.token_id == '903', 'nft_rank' ] = 18 + for c in [ 'std_timestamp','mn_20','log_mn_20' ]: + if c in tail.columns: + test[c] = tail[c].values[0] - for tmp in [df, test]: - for i in [100, 250, 1000]: - if collection in ['Levana Dragon Eggs']: - tmp['is_top_{}'.format(i)] = (tmp.collection_rank <= i).astype(int) - else: - tmp['is_top_{}'.format(i)] = (tmp.nft_rank <= i).astype(int) - pred_cols += [ 'is_top_100','is_top_250','is_top_1000' ] - if 'collection_rank' in pred_cols: - pred_cols = [ x for x in pred_cols if not x in ['nft_rank'] ] - df.sort_values('price', ascending=0)[['price']].head(20) - # df.groupby(['rarity','weight']).price.mean() + for tmp in [df, test]: + for i in [100, 250, 1000]: + if collection in ['Levana Dragon Eggs']: + tmp['is_top_{}'.format(i)] = (tmp.collection_rank <= i).astype(int) + else: + tmp['is_top_{}'.format(i)] = (tmp.nft_rank <= i).astype(int) + pred_cols += [ 'is_top_100','is_top_250','is_top_1000' ] + if 'collection_rank' in pred_cols: + pred_cols = [ x for x in pred_cols if not x in ['nft_rank'] ] + df.sort_values('price', ascending=0)[['price']].head(20) + # df.groupby(['rarity','weight']).price.mean() - # create target cols - target_col = 'adj_price' - df[target_col] = df.apply(lambda x: max(0.7 * (x['mn_20'] - 0.2), x['price']), 1 ) - # df['mn_20'] = df.mn_20 * 1.01 - df = df[df[target_col].notnull()] - df['log_price'] = df[target_col].apply(lambda x: np.log(x) ) - df['rel_price_0'] = df[target_col] - df.mn_20 - df['rel_price_1'] = df[target_col] / df.mn_20 - df = df[df.mn_20 > 0] - df['log_mn_20'] = np.log(df.mn_20) - print('Training on {} sales'.format(len(df))) - df = standardize_df(df, pred_cols) - test = standardize_df(test, pred_cols, df) + # create target cols + target_col = 'adj_price' + df[target_col] = df.apply(lambda x: max(0.7 * (x['mn_20'] - 0.2), x['price']), 1 ) + # df['mn_20'] = df.mn_20 * 1.01 + df = df[df[target_col].notnull()] + df['log_price'] = df[target_col].apply(lambda x: np.log(x) ) + df['rel_price_0'] = df[target_col] - df.mn_20 + df['rel_price_1'] = df[target_col] / df.mn_20 + df = df[df.mn_20 > 0] + df['log_mn_20'] = np.log(df.mn_20) + print('Training on {} sales'.format(len(df))) + df = standardize_df(df, pred_cols) + test = standardize_df(test, pred_cols, df) - std_pred_cols_0 = [ 'std_{}'.format(c) for c in pred_cols ] - std_pred_cols = [ 'std_{}'.format(c) for c in pred_cols ] - df.sort_values('rel_price_0', ascending=0).head()[['token_id','nft_rank','price','rel_price_0']] + std_pred_cols_0 = [ 'std_{}'.format(c) for c in pred_cols ] + std_pred_cols = [ 'std_{}'.format(c) for c in pred_cols ] + df.sort_values('rel_price_0', ascending=0).head()[['token_id','nft_rank','price','rel_price_0']] - ######################### - # Run the Model # - ######################### - tmp = df[std_pred_cols].count().reset_index() - tmp.columns = ['a','b'] - tmp.sort_values('b').head(20) - rem = list(tmp[tmp.b==0].a.values) - std_pred_cols = [ c for c in std_pred_cols if not c in rem ] - # if collection == 'Levana Dragon Eggs': - # std_pred_cols = [ 'std_essence_Dark','std_collection_rank_group_0','std_rarity_Legendary','std_rarity_Rare','std_rarity_Ancient','std_collection_rank','std_transformed_collection_rank' ] - mn = df.timestamp.min() - mx = df.timestamp.max() - a_week_ago = (time.time() * 1000000000) - (60 * 60 * 24 * 7 * 1000000000) - df['wt'] = df.timestamp.apply(lambda x: 4.0 ** ((x - mn) / (mx - mn)) ) - df.loc[ df.timestamp >= a_week_ago, 'wt' ] = 5 - df['wt'] = df.apply(lambda x: 0 if (x['train_exclude']==1 and (x['train_exclude_price'] != x['train_exclude_price'] or x['train_exclude_price'] == x['price'])) else x['wt'], 1 ) - df.loc[ (df.collection == 'Aurory') & (df.block_timestamp <= '2021-09-05'), 'wt' ] = 0.05 - if collection == 'Levana Dragon Eggs': - df['wt'] = 1 - # df['wt'] = df.price.apply(lambda x: 1.0 / (x ** 0.9) ) - # df.sort_values('price', ascending=0)[['price','wt']].head(20) - # std_pred_cols = [ 'std_Hat_Crown','std_adj_nft_rank_0','std_Hat_None','std_Eyes_None','std_Clothes_None','std_Attribute Count_4','std_Mouth_None','std_adj_nft_rank_1','std_Type_Dark','std_Ears_None','std_Background_Light purple','std_Hat_Black Fedora 2','std_Hat_White Fedora 2','std_Attribute Count_0','std_Type_Skeleton','std_Attribute Count_2','std_Attribute Count_1','std_Hat_Protagonist Black Hat','std_Clothes_Sailor Vest','std_Mouth_Pipe','std_Hat_Protagonist White Hat','std_Clothes_Pirate Vest','std_Hat_Roman Helmet','std_Type_Solana','std_Clothes_Beige Smoking','std_Hat_Military Helmet','std_Hat_White Fedora 1','std_naked_1_att','std_Type_Zombie','std_Clothes_Roman Armor','std_Eyes_3D Glasses','std_Clothes_Orange Kimono','std_Hat_Green Punk Hair','std_Hat_Sombrero','std_Clothes_Military Vest','std_Hat_Space Warrior Hair','std_Hat_Blue Punk Hair','std_Clothes_Orange Jacket','std_Ears_Earing Silver','std_Eyes_Laser Eyes','std_Eyes_Vipers','std_Type_Alien','std_Type_Red','std_Hat_Admiral Hat' ] - # cur_std_pred_cols = [ 'std_adj_nft_rank_0','std_Hat_Crown','std_adj_nft_rank_1','std_Type_Skeleton','std_Type_Alien','std_Clothes_None','std_Eyes_Vipers','std_Hat_Space Warrior Hair','std_Type_Zombie','std_Clothes_Pirate Vest','std_Clothes_Orange Kimono','std_Eyes_Laser Eyes','std_Type_Solana','std_Hat_Ninja Bandana','std_Hat_Solana Backwards Cap','std_Eyes_Solana Vipers','std_Attribute Count_0','std_Attribute Count_1','std_Attribute Count_2','std_Attribute Count_3','std_Attribute Count_5','std_Hat_Strawhat','std_Hat_Admiral Hat','std_matching_top','std_Hat_Sombrero','std_matching_cop','std_Hat_Cowboy Hat','std_Hat_None' ] - # cur_std_pred_cols = deepcopy(std_pred_cols) - # g = df[std_pred_cols].sum().reset_index() - # g.columns = [ 'col','cnt' ] - # g = g.sort_values('cnt') - # g.head(20) - if collection == 'Solana Monkey Busines': - df.loc[ df.token_id == '903', 'nft_rank' ] = 18 - df[df.token_id=='903'] - df[df.token_id==903] + ######################### + # Run the Model # + ######################### + tmp = df[std_pred_cols].count().reset_index() + tmp.columns = ['a','b'] + tmp.sort_values('b').head(20) + rem = list(tmp[tmp.b==0].a.values) + std_pred_cols = [ c for c in std_pred_cols if not c in rem ] + # if collection == 'Levana Dragon Eggs': + # std_pred_cols = [ 'std_essence_Dark','std_collection_rank_group_0','std_rarity_Legendary','std_rarity_Rare','std_rarity_Ancient','std_collection_rank','std_transformed_collection_rank' ] + mn = df.timestamp.min() + mx = df.timestamp.max() + a_week_ago = (time.time() * 1000000000) - (60 * 60 * 24 * 7 * 1000000000) + df['wt'] = df.timestamp.apply(lambda x: 4.0 ** ((x - mn) / (mx - mn)) ) + df.loc[ df.timestamp >= a_week_ago, 'wt' ] = 5 + df['wt'] = df.apply(lambda x: 0 if (x['train_exclude']==1 and (x['train_exclude_price'] != x['train_exclude_price'] or x['train_exclude_price'] == x['price'])) else x['wt'], 1 ) + df.loc[ (df.collection == 'Aurory') & (df.block_timestamp <= '2021-09-05'), 'wt' ] = 0.05 + if collection == 'Levana Dragon Eggs': + df['wt'] = 1 + # df['wt'] = df.price.apply(lambda x: 1.0 / (x ** 0.9) ) + # df.sort_values('price', ascending=0)[['price','wt']].head(20) + # std_pred_cols = [ 'std_Hat_Crown','std_adj_nft_rank_0','std_Hat_None','std_Eyes_None','std_Clothes_None','std_Attribute Count_4','std_Mouth_None','std_adj_nft_rank_1','std_Type_Dark','std_Ears_None','std_Background_Light purple','std_Hat_Black Fedora 2','std_Hat_White Fedora 2','std_Attribute Count_0','std_Type_Skeleton','std_Attribute Count_2','std_Attribute Count_1','std_Hat_Protagonist Black Hat','std_Clothes_Sailor Vest','std_Mouth_Pipe','std_Hat_Protagonist White Hat','std_Clothes_Pirate Vest','std_Hat_Roman Helmet','std_Type_Solana','std_Clothes_Beige Smoking','std_Hat_Military Helmet','std_Hat_White Fedora 1','std_naked_1_att','std_Type_Zombie','std_Clothes_Roman Armor','std_Eyes_3D Glasses','std_Clothes_Orange Kimono','std_Hat_Green Punk Hair','std_Hat_Sombrero','std_Clothes_Military Vest','std_Hat_Space Warrior Hair','std_Hat_Blue Punk Hair','std_Clothes_Orange Jacket','std_Ears_Earing Silver','std_Eyes_Laser Eyes','std_Eyes_Vipers','std_Type_Alien','std_Type_Red','std_Hat_Admiral Hat' ] + # cur_std_pred_cols = [ 'std_adj_nft_rank_0','std_Hat_Crown','std_adj_nft_rank_1','std_Type_Skeleton','std_Type_Alien','std_Clothes_None','std_Eyes_Vipers','std_Hat_Space Warrior Hair','std_Type_Zombie','std_Clothes_Pirate Vest','std_Clothes_Orange Kimono','std_Eyes_Laser Eyes','std_Type_Solana','std_Hat_Ninja Bandana','std_Hat_Solana Backwards Cap','std_Eyes_Solana Vipers','std_Attribute Count_0','std_Attribute Count_1','std_Attribute Count_2','std_Attribute Count_3','std_Attribute Count_5','std_Hat_Strawhat','std_Hat_Admiral Hat','std_matching_top','std_Hat_Sombrero','std_matching_cop','std_Hat_Cowboy Hat','std_Hat_None' ] + # cur_std_pred_cols = deepcopy(std_pred_cols) + # g = df[std_pred_cols].sum().reset_index() + # g.columns = [ 'col','cnt' ] + # g = g.sort_values('cnt') + # g.head(20) + if collection == 'Solana Monkey Busines': + df.loc[ df.token_id == '903', 'nft_rank' ] = 18 + df[df.token_id=='903'] + df[df.token_id==903] - # CUR_FLOOR = df.sort_values('block_timestamp', ascending=0).mn_20.values[0] - CUR_FLOOR = listings[(listings.collection == collection) & (listings.price.notnull())].price.min() - print('CUR_FLOOR = {}'.format(CUR_FLOOR)) - df['tmp'] = df.nft_rank.apply(lambda x: int(x / 1000) ) - df.groupby('tmp').rel_price_0.mean() - df.groupby('tmp').rel_price_0.median() - df.groupby('tmp').rel_price_1.median() - df.groupby('tmp').rel_price_1.mean() - df[[ 'nft_rank','rel_price_0' ]].to_csv('~/Downloads/tmp.csv') - if collection == 'MAYC': - df = df[-((df.rel_price_0 >= 100) & (df.nft_rank > 1000))] - df = df[(df.mn_20 >= 1)] - if collection == 'BAYC': - df = df[(df.mn_20 >= 10)] - if collection == 'MAYC': - df = df[(df.mn_20 >= 2)] - df.sort_values('price', ascending=0).head(30)[['token_id','price','rel_price_0','rel_price_1','nft_rank','block_timestamp']] - df.sort_values('rel_price_1', ascending=0).head(30)[['token_id','price','rel_price_0','rel_price_1','nft_rank','block_timestamp']] - df.sort_values('rel_price_1', ascending=0).head(100)[['token_id','mn_20','price','rel_price_0','rel_price_1','nft_rank','block_timestamp']].to_csv('~/Downloads/tmp.csv', index=False) + # CUR_FLOOR = df.sort_values('block_timestamp', ascending=0).mn_20.values[0] + CUR_FLOOR = listings[(listings.collection == collection) & (listings.price.notnull())].price.min() + print('CUR_FLOOR = {}'.format(CUR_FLOOR)) + if not CUR_FLOOR: + continue + df['tmp'] = df.nft_rank.apply(lambda x: int(x / 1000) ) + df.groupby('tmp').rel_price_0.mean() + df.groupby('tmp').rel_price_0.median() + df.groupby('tmp').rel_price_1.median() + df.groupby('tmp').rel_price_1.mean() + df[[ 'nft_rank','rel_price_0' ]].to_csv('~/Downloads/tmp.csv') + if collection == 'MAYC': + df = df[-((df.rel_price_0 >= 100) & (df.nft_rank > 1000))] + df = df[(df.mn_20 >= 1)] + if collection == 'BAYC': + df = df[(df.mn_20 >= 10)] + if collection == 'MAYC': + df = df[(df.mn_20 >= 2)] + df.sort_values('price', ascending=0).head(30)[['token_id','price','rel_price_0','rel_price_1','nft_rank','block_timestamp']] + df.sort_values('rel_price_1', ascending=0).head(30)[['token_id','price','rel_price_0','rel_price_1','nft_rank','block_timestamp']] + df.sort_values('rel_price_1', ascending=0).head(100)[['token_id','mn_20','price','rel_price_0','rel_price_1','nft_rank','block_timestamp']].to_csv('~/Downloads/tmp.csv', index=False) - df = df.reset_index(drop=True) - X = df[std_pred_cols].values - wt = df['wt'].values - y_0 = df.rel_price_0.values - y_1 = df.rel_price_1.values - df.sort_values('price', ascending=0).head(15)[['price','token_id','nft_rank','block_timestamp']] - df[df.sim == 0].block_timestamp.max() + df = df.reset_index(drop=True) + X = df[std_pred_cols].values + wt = df['wt'].values + y_0 = df.rel_price_0.values + y_1 = df.rel_price_1.values + df.sort_values('price', ascending=0).head(15)[['price','token_id','nft_rank','block_timestamp']] + df[df.sim == 0].block_timestamp.max() - for target_col in [ 'rel_price_0', 'rel_price_1' ]: - it = target_col[-1] - y_val = df[target_col].values - print('target_col = {}'.format(target_col)) - mn = -1 - cols = [ 'std_nft_rank','std_adj_nft_rank_0','std_adj_nft_rank_1','std_adj_nft_rank_2' ] - clf = Ridge(alpha = 1) - # while mn < 0 and len(cols): - # clf.fit(df[cols].values, y_val, df.wt.values) - # coefs = get_coefs(cols, clf.coef_) - # mn = min(coefs.val) if len(coefs) else 0 - # if mn < 0: - # cols.remove(coefs.col.values[-1]) + for target_col in [ 'rel_price_0', 'rel_price_1' ]: + it = target_col[-1] + y_val = df[target_col].values + print('target_col = {}'.format(target_col)) + mn = -1 + cols = [ 'std_nft_rank','std_adj_nft_rank_0','std_adj_nft_rank_1','std_adj_nft_rank_2' ] + clf = Ridge(alpha = 1) + # while mn < 0 and len(cols): + # clf.fit(df[cols].values, y_val, df.wt.values) + # coefs = get_coefs(cols, clf.coef_) + # mn = min(coefs.val) if len(coefs) else 0 + # if mn < 0: + # cols.remove(coefs.col.values[-1]) - col = 'rarity_value_'+it - model = 'ridge' - df[col] = 0 - test[col] = 0 - # df, bst_p, bst_r = ku.get_bst_params( model, df, df[cols].values, y_val, target_col, col, verbose = True, wt_col='wt' ) - # test = ku.apply_model( model, bst_p, df, test, cols, target_col, col) + col = 'rarity_value_'+it + model = 'ridge' + df[col] = 0 + test[col] = 0 + # df, bst_p, bst_r = ku.get_bst_params( model, df, df[cols].values, y_val, target_col, col, verbose = True, wt_col='wt' ) + # test = ku.apply_model( model, bst_p, df, test, cols, target_col, col) - # df['rarity_value_'+it] = clf.predict(df[cols].values) - rar_adj_target_col = 'rar_adj_'+target_col - df[rar_adj_target_col] = df[target_col] - df['rarity_value_'+it] - # test[rar_adj_target_col] = test[target_col] - test['rarity_value_'+it] - y_val_rar_adj = df[rar_adj_target_col].values - models = ['las','ridge'] if target_col == 'rel_price_1' or len(sales) < 1000 else ['las','ridge','rfr'] - for model in models: - cur_std_pred_cols = deepcopy(std_pred_cols) - print(model) - y = y_val_rar_adj if model in ['rfr'] else y_val - col = 'y_pred_{}_{}'.format(model, it) - params = [saved_params[collection][col]] if col in saved_params[collection].keys() and use_saved_params else [] - df, bst_p, bst_r = ku.get_bst_params( model, df, X, y, target_col, col, verbose = True, wt_col='wt', params = params ) - saved_params[collection][col] = bst_p + # df['rarity_value_'+it] = clf.predict(df[cols].values) + rar_adj_target_col = 'rar_adj_'+target_col + df[rar_adj_target_col] = df[target_col] - df['rarity_value_'+it] + # test[rar_adj_target_col] = test[target_col] - test['rarity_value_'+it] + y_val_rar_adj = df[rar_adj_target_col].values + models = ['las','ridge'] if target_col == 'rel_price_1' or len(sales) < 1000 else ['las','ridge','rfr'] + for model in models: + cur_std_pred_cols = deepcopy(std_pred_cols) + print(model) + y = y_val_rar_adj if model in ['rfr'] else y_val + col = 'y_pred_{}_{}'.format(model, it) + params = [saved_params[collection][col]] if col in saved_params[collection].keys() and use_saved_params else [] + df, bst_p, bst_r = ku.get_bst_params( model, df, X, y, target_col, col, verbose = True, wt_col='wt', params = params ) + saved_params[collection][col] = bst_p - # if model == 'ridge': - # while len(cur_std_pred_cols) > 50: - # coefs = get_coefs(cur_std_pred_cols, clf.coef_) - # cur_std_pred_cols.remove(coefs.col.values[-1]) - # new_X = df[cur_std_pred_cols].values - # clf = ku.get_model(model, bst_p) - # clf.fit(new_X, y) - # # coefs.to_csv('./data/coefs/{}_{}_{}.csv'.format(collection, model, it)) - # new_X = df[cur_std_pred_cols].values - # df, bst_p, bst_r = ku.get_bst_params( model, df, new_X, y, target_col, col, verbose = True, wt_col='wt' ) + # if model == 'ridge': + # while len(cur_std_pred_cols) > 50: + # coefs = get_coefs(cur_std_pred_cols, clf.coef_) + # cur_std_pred_cols.remove(coefs.col.values[-1]) + # new_X = df[cur_std_pred_cols].values + # clf = ku.get_model(model, bst_p) + # clf.fit(new_X, y) + # # coefs.to_csv('./data/coefs/{}_{}_{}.csv'.format(collection, model, it)) + # new_X = df[cur_std_pred_cols].values + # df, bst_p, bst_r = ku.get_bst_params( model, df, new_X, y, target_col, col, verbose = True, wt_col='wt' ) - if model in ['las','ridge']: - clf = ku.get_model(model, bst_p) - clf.fit(X, y, wt) - coefs = get_coefs(cur_std_pred_cols, clf.coef_) - mn = coefs.val.min() - while mn < 0: - cur_std_pred_cols = [ c for c in coefs[coefs.val >= 0 ].col.unique() ] - X_new = df[cur_std_pred_cols].values - clf.fit(X_new, y, wt) + if model in ['las','ridge']: + clf = ku.get_model(model, bst_p) + clf.fit(X, y, wt) coefs = get_coefs(cur_std_pred_cols, clf.coef_) mn = coefs.val.min() - if mn >= 0: - df, bst_p, bst_r = ku.get_bst_params( model, df, X_new, y, target_col, col, verbose = True, wt_col='wt', params = [bst_p] ) - coefs['col'] = coefs.col.apply(lambda x: re.sub('std_', '', x) ) - coefs['n'] = 0 - n = pd.DataFrame() - for c in cat_metadata.columns: - if not c in [ 'collection','token_id' ]: - coefs.loc[ coefs.col == c, 'n' ] = len(cat_metadata[cat_metadata[c] == 1]) - coefs.to_csv('./data/coefs/{}_{}_{}.csv'.format(collection, model, it), index=False) + while mn < 0: + cur_std_pred_cols = [ c for c in coefs[coefs.val >= 0 ].col.unique() ] + X_new = df[cur_std_pred_cols].values + clf.fit(X_new, y, wt) + coefs = get_coefs(cur_std_pred_cols, clf.coef_) + mn = coefs.val.min() + if mn >= 0: + df, bst_p, bst_r = ku.get_bst_params( model, df, X_new, y, target_col, col, verbose = True, wt_col='wt', params = [bst_p] ) + coefs['col'] = coefs.col.apply(lambda x: re.sub('std_', '', x) ) + coefs['n'] = 0 + n = pd.DataFrame() + for c in cat_metadata.columns: + if not c in [ 'collection','token_id' ]: + coefs.loc[ coefs.col == c, 'n' ] = len(cat_metadata[cat_metadata[c] == 1]) + coefs.to_csv('./data/coefs/{}_{}_{}.csv'.format(collection, model, it), index=False) - test = ku.apply_model( model, bst_p, df, test, cur_std_pred_cols, target_col, col) - if model in ['rfr']: - df[col] = df[col] + df['rarity_value_'+it] - test[col] = test[col] + test['rarity_value_'+it] + test = ku.apply_model( model, bst_p, df, test, cur_std_pred_cols, target_col, col) + if model in ['rfr']: + df[col] = df[col] + df['rarity_value_'+it] + test[col] = test[col] + test['rarity_value_'+it] - mn = -1 - cols = [ c for c in df.columns if c[:7] == 'y_pred_' and c[-1] == it ] - clf = LinearRegression() - df[cols].mean() - df[cols].median() - test[cols].mean() - test[cols].median() - while mn < 0 and len(cols): - clf.fit(df[cols].values, df[target_col].values) - coefs = get_coefs(cols, clf.coef_) - mn = min(coefs.val) if len(coefs) else 0 - if mn < 0: - cols.remove(coefs.col.values[-1]) + mn = -1 + cols = [ c for c in df.columns if c[:7] == 'y_pred_' and c[-1] == it ] + clf = LinearRegression() + df[cols].mean() + df[cols].median() + test[cols].mean() + test[cols].median() + while mn < 0 and len(cols): + clf.fit(df[cols].values, df[target_col].values) + coefs = get_coefs(cols, clf.coef_) + mn = min(coefs.val) if len(coefs) else 0 + if mn < 0: + cols.remove(coefs.col.values[-1]) + else: + print(coefs) + if it == '0': + df['pred_lin'] = clf.predict(df[cols].values) + df.mn_20 + test['pred_lin'] = clf.predict(test[cols].values) + CUR_FLOOR + # df['pred_lin'] = df.pred_lin.apply(lambda x: max(0, x)) + df.mn_20 else: - print(coefs) - if it == '0': - df['pred_lin'] = clf.predict(df[cols].values) + df.mn_20 - test['pred_lin'] = clf.predict(test[cols].values) + CUR_FLOOR - # df['pred_lin'] = df.pred_lin.apply(lambda x: max(0, x)) + df.mn_20 - else: - df['pred_log'] = clf.predict(df[cols].values) - df['pred_log'] = df.pred_log.apply(lambda x: max(1, x)) * df.mn_20 - test['pred_log'] = clf.predict(test[cols].values) - test['pred_log'] = test.pred_log.apply(lambda x: max(1, x)) * CUR_FLOOR + df['pred_log'] = clf.predict(df[cols].values) + df['pred_log'] = df.pred_log.apply(lambda x: max(1, x)) * df.mn_20 + test['pred_log'] = clf.predict(test[cols].values) + test['pred_log'] = test.pred_log.apply(lambda x: max(1, x)) * CUR_FLOOR - clf = LinearRegression(fit_intercept=False) - target_col = 'adj_price' - clf.fit( df[['pred_lin','pred_log']].values, df[target_col].values, df.wt.values ) - score = clf.score( df[['pred_lin','pred_log']].values, df[target_col].values, df.wt.values ) - tmp = df[['token_id','block_timestamp','wt','mn_20','pred_lin','pred_log','price','nft_rank']] - tmp['block_timestamp'] = tmp.block_timestamp.apply(lambda x: str(x)[:10] ) - tmp['err_0'] = tmp.pred_lin - tmp.price - tmp['err_1'] = tmp.pred_log / tmp.price - tmp.to_csv('~/Downloads/tmp.csv', index=False) - print('R-Sq: {}'.format(round(score * 100, 1))) - # df[['pred_lin','pred_log',target_col]].mean() - # df[['pred_lin','pred_log',target_col]].median() - # test[['pred_lin','pred_log']].mean() - # test[['pred_lin','pred_log']].median() + clf = LinearRegression(fit_intercept=False) + target_col = 'adj_price' + clf.fit( df[['pred_lin','pred_log']].values, df[target_col].values, df.wt.values ) + score = clf.score( df[['pred_lin','pred_log']].values, df[target_col].values, df.wt.values ) + tmp = df[['token_id','block_timestamp','wt','mn_20','pred_lin','pred_log','price','nft_rank']] + tmp['block_timestamp'] = tmp.block_timestamp.apply(lambda x: str(x)[:10] ) + tmp['err_0'] = tmp.pred_lin - tmp.price + tmp['err_1'] = tmp.pred_log / tmp.price + tmp.to_csv('~/Downloads/tmp.csv', index=False) + print('R-Sq: {}'.format(round(score * 100, 1))) + # df[['pred_lin','pred_log',target_col]].mean() + # df[['pred_lin','pred_log',target_col]].median() + # test[['pred_lin','pred_log']].mean() + # test[['pred_lin','pred_log']].median() + + print('Price = {} * lin + {} * log'.format( round(clf.coef_[0], 2), round(clf.coef_[1], 2) )) + tmp = pd.DataFrame([[collection, clf.coef_[0], clf.coef_[1], CUR_FLOOR]], columns=['collection','lin_coef','log_coef','floor_price']) + if clf.coef_[0] < 0: + print('Only using log') + df['pred'] = df.pred_log + test['pred'] = test.pred_log + tmp['lin_coef'] = 0 + tmp['log_coef'] = 1 + elif clf.coef_[1] < 0: + print('Only using lin') + df['pred'] = df.pred_lin + test['pred'] = test.pred_lin + tmp['lin_coef'] = 1 + tmp['log_coef'] = 0 + else: + print('Only using BOTH!') + df['pred'] = clf.predict( df[['pred_lin','pred_log']].values ) + test['pred'] = clf.predict( test[['pred_lin','pred_log']].values ) + coefsdf = coefsdf.append(tmp) + df['err'] = (df.pred / df[target_col]).apply(lambda x: abs(x-1) ) + df['err'] = df[target_col] - df.pred + df.head() + # df[df['std_Attribute count_4']==1]['err'] + df['w_err'] = df.err * df.wt + # df[df['std_Attribute count_4']==1].sort_values('timestamp')[['err','w_err']].mean() + # df[(df['std_Attribute count_4']==1)].sort_values('timestamp')[['err','w_err']].mean() + # df[(df['std_Attribute count_4']==1) & (df.wt>=15)].sort_values('timestamp')[['err','w_err']].mean() + # df[(df['std_Attribute count_4']==1) & (df.wt>=15)].sort_values('timestamp')[['err','w_err']].sum() + # df[(df['std_Attribute count_4']==1) & (df.wt<15)].sort_values('timestamp')[['err','w_err']].mean() + # df[(df['std_Attribute count_4']==1) & (df.wt<15)].sort_values('timestamp')[['err','w_err']].sum() + # df[df['std_Attribute count_4']==1].sort_values('timestamp')[['err','price','pred','block_timestamp']].tail(20) + # df[(df['std_Attribute count_4']==1) & (df.wt>=1)].sort_values('timestamp')[['err','price','pred','block_timestamp','wt','w_err']].tail(50) + # df[df.wt >= 15].wt.sum() / df.wt.sum() + # df[df.wt < 15].wt.sum() + + recent_errs = [] + recent = df[df.timestamp >= a_week_ago] + for c in [ c for c in cur_std_pred_cols if len(df[c].unique()) == 2]: + a = recent[recent[c] == 1] + recent_errs += [[ c, len(a), a.err.mean(), a.err.sum(), a.err.sum() / recent.price.sum() ]] + recent_errs = pd.DataFrame(recent_errs, columns=['col','n','avg','tot','rat']).sort_values('tot') + recent_errs['abs_rat'] = abs(recent_errs.rat) + recent_errs.sort_values('rat') + correct = recent_errs[(recent_errs.abs_rat > 0.003) & (recent_errs.n >= 10)] + if len(correct): + mx = max(0.001, correct.abs_rat.max()) + correct['chg'] = (correct.abs_rat / mx).apply(lambda x: min(1, x) * .7 ) * correct.avg + correct['abs_chg'] = abs(correct.chg) + print(correct.sort_values('chg')) + for row in correct.iterrows(): + row = row[1] + c = row['col'] + test['pred'] = test.apply(lambda x: x['pred'] if x[c] == 0 else x['pred'] + row['chg'], 1 ) + + # print out some summary stats + df['err'] = df[target_col] - df.pred + df['q'] = (df.pred.rank() ** 1.5 * .2) / len(df) + df['q'] = df.q.apply(lambda x: int(round(x)) ) + df['pct_err'] = (df[target_col] / df.pred) - 1 + pe_mu = df.pct_err.mean() + pe_sd = df[ (df.pct_err > -.9) & (df.pct_err < 0.9) & (df.days_ago<=50) ].pct_err.std() + if pe_sd != pe_sd: + pe_sd = df[ (df.pct_err > -.9) & (df.pct_err < 0.9) ].pct_err.std() + df['pred_price'] = df.pred#.apply(lambda x: x*(1+pe_mu) ) + if collection == 'Levana Dragon Eggs': + df['pred_price'] = df.pred.apply(lambda x: x*1.01 ) + df['pred_sd'] = df.pred * pe_sd + # print(df[df.wt >= df.wt.median()].groupby('q')[['err','pred',target_col]].mean()) + # print(df.groupby('q')[['err','pred',target_col]].mean()) + # df.err.mean() + # df[df.weight >= 3.5].err.mean() + df[df.pred < 200].err.mean() + df['collection'] = collection + print('Avg err last 100: {}'.format(round(df.sort_values('block_timestamp').head(100).err.mean(), 2))) + # salesdf = salesdf.append( df.rename(columns={'collection_rank':'nft_rank'}).merge(s_df[s_df.sim == 0][['collection','token_id','block_timestamp','price']] )[[ 'collection','token_id','block_timestamp','price','pred','mn_20','nft_rank' ]].sort_values('block_timestamp', ascending=0) ) + salesdf = salesdf.append( df.merge(s_df[s_df.sim == 0][['collection','token_id','block_timestamp','price']] )[[ 'collection','token_id','block_timestamp','price','pred','mn_20','nft_rank' ]].sort_values('block_timestamp', ascending=0) ) + + ############################################################ + # Create Predictions for Each NFT in The Collection # + ############################################################ + # test = merge(num_metadata, cat_metadata, ['collection','token_id']) + # for c in num_features: + # test[c] = test[c].apply(lambda x: just_float(x) ) + # tail = df.sort_values('timestamp').tail(1) + # test.loc[ test.token_id == '903', 'nft_rank' ] = 18 + # for c in [ 'std_timestamp','mn_20','log_mn_20' ]: + # if c in tail.columns: + # test[c] = tail[c].values[0] + # test = standardize_df(test, pred_cols, df) + + # test['pred_lin'] = clf_lin.predict(test[lin_std_pred_cols].values) + # test['pred_lin'] = test.pred_lin.apply(lambda x: max(0, x) + l) + # test['pred_log'] = clf_log.predict(test[log_std_pred_cols].values) + # test['pred_log'] = test.pred_log.apply(lambda x: max(1, x)) * l + + # test['pred_price'] = test.pred.apply(lambda x: x if x < 400 else (x-400)**0.96 + 400 ) + def f(p): + c = CUR_FLOOR * 2.5 + if collection == 'Degen Apes': + return( p if p <= c else c+((p-c) ** 0.94) ) + return( p if p <= c else c+((p-c) ** 0.95) ) + test['pred_price'] = test.pred.apply(lambda x: f(x) ) + len(test[test.pred <= CUR_FLOOR * 1.01]) + len(test[test.pred <= CUR_FLOOR * 1.02]) + if not check_exclude: + test['pred_price'] = test.pred_price.apply(lambda x: (x*0.985) ) + if collection == 'BAYC': + test['pred_price'] = test.pred_price.apply(lambda x: (x*1.03) ) + if collection == 'Galactic Angels': + test['pred_price'] = test.pred_price.apply(lambda x: (x** 1.05) * 1.2 ) + + test['token_id'] = test.token_id.astype(int) + listings['token_id'] = listings.token_id.astype(int) + tmp = listings[listings.collection == collection][['token_id','price']].merge(test[['token_id','pred_price']]) + tmp['ratio'] = (tmp.pred_price / tmp.price) + tmp['is_deal'] = (tmp.ratio > 1).astype(int) + mx = tmp.ratio.max() + if mx < 1.15: + test['pred_price'] = test.pred_price * 1.15 / mx + + # make sure the lowest pred price is the floor + dff = test.pred_price.min() - CUR_FLOOR + if dff > 0: + test['pred_price'] = test.pred_price - dff + + test['pred_sd'] = test.pred_price * pe_sd + test = test.sort_values(['collection','token_id']) + test['rk'] = test.pred_price.rank(ascending=0, method='first') + test['collection'] = collection + if 'collection_rank' in test.columns and (not 'nft_rank' in test.columns or len(test[test.nft_rank.notnull()]) < len(test[test.collection_rank.notnull()])): + test['nft_rank'] = test.collection_rank + pred_price = pred_price.append( test[[ 'collection','token_id','nft_rank','rk','pred_price','pred_sd' ]].sort_values('pred_price') ).drop_duplicates(subset=['collection','token_id'], keep='last') + + + imp = [] + # a = [ 'matching' ] if collection == 'Solana Monkey Business' else [] + for c in list(dummies.columns): + md = test[test[c] == 1].pred_price.median() + md_0 = test.pred_price.quantile(0.475) + imp += [[ collection, c, md_0, md ]] + # imp = pd.DataFrame(imp, columns=['collection','feature_name','']) + imp = pd.DataFrame(imp, columns=['collection','col','col_md','md']).sort_values('md', ascending=0) + imp['pct_vs_baseline'] = ((imp.md / imp.col_md) - 1).apply(lambda x: max(0, x)) + imp['feature_name'] = imp.col.apply(lambda x: re.split('_', x)[0].title() ) + imp['feature_value'] = imp.col.apply(lambda x: re.split('_', x)[1] if '_' in x else None ) + sorted(imp.feature_name.unique()) + imp.loc[imp.col == 'Matching_No', 'pct_vs_baseline'] = 0 + imp[imp.feature_name == 'Attribute Count'] + # if 'matching' in a: + # imp.loc[imp.feature_name == 'Matching', 'feature_value'] = 'Yes' + # test[test.matching==1].to_csv('~/Downloads/tmp1.csv', index=False) + feature_values = feature_values.append(imp[['collection','feature_name','feature_value','pct_vs_baseline']]) + + cols = metadata.feature_name.unique() + cols = [ x for x in cols if not x in (ATT_EXCLUDE_COLS[collection] if collection in ATT_EXCLUDE_COLS.keys() else []) + ALL_NUMERIC_COLS ] + exclude = RARITY_EXCLUDE_COLS[collection] if collection in RARITY_EXCLUDE_COLS.keys() else [] + for c in cols: + cur = metadata[metadata.feature_name == c][['collection','token_id','feature_name','feature_value']] + l = len(cur.token_id.unique()) + if c in exclude: + cur['rarity'] = None + else: + g = cur.groupby('feature_value').token_id.count().reset_index() + g['rarity'] = g.token_id / l + cur = merge(cur, g[['feature_value','rarity']]) + attributes = attributes.append(cur) + # except: + # print('Error') - print('Price = {} * lin + {} * log'.format( round(clf.coef_[0], 2), round(clf.coef_[1], 2) )) - tmp = pd.DataFrame([[collection, clf.coef_[0], clf.coef_[1], CUR_FLOOR]], columns=['collection','lin_coef','log_coef','floor_price']) - if clf.coef_[0] < 0: - print('Only using log') - df['pred'] = df.pred_log - test['pred'] = test.pred_log - tmp['lin_coef'] = 0 - tmp['log_coef'] = 1 - elif clf.coef_[1] < 0: - print('Only using lin') - df['pred'] = df.pred_lin - test['pred'] = test.pred_lin - tmp['lin_coef'] = 1 - tmp['log_coef'] = 0 - else: - print('Only using BOTH!') - df['pred'] = clf.predict( df[['pred_lin','pred_log']].values ) - test['pred'] = clf.predict( test[['pred_lin','pred_log']].values ) - coefsdf = coefsdf.append(tmp) - df['err'] = (df.pred / df[target_col]).apply(lambda x: abs(x-1) ) - df['err'] = df[target_col] - df.pred - df.head() - # df[df['std_Attribute count_4']==1]['err'] - df['w_err'] = df.err * df.wt - # df[df['std_Attribute count_4']==1].sort_values('timestamp')[['err','w_err']].mean() - # df[(df['std_Attribute count_4']==1)].sort_values('timestamp')[['err','w_err']].mean() - # df[(df['std_Attribute count_4']==1) & (df.wt>=15)].sort_values('timestamp')[['err','w_err']].mean() - # df[(df['std_Attribute count_4']==1) & (df.wt>=15)].sort_values('timestamp')[['err','w_err']].sum() - # df[(df['std_Attribute count_4']==1) & (df.wt<15)].sort_values('timestamp')[['err','w_err']].mean() - # df[(df['std_Attribute count_4']==1) & (df.wt<15)].sort_values('timestamp')[['err','w_err']].sum() - # df[df['std_Attribute count_4']==1].sort_values('timestamp')[['err','price','pred','block_timestamp']].tail(20) - # df[(df['std_Attribute count_4']==1) & (df.wt>=1)].sort_values('timestamp')[['err','price','pred','block_timestamp','wt','w_err']].tail(50) - # df[df.wt >= 15].wt.sum() / df.wt.sum() - # df[df.wt < 15].wt.sum() + attributes['feature_name'] = attributes.feature_name.apply(lambda x: re.sub('_', ' ', x).title().strip() ) + attributes['feature_value'] = attributes.feature_value.apply(lambda x: str(x).strip() ) + sorted(attributes['feature_name'].unique()) + if len(feature_values): + feature_values['feature_name'] = feature_values.feature_name.apply(lambda x: re.sub('_', ' ', x).title() ) + # feature_values = pd.read_csv('./data/feature_values.csv') + feature_values = feature_values.merge(attributes[['collection','feature_name']].drop_duplicates()) + # n = feature_values[['collection', 'feature_name']].drop_duplicates().groupby(['collection']).feature_name.count().reset_index().rename(columns={'feature_name': 'n'}) + # feature_values = feature_values.merge(n) + # feature_values['pct_vs_baseline'] = feature_values.pct_vs_baseline / feature_values.n + # del feature_values['n'] + feature_values[ (feature_values.collection == 'Galactic Angels') ] + feature_values[ (feature_values.collection == 'Solana Monkey Business') & (feature_values.feature_name == 'Clothes') ] + feature_values[ (feature_values.collection == 'Solana Monkey Business') & (feature_values.feature_name == 'Clothes') & (feature_values.feature_value == 'Poncho') ] + feature_values[ (feature_values.collection == 'Okay Bears') & (feature_values.feature_name == 'Attribute Count')] + attributes[ (attributes.collection == 'Solana Monkey Business') & (attributes.feature_name == 'Clothes') & (attributes.feature_value == 'Poncho') & (attributes.token_id == '1') ] + attributes[ (attributes.collection == 'Solana Monkey Business') & (attributes.feature_name == 'Clothes') & (attributes.feature_value == 'Poncho') & (attributes.token_id == 1) ] - recent_errs = [] - recent = df[df.timestamp >= a_week_ago] - for c in [ c for c in cur_std_pred_cols if len(df[c].unique()) == 2]: - a = recent[recent[c] == 1] - recent_errs += [[ c, len(a), a.err.mean(), a.err.sum(), a.err.sum() / recent.price.sum() ]] - recent_errs = pd.DataFrame(recent_errs, columns=['col','n','avg','tot','rat']).sort_values('tot') - recent_errs['abs_rat'] = abs(recent_errs.rat) - recent_errs.sort_values('rat') - correct = recent_errs[(recent_errs.abs_rat > 0.003) & (recent_errs.n >= 10)] - if len(correct): - mx = max(0.001, correct.abs_rat.max()) - correct['chg'] = (correct.abs_rat / mx).apply(lambda x: min(1, x) * .7 ) * correct.avg - correct['abs_chg'] = abs(correct.chg) - print(correct.sort_values('chg')) - for row in correct.iterrows(): - row = row[1] - c = row['col'] - test['pred'] = test.apply(lambda x: x['pred'] if x[c] == 0 else x['pred'] + row['chg'], 1 ) + coefsdf.to_csv('./data/coefsdf.csv', index=False) + salesdf.to_csv('./data/model_sales.csv', index=False) + # salesdf[salesdf.collection] + salesdf['block_timestamp'] = salesdf.block_timestamp.apply(lambda x: str(x)[:19] ) + salesdf[salesdf.collection == 'BAYC'].sort_values('block_timestamp', ascending=0).head()[['token_id','block_timestamp','price']] + salesdf[salesdf.block_timestamp.isnull()] + salesdf.block_timestamp.max() + salesdf.groupby('collection').block_timestamp.max() + pred_price[pred_price.collection == 'SOLGods'].to_csv('~/Downloads/tmp1.csv', index=False) + # old = pd.read_csv('./data/pred_price.csv') + # old = old[old.collection == 'DeGods'] + # old['token_id'] = old.token_id.astype(str) + # old = pred_price.merge(old, on=['collection','token_id']) + # old['ratio'] = old.pred_price_x / old.pred_price_y + # old = old.sort_values('ratio') + # old.columns = [ 'collection', 'token_id', 'nft_rank', 'rk_new', 'pred_price_new', 'pred_sd_x', 'rank', 'rk_old', 'pred_price_old', 'pred_sd_y', 'ratio' ] + # # old.columns = [ 'collection', 'token_id', 'nft_rank', 'rk_new', 'pred_price_new', 'pred_sd_x', 'rank', 'rk_old', 'pred_price_old', 'pred_sd_y', 'clean_token_id', 'ratio' ] + # m = m_df[(m_df.collection.isin(pred_price.collection.unique())) & (-(m_df.feature_name.isin(['nft_rank','adj_nft_rank_0','adj_nft_rank_1','adj_nft_rank_2'])))] + # m_p = m.pivot(['collection','token_id'], ['feature_name'], ['feature_value']).reset_index() + # m_p.columns = [ 'collection','token_id' ] + sorted(m.feature_name.unique()) + # m_p.head() + # old = old.merge(m_p, on=['collection','token_id']) + # if len(old) and 'rank' in old.columns: + # # old = old[[ 'token_id', 'nft_rank', 'rk_old', 'rk_new', 'pred_price_old', 'pred_price_new', 'ratio' ] + [c for c in m_p.columns if not c in ['token_id','collection']]] + # old = old[[ 'token_id', 'nft_rank', 'rk_old', 'rk_new', 'pred_price_old', 'pred_price_new', 'ratio' ] + [c for c in m_p.columns if not c in ['token_id','collection','rank']]] + # old.to_csv('~/Downloads/tmp1.csv', index=False) + # pred_price.head() + # old[old.token_id == '4857'] + # old.head() + # old.tail() - # print out some summary stats - df['err'] = df[target_col] - df.pred - df['q'] = (df.pred.rank() ** 1.5 * .2) / len(df) - df['q'] = df.q.apply(lambda x: int(round(x)) ) - df['pct_err'] = (df[target_col] / df.pred) - 1 - pe_mu = df.pct_err.mean() - pe_sd = df[ (df.pct_err > -.9) & (df.pct_err < 0.9) & (df.days_ago<=50) ].pct_err.std() - if pe_sd != pe_sd: - pe_sd = df[ (df.pct_err > -.9) & (df.pct_err < 0.9) ].pct_err.std() - df['pred_price'] = df.pred#.apply(lambda x: x*(1+pe_mu) ) - if collection == 'Levana Dragon Eggs': - df['pred_price'] = df.pred.apply(lambda x: x*1.01 ) - df['pred_sd'] = df.pred * pe_sd - # print(df[df.wt >= df.wt.median()].groupby('q')[['err','pred',target_col]].mean()) - # print(df.groupby('q')[['err','pred',target_col]].mean()) - # df.err.mean() - # df[df.weight >= 3.5].err.mean() - df[df.pred < 200].err.mean() - df['collection'] = collection - print('Avg err last 100: {}'.format(round(df.sort_values('block_timestamp').head(100).err.mean(), 2))) - # salesdf = salesdf.append( df.rename(columns={'collection_rank':'nft_rank'}).merge(s_df[s_df.sim == 0][['collection','token_id','block_timestamp','price']] )[[ 'collection','token_id','block_timestamp','price','pred','mn_20','nft_rank' ]].sort_values('block_timestamp', ascending=0) ) - salesdf = salesdf.append( df.merge(s_df[s_df.sim == 0][['collection','token_id','block_timestamp','price']] )[[ 'collection','token_id','block_timestamp','price','pred','mn_20','nft_rank' ]].sort_values('block_timestamp', ascending=0) ) + # nft_rank = m_df[m_df.feature_name=='nft_rank'][['collection','token_id','feature_value']].rename(columns={'feature_value': 'nft_rank'}) + # nft_rank['token_id'] = nft_rank.token_id.astype(str) + # pred_price['token_id'] = pred_price.token_id.astype(str) + # pred_price = pred_price.merge(nft_rank, how='left', on=['collection','token_id']) + # pred_price = pred_price[pred_price.collection != 'LunaBulls'] + pred_price['collection'] = pred_price.collection.apply(lambda x: clean_name(x)) + pred_price = pred_price.drop_duplicates(subset=['collection','token_id'], keep='last') + pred_price.to_csv('./data/pred_price.csv', index=False) + # pred_price = pd.read_csv('./data/pred_price.csv') + pred_price.groupby('collection')[['pred_price']].min() + attributes.to_csv('./data/attributes.csv', index=False) + attributes = pd.read_csv('./data/attributes.csv') + attributes[attributes.rarity.isnull()] + feature_values.to_csv('./data/feature_values.csv', index=False) + feature_values[feature_values.collection == 'Galactic Angels'].pct_vs_baseline.unique() + feature_values[ (feature_values.collection == 'Galactic Angels') & (feature_values.feature_name == 'Background')].feature_value.unique() + attributes[attributes.collection == 'Galactic Angels'].head() - ############################################################ - # Create Predictions for Each NFT in The Collection # - ############################################################ - # test = merge(num_metadata, cat_metadata, ['collection','token_id']) - # for c in num_features: - # test[c] = test[c].apply(lambda x: just_float(x) ) - # tail = df.sort_values('timestamp').tail(1) - # test.loc[ test.token_id == '903', 'nft_rank' ] = 18 - # for c in [ 'std_timestamp','mn_20','log_mn_20' ]: - # if c in tail.columns: - # test[c] = tail[c].values[0] - # test = standardize_df(test, pred_cols, df) - - # test['pred_lin'] = clf_lin.predict(test[lin_std_pred_cols].values) - # test['pred_lin'] = test.pred_lin.apply(lambda x: max(0, x) + l) - # test['pred_log'] = clf_log.predict(test[log_std_pred_cols].values) - # test['pred_log'] = test.pred_log.apply(lambda x: max(1, x)) * l - - # test['pred_price'] = test.pred.apply(lambda x: x if x < 400 else (x-400)**0.96 + 400 ) - def f(p): - c = CUR_FLOOR * 2.5 - if collection == 'Degen Apes': - return( p if p <= c else c+((p-c) ** 0.94) ) - return( p if p <= c else c+((p-c) ** 0.95) ) - test['pred_price'] = test.pred.apply(lambda x: f(x) ) - len(test[test.pred <= CUR_FLOOR * 1.01]) - len(test[test.pred <= CUR_FLOOR * 1.02]) - if not check_exclude: - test['pred_price'] = test.pred_price.apply(lambda x: (x*0.985) ) - if collection == 'BAYC': - test['pred_price'] = test.pred_price.apply(lambda x: (x*1.03) ) - if collection == 'Galactic Angels': - test['pred_price'] = test.pred_price.apply(lambda x: (x** 1.05) * 1.2 ) - - tmp = listings[listings.collection == collection][['token_id','price']].merge(test[['token_id','pred_price']]) - tmp['ratio'] = (tmp.pred_price / tmp.price) - tmp['is_deal'] = (tmp.ratio > 1).astype(int) - mx = tmp.ratio.max() - if mx < 1.15: - test['pred_price'] = test.pred_price * 1.15 / mx - - # make sure the lowest pred price is the floor - dff = test.pred_price.min() - CUR_FLOOR - if dff > 0: - test['pred_price'] = test.pred_price - dff - - test['pred_sd'] = test.pred_price * pe_sd - test = test.sort_values(['collection','token_id']) - test['rk'] = test.pred_price.rank(ascending=0, method='first') - test['collection'] = collection - if 'collection_rank' in test.columns and (not 'nft_rank' in test.columns or len(test[test.nft_rank.notnull()]) < len(test[test.collection_rank.notnull()])): - test['nft_rank'] = test.collection_rank - pred_price = pred_price.append( test[[ 'collection','token_id','nft_rank','rk','pred_price','pred_sd' ]].sort_values('pred_price') ).drop_duplicates(subset=['collection','token_id'], keep='last') + # metadata = pd.read_csv('./data/metadata.csv') + # metadata['collection'] = metadata.collection.apply(lambda x: clean_name(x)) + # metadata['token_id'] = metadata.token_id.astype(str) + # metadata.head() + # nft_rank = pred_price[[ 'collection','token_id','nft_rank' ]].rename(columns={'nft_rank':'feature_value'}) + # nft_rank['feature_name'] = 'nft_rank' + # metadata = metadata[metadata.feature_name != 'nft_rank'] + # nft_rank = merge(nft_rank, metadata[['collection','chain']].fillna('Solana').drop_duplicates()) + # metadata = metadata.append(nft_rank) + # metadata.to_csv('./data/metadata.csv', index=False) - imp = [] - # a = [ 'matching' ] if collection == 'Solana Monkey Business' else [] - for c in list(dummies.columns): - md = test[test[c] == 1].pred_price.median() - md_0 = test.pred_price.quantile(0.475) - imp += [[ collection, c, md_0, md ]] - # imp = pd.DataFrame(imp, columns=['collection','feature_name','']) - imp = pd.DataFrame(imp, columns=['collection','col','col_md','md']).sort_values('md', ascending=0) - imp['pct_vs_baseline'] = ((imp.md / imp.col_md) - 1).apply(lambda x: max(0, x)) - imp['feature_name'] = imp.col.apply(lambda x: re.split('_', x)[0].title() ) - imp['feature_value'] = imp.col.apply(lambda x: re.split('_', x)[1] if '_' in x else None ) - sorted(imp.feature_name.unique()) - imp.loc[imp.col == 'Matching_No', 'pct_vs_baseline'] = 0 - imp[imp.feature_name == 'Attribute Count'] - # if 'matching' in a: - # imp.loc[imp.feature_name == 'Matching', 'feature_value'] = 'Yes' - # test[test.matching==1].to_csv('~/Downloads/tmp1.csv', index=False) - feature_values = feature_values.append(imp[['collection','feature_name','feature_value','pct_vs_baseline']]) + # feature_values.to_csv('./data/feature_values.csv', index=False) - cols = metadata.feature_name.unique() - cols = [ x for x in cols if not x in (ATT_EXCLUDE_COLS[collection] if collection in ATT_EXCLUDE_COLS.keys() else []) + ALL_NUMERIC_COLS ] - exclude = RARITY_EXCLUDE_COLS[collection] if collection in RARITY_EXCLUDE_COLS.keys() else [] - for c in cols: - cur = metadata[metadata.feature_name == c][['collection','token_id','feature_name','feature_value']] - l = len(cur.token_id.unique()) - if c in exclude: - cur['rarity'] = None - else: - g = cur.groupby('feature_value').token_id.count().reset_index() - g['rarity'] = g.token_id / l - cur = merge(cur, g[['feature_value','rarity']]) - attributes = attributes.append(cur) - - attributes['feature_name'] = attributes.feature_name.apply(lambda x: re.sub('_', ' ', x).title().strip() ) - attributes['feature_value'] = attributes.feature_value.apply(lambda x: str(x).strip() ) - sorted(attributes['feature_name'].unique()) - if len(feature_values): - feature_values['feature_name'] = feature_values.feature_name.apply(lambda x: re.sub('_', ' ', x).title() ) - # feature_values = pd.read_csv('./data/feature_values.csv') - feature_values = feature_values.merge(attributes[['collection','feature_name']].drop_duplicates()) - # n = feature_values[['collection', 'feature_name']].drop_duplicates().groupby(['collection']).feature_name.count().reset_index().rename(columns={'feature_name': 'n'}) - # feature_values = feature_values.merge(n) - # feature_values['pct_vs_baseline'] = feature_values.pct_vs_baseline / feature_values.n - # del feature_values['n'] - feature_values[ (feature_values.collection == 'Galactic Angels') ] - feature_values[ (feature_values.collection == 'Solana Monkey Business') & (feature_values.feature_name == 'Clothes') ] - feature_values[ (feature_values.collection == 'Solana Monkey Business') & (feature_values.feature_name == 'Clothes') & (feature_values.feature_value == 'Poncho') ] - feature_values[ (feature_values.collection == 'Okay Bears') & (feature_values.feature_name == 'Attribute Count')] - attributes[ (attributes.collection == 'Solana Monkey Business') & (attributes.feature_name == 'Clothes') & (attributes.feature_value == 'Poncho') & (attributes.token_id == '1') ] - attributes[ (attributes.collection == 'Solana Monkey Business') & (attributes.feature_name == 'Clothes') & (attributes.feature_value == 'Poncho') & (attributes.token_id == 1) ] + file_to_store = open('./objects/saved_params.pickle', 'wb') + pickle.dump(saved_params, file_to_store) + # saved_params.keys() - coefsdf.to_csv('./data/coefsdf.csv', index=False) - salesdf.to_csv('./data/model_sales.csv', index=False) - # salesdf[salesdf.collection] - salesdf['block_timestamp'] = salesdf.block_timestamp.apply(lambda x: str(x)[:19] ) - salesdf[salesdf.collection == 'BAYC'].sort_values('block_timestamp', ascending=0).head()[['token_id','block_timestamp','price']] - salesdf[salesdf.block_timestamp.isnull()] - salesdf.block_timestamp.max() - salesdf.groupby('collection').block_timestamp.max() - pred_price[pred_price.collection == 'SOLGods'].to_csv('~/Downloads/tmp1.csv', index=False) - # old = pd.read_csv('./data/pred_price.csv') - # old = old[old.collection == 'DeGods'] - # old['token_id'] = old.token_id.astype(str) - # old = pred_price.merge(old, on=['collection','token_id']) - # old['ratio'] = old.pred_price_x / old.pred_price_y - # old = old.sort_values('ratio') - # old.columns = [ 'collection', 'token_id', 'nft_rank', 'rk_new', 'pred_price_new', 'pred_sd_x', 'rank', 'rk_old', 'pred_price_old', 'pred_sd_y', 'ratio' ] - # # old.columns = [ 'collection', 'token_id', 'nft_rank', 'rk_new', 'pred_price_new', 'pred_sd_x', 'rank', 'rk_old', 'pred_price_old', 'pred_sd_y', 'clean_token_id', 'ratio' ] - # m = m_df[(m_df.collection.isin(pred_price.collection.unique())) & (-(m_df.feature_name.isin(['nft_rank','adj_nft_rank_0','adj_nft_rank_1','adj_nft_rank_2'])))] - # m_p = m.pivot(['collection','token_id'], ['feature_name'], ['feature_value']).reset_index() - # m_p.columns = [ 'collection','token_id' ] + sorted(m.feature_name.unique()) - # m_p.head() - # old = old.merge(m_p, on=['collection','token_id']) - # if len(old) and 'rank' in old.columns: - # # old = old[[ 'token_id', 'nft_rank', 'rk_old', 'rk_new', 'pred_price_old', 'pred_price_new', 'ratio' ] + [c for c in m_p.columns if not c in ['token_id','collection']]] - # old = old[[ 'token_id', 'nft_rank', 'rk_old', 'rk_new', 'pred_price_old', 'pred_price_new', 'ratio' ] + [c for c in m_p.columns if not c in ['token_id','collection','rank']]] - # old.to_csv('~/Downloads/tmp1.csv', index=False) - # pred_price.head() - # old[old.token_id == '4857'] - # old.head() - # old.tail() - - # nft_rank = m_df[m_df.feature_name=='nft_rank'][['collection','token_id','feature_value']].rename(columns={'feature_value': 'nft_rank'}) - # nft_rank['token_id'] = nft_rank.token_id.astype(str) - # pred_price['token_id'] = pred_price.token_id.astype(str) - # pred_price = pred_price.merge(nft_rank, how='left', on=['collection','token_id']) - # pred_price = pred_price[pred_price.collection != 'LunaBulls'] - pred_price['collection'] = pred_price.collection.apply(lambda x: clean_name(x)) - pred_price = pred_price.drop_duplicates(subset=['collection','token_id'], keep='last') - pred_price.to_csv('./data/pred_price.csv', index=False) - # pred_price = pd.read_csv('./data/pred_price.csv') - pred_price.groupby('collection')[['pred_price']].min() - attributes.to_csv('./data/attributes.csv', index=False) - attributes = pd.read_csv('./data/attributes.csv') - attributes[attributes.rarity.isnull()] - feature_values.to_csv('./data/feature_values.csv', index=False) - feature_values[feature_values.collection == 'Galactic Angels'].pct_vs_baseline.unique() - feature_values[ (feature_values.collection == 'Galactic Angels') & (feature_values.feature_name == 'Background')].feature_value.unique() - attributes[attributes.collection == 'Galactic Angels'].head() - - # metadata = pd.read_csv('./data/metadata.csv') - # metadata['collection'] = metadata.collection.apply(lambda x: clean_name(x)) - # metadata['token_id'] = metadata.token_id.astype(str) - # metadata.head() - # nft_rank = pred_price[[ 'collection','token_id','nft_rank' ]].rename(columns={'nft_rank':'feature_value'}) - # nft_rank['feature_name'] = 'nft_rank' - # metadata = metadata[metadata.feature_name != 'nft_rank'] - # nft_rank = merge(nft_rank, metadata[['collection','chain']].fillna('Solana').drop_duplicates()) - # metadata = metadata.append(nft_rank) - # metadata.to_csv('./data/metadata.csv', index=False) - - - # feature_values.to_csv('./data/feature_values.csv', index=False) - - file_to_store = open('./objects/saved_params.pickle', 'wb') - pickle.dump(saved_params, file_to_store) - - if True or check_exclude: - exclude = pd.read_csv('./data/exclude.csv') - salesdf['rat'] = salesdf.price / salesdf.pred - salesdf['dff'] = salesdf.price - salesdf.pred - salesdf['exclude_1'] = (((salesdf.dff >= 20) & (salesdf.rat > 4)) | ((salesdf.dff >= 40) & (salesdf.rat > 3)) | ((salesdf.dff >= 60) & (salesdf.rat > 2.5)) | ((salesdf.dff >= 80) & (salesdf.rat > 2.5))).astype(int) - salesdf['rat'] = salesdf.pred / salesdf.price - salesdf['dff'] = salesdf.pred - salesdf.price - salesdf['exclude_2'] = (((salesdf.dff >= 20) & (salesdf.rat > 4)) | ((salesdf.dff >= 40) & (salesdf.rat > 3)) | ((salesdf.dff >= 60) & (salesdf.rat > 2.5)) | ((salesdf.dff >= 80) & (salesdf.rat > 2.5))).astype(int) - salesdf['exclude'] = (salesdf.exclude_1 + salesdf.exclude_2).apply(lambda x: int(x>0)) - # print(salesdf.exclude_1.mean()) - # print(salesdf.exclude_2.mean()) - # print(salesdf.exclude.mean()) - salesdf[salesdf.token_id == '2239'][['collection','price','exclude']] - exclude = exclude.append(salesdf[salesdf.exclude == 1][[ 'collection','token_id','price','exclude' ]]) - # salesdf[salesdf.exclude == 1][[ 'collection','token_id','price','exclude' ]].to_csv('./data/exclude.csv', index=False) - exclude.to_csv('./data/exclude.csv', index=False) - # tokens[tokens.collection == 'Meerkat Millionaires'] - # tokens[tokens.collection == 'Cets on Creck'].sort_values('nft_rank', ascending=0) - # tokens[tokens.collection == 'Cets on Creck'] - # tokens = tokens.drop_duplicates(subset=['collection','token_id'], keep='last') - # tokens['chain'] = tokens.chain.fillna('Solana') - # tokens['clean_token_id'] = tokens.clean_token_id.fillna(tokens.token_id) - # tokens.to_csv('./data/tokens.csv', index=False) + if True or check_exclude: + exclude = pd.read_csv('./data/exclude.csv') + salesdf['rat'] = salesdf.price / salesdf.pred + salesdf['dff'] = salesdf.price - salesdf.pred + salesdf['exclude_1'] = (((salesdf.dff >= 20) & (salesdf.rat > 4)) | ((salesdf.dff >= 40) & (salesdf.rat > 3)) | ((salesdf.dff >= 60) & (salesdf.rat > 2.5)) | ((salesdf.dff >= 80) & (salesdf.rat > 2.5))).astype(int) + salesdf['rat'] = salesdf.pred / salesdf.price + salesdf['dff'] = salesdf.pred - salesdf.price + salesdf['exclude_2'] = (((salesdf.dff >= 20) & (salesdf.rat > 4)) | ((salesdf.dff >= 40) & (salesdf.rat > 3)) | ((salesdf.dff >= 60) & (salesdf.rat > 2.5)) | ((salesdf.dff >= 80) & (salesdf.rat > 2.5))).astype(int) + salesdf['exclude'] = (salesdf.exclude_1 + salesdf.exclude_2).apply(lambda x: int(x>0)) + # print(salesdf.exclude_1.mean()) + # print(salesdf.exclude_2.mean()) + # print(salesdf.exclude.mean()) + salesdf[salesdf.token_id == '2239'][['collection','price','exclude']] + exclude = exclude.append(salesdf[salesdf.exclude == 1][[ 'collection','token_id','price','exclude' ]]) + # salesdf[salesdf.exclude == 1][[ 'collection','token_id','price','exclude' ]].to_csv('./data/exclude.csv', index=False) + exclude.to_csv('./data/exclude.csv', index=False) + # tokens[tokens.collection == 'Meerkat Millionaires'] + # tokens[tokens.collection == 'Cets on Creck'].sort_values('nft_rank', ascending=0) + # tokens[tokens.collection == 'Cets on Creck'] + # tokens = tokens.drop_duplicates(subset=['collection','token_id'], keep='last') + # tokens['chain'] = tokens.chain.fillna('Solana') + # tokens['clean_token_id'] = tokens.clean_token_id.fillna(tokens.token_id) + # tokens.to_csv('./data/tokens.csv', index=False) # train_model(True, False) # train_model(False, False) diff --git a/utils.py b/utils.py index 3c7ec6e0..de4c66d1 100644 --- a/utils.py +++ b/utils.py @@ -1,5 +1,7 @@ +import os import re import pandas as pd +import snowflake.connector clean_names = { @@ -20,9 +22,26 @@ clean_names = { ,'mayc': 'MAYC' ,'solgods': 'SOLGods' ,'meerkatmillionairescc': 'Meerkat Millionaires' + ,'ggsg:galacticgeckos': 'Galactic Geckos' + ,'solstein': 'SolStein' # ,'stonedapecrew': 'Stoned Ape Crew' } +def get_ctx(): + usr = os.getenv('SNOWFLAKE_USR') + pwd = os.getenv('SNOWFLAKE_PWD') + # with open('snowflake.pwd', 'r') as f: + # pwd = f.readlines()[0].strip() + # with open('snowflake.usr', 'r') as f: + # usr = f.readlines()[0].strip() + + ctx = snowflake.connector.connect( + user=usr, + password=pwd, + account='vna27887.us-east-1' + ) + return(ctx) + def format_num(x): return('{:,}'.format(round(x, 2))) @@ -48,6 +67,7 @@ def clean_name(name): name = re.sub('-', ' ', name) name = re.sub(' On ', ' on ', name) name = re.sub('Defi ', 'DeFi ', name) + # name = re.sub(r'[^a-zA-Z0-9\s]', '', name) return(name) diff --git a/viz/.DS_Store b/viz/.DS_Store index 4809e0e62708acae8b9d0f7922b6a4d6dd06455b..a92823d539ef0efcf82a42aec6ed65853abe3570 100644 GIT binary patch delta 33 ocmZoMXffEZlu5+M&_YMS)YP 0 ] app['tmp'] = app.apply(lambda x: x['collection']+str(int(float(x['token_id'])))+x['sale_date'][:10], 1 ) if len(app[app.tx_id.isnull()]): @@ -115,7 +118,7 @@ def add_sales(query, usr, pwd, do_clean_token_id = False, data_folder = '/rstudi else: app = app.drop_duplicates(subset=['tx_id']) old = old[-old.collection.isin(app.collection.unique())] - old = old.append(app) + old = pd.concat([old, app]) old = old[[ 'collection','token_id','sale_date','price','tx_id' ]] @@ -142,22 +145,43 @@ def add_solana_sales(usr, pwd, data_folder = '/rstudio-data/'): , sales_amount AS price FROM solana.fact_nft_sales s JOIN solana.dim_nft_metadata m ON LOWER(m.mint) = LOWER(s.mint) - WHERE block_timestamp >= CURRENT_DATE - 14 + WHERE block_timestamp >= CURRENT_DATE - 7 AND m.project_name IN ( 'Astrals', 'Aurory', - 'Cets On Creck', + 'Blocksmith Labs', + 'Bohemia', + 'Bot Head', + 'Bubblegoose Ballers', + 'Cat Cartel', 'Catalina Whale Mixer', + 'Cets On Creck', + 'Citizens by Solsteads', + 'Communi3: Mad Scientists', + 'DeFi Pirates', 'DeFi Pirates', 'DeGods', 'Degen Apes', + 'Degen Dojo', + 'Doge Capital', + 'Famous Fox Federation', + 'GGSG: Galactic Geckos', + 'Just Ape.', + 'Looties', 'Meerkat Millionaires', + 'Monkey Baby Business', 'Okay Bears', 'Pesky Penguins', + 'Primates', + 'Quantum Traders', 'SOLGods', + 'SolStein', + 'Solana Monke Rejects', 'Solana Monkey Business', + 'Solanauts', 'Stoned Ape Crew', - 'Thugbirdz' + 'Thugbirdz', + 'Trippin Ape Tribe' ) ''' add_sales(query, usr, pwd, False, data_folder) @@ -172,10 +196,10 @@ def add_ethereum_sales(usr, pwd, data_folder = '/rstudio-data/'): , price , tx_id FROM ethereum.nft_events - WHERE project_name IN ( - 'BoredApeYachtClub' - , 'MutantApeYachtClub' - , 'BoredApeKennelClub' + WHERE LOWER(project_name) IN ( + 'boredapeyachtclub' + , 'mutantapeyachtclub' + , 'boredapekennelclub' ) AND price IS NOT NULL AND block_timestamp >= CURRENT_DATE - 14 diff --git a/viz/rsconnect/documents/update_nft_deal_score_data.RMD/science.flipsidecrypto.xyz/kellen/update_nft_deal_score_data4.dcf b/viz/rsconnect/documents/update_nft_deal_score_data.RMD/science.flipsidecrypto.xyz/kellen/update_nft_deal_score_data4.dcf index 4a4c8009..38f27995 100644 --- a/viz/rsconnect/documents/update_nft_deal_score_data.RMD/science.flipsidecrypto.xyz/kellen/update_nft_deal_score_data4.dcf +++ b/viz/rsconnect/documents/update_nft_deal_score_data.RMD/science.flipsidecrypto.xyz/kellen/update_nft_deal_score_data4.dcf @@ -5,10 +5,10 @@ account: kellen server: science.flipsidecrypto.xyz hostUrl: https://science.flipsidecrypto.xyz/__api__ appId: 114 -bundleId: 405 +bundleId: 434 url: https://science.flipsidecrypto.xyz/content/67141ada-46fd-4750-a690-0be248c461f3/ -when: 1654490491.17543 -lastSyncTime: 1654490491.17545 +when: 1656289651.71961 +lastSyncTime: 1656289651.71962 asMultiple: FALSE asStatic: FALSE ignoredFiles: scrape_terra_nfts.py|add_sales.py|nft_deal_score_data.RData|nft_deal_score_listings_data.RData|nft_deal_score_sales_data.RData|nft_deal_score_sales.csv|nft_deal_score_listings.csv diff --git a/viz/rsconnect/science.flipsidecrypto.xyz/kellen/nft-deal-score2.dcf b/viz/rsconnect/science.flipsidecrypto.xyz/kellen/nft-deal-score2.dcf index b5a4bea0..e6796e29 100644 --- a/viz/rsconnect/science.flipsidecrypto.xyz/kellen/nft-deal-score2.dcf +++ b/viz/rsconnect/science.flipsidecrypto.xyz/kellen/nft-deal-score2.dcf @@ -5,10 +5,10 @@ account: kellen server: science.flipsidecrypto.xyz hostUrl: https://science.flipsidecrypto.xyz/__api__ appId: 93 -bundleId: 410 +bundleId: 435 url: https://science.flipsidecrypto.xyz/nft-deal-score/ -when: 1654526436.70554 -lastSyncTime: 1654526436.70555 +when: 1656375887.36944 +lastSyncTime: 1656375887.36945 asMultiple: FALSE asStatic: FALSE ignoredFiles: add_sales.py|data (2).Rdata|data copy 2.Rdata|data copy.Rdata|data.Rdata|exploration.R|loan_score_model.py|nft_deal_score_data.RData|nft_deal_score_listings_data.RData|nft_deal_score_listings.csv|nft_deal_score_sales_data.RData|nft_deal_score_sales.csv|nft_deal_score_tokens.csv|scrape_eth_nfts.py|scrape_terra_nfts.py|update_data.R|update_nft_deal_score_data.RMD|upload_data.R|utils.py diff --git a/viz/rsconnect/science.flipsidecrypto.xyz/kellen/update_nft_deal_score_data4.dcf b/viz/rsconnect/science.flipsidecrypto.xyz/kellen/update_nft_deal_score_data4.dcf new file mode 100644 index 00000000..8d877b94 --- /dev/null +++ b/viz/rsconnect/science.flipsidecrypto.xyz/kellen/update_nft_deal_score_data4.dcf @@ -0,0 +1,14 @@ +name: update_nft_deal_score_data4 +title: +username: kellen +account: kellen +server: science.flipsidecrypto.xyz +hostUrl: https://science.flipsidecrypto.xyz/__api__ +appId: 114 +bundleId: 430 +url: https://science.flipsidecrypto.xyz/content/67141ada-46fd-4750-a690-0be248c461f3/ +when: 1656200355.03703 +lastSyncTime: 1656200355.03704 +asMultiple: FALSE +asStatic: FALSE +ignoredFiles: add_sales.py|data (2).Rdata|data copy 2.Rdata|data copy.Rdata|data.Rdata|exploration.R|loan_score_model.py|nft_deal_score_data.RData|nft_deal_score_listings_data.RData|nft_deal_score_listings.csv|nft_deal_score_sales_data.RData|nft_deal_score_sales.csv|nft_deal_score_tokens.csv|scrape_eth_nfts.py|scrape_terra_nfts.py|update_data.R|update_nft_deal_score_data.RMD|upload_data.R|utils.py diff --git a/viz/scrape_terra_nfts.py b/viz/scrape_terra_nfts.py index 6723ff50..e71962bb 100644 --- a/viz/scrape_terra_nfts.py +++ b/viz/scrape_terra_nfts.py @@ -47,6 +47,7 @@ def clean_name(name): name = re.sub('-', ' ', name) name = re.sub(' On ', ' on ', name) name = re.sub('Defi ', 'DeFi ', name) + # name = re.sub(r'[^a-zA-Z0-9\s]', '', name) return(name) def scrape_randomearth(data_folder = '/rstudio-data/'): diff --git a/viz/server.R b/viz/server.R index 23879461..05818779 100644 --- a/viz/server.R +++ b/viz/server.R @@ -3,8 +3,10 @@ server <- function(input, output, session) { user <- Sys.info()[['user']] # options(warn=-1) + isRstudio <- user != 'kellenblumberg' + # isRstudio <- TRUE base_dir <- ifelse( - user == 'rstudio-connect' + isRstudio , '/rstudio-data/' , ifelse(user == 'fcaster' , '/srv/shiny-server/nft-deal-score/' @@ -472,7 +474,7 @@ server <- function(input, output, session) { selectInput( inputId = 'collectionname' , label = NULL - , selected = 'Catalina Whale Mixer' + , selected = 'Famous Fox Federation' , choices = choices , width = "100%" ) @@ -561,7 +563,7 @@ updateSelectizeInput(session, 'tokenid', choices = choices, server = TRUE) selected %in% c('Cets on Creck') , strsplit(selected, ' |s ')[[1]][1] , ifelse( - selected %in% c('Stoned Ape Crew', 'Catalina Whale Mixer') + selected %in% c('Stoned Ape Crew', 'Catalina Whale Mixer','Famous Fox Federation') , paste(strsplit(selected, ' ')[[1]][1], strsplit(selected, ' ')[[1]][2], sep = ' ') , substr(selected, 1, nchar(selected) - 1) ) diff --git a/viz/ui.R b/viz/ui.R index 4f3e7b05..e0d99cb3 100644 --- a/viz/ui.R +++ b/viz/ui.R @@ -30,9 +30,9 @@ fluidPage( class="hero" , fluidRow( class = "header-images", - column(4, uiOutput("solanaimg")), - column(4, uiOutput("terraimg")), - column(4, uiOutput("ethereumimg")) + column(6, uiOutput("solanaimg")), + # column(4, uiOutput("terraimg")), + column(6, uiOutput("ethereumimg")) ) , h1( class="header", diff --git a/viz/update_data.R b/viz/update_data.R index f48a648d..49b1394f 100644 --- a/viz/update_data.R +++ b/viz/update_data.R @@ -3,143 +3,75 @@ library(reticulate) library(httr) library(jsonlite) -user <- Sys.info()[['user']] -isRstudio <- user == 'rstudio-connect' -# nft_deal_score_listings_data.RData +.topic = 'prod-nft-metadata-uploads' +.key = 'solana-nft-metadata' +.url = 'https://kafka-rest-proxy.flipside.systems' + +user <- Sys.info()[['user']] +isRstudio <- user %in% c('rstudio-connect','data-science') + base_dir <- ifelse( - user == 'rstudio-connect' + isRstudio , '/rstudio-data/' - , ifelse(user == 'fcaster' - , '/srv/shiny-server/nft-deal-score/' - , '~/git/nft-deal-score/viz/' - ) + , '~/git/nft-deal-score/viz/' ) -# base_dir <- '/srv/shiny-server/nft-deal-score/' -listings_file <- paste0(base_dir,'nft_deal_score_listings_data.RData') -load(listings_file) if(isRstudio) { source('/home/data-science/data_science/util/util_functions.R') - source_python('/home/data-science/data_science/nft-deal-score/scrape_terra_nfts.py') + source_python('/home/data-science/data_science/viz/nft-deal-score/upload_solana_nft_labels.py') } else { source('~/data_science/util/util_functions.R') - source_python(paste0(base_dir, 'scrape_terra_nfts.py')) + source_python(paste0(base_dir, 'upload_solana_nft_labels.py')) } -# py_install('pandas', pip = TRUE) -# py_install('cloudscraper', pip = TRUE) -# py_install('snowflake-connector-python', pip = TRUE) -# cloudscraper <- import('cloudscraper') -base_dir <- ifelse( - user == 'rstudio-connect' - , '/rstudio-data/' - , ifelse(user == 'fcaster' - , '/srv/shiny-server/nft-deal-score/' - , '~/git/nft-deal-score/viz/' - ) -) -source_python(paste0(base_dir, 'scrape_terra_nfts.py')) -source_python(paste0(base_dir, 'add_sales.py')) - -query <- ' - SELECT DISTINCT project_name AS collection - , mint AS tokenMint - , token_id - FROM solana.dim_nft_metadata -' -mints <- QuerySnowflake(query) -colnames(mints) <- c('collection','tokenMint','token_id') - -# pull terra listings -terra_listings <- scrape_randomearth(base_dir) -head(terra_listings) -unique(terra_listings$collection) +######################### +# Load NFT Data # +######################### +mints_from_me() +pull_from_metaboss() +how_rare_is_api() +compile() -get_me_url <- function(collection, offset) { - return(paste0('https://api-mainnet.magiceden.dev/v2/collections/',collection,'/listings?offset=',offset,'&limit=20')) -} -get_smb_url <- function(page) { - return(paste0('https://market.solanamonkey.business/api/items?limit=40&page=',page)) -} -solana_listings <- data.table() - -solana_collections <- c( - 'okay_bears','the_catalina_whale_mixer','meerkat_millionaires_country_club','solgods','cets_on_creck','stoned_ape_crew','degods','aurory','thugbirdz','solana_monkey_business','degenerate_ape_academy','pesky_penguins' -) -for(collection in solana_collections) { - print(paste0('Working on ', collection, '...')) - has_more <- TRUE - offset <- 0 - while(has_more) { - Sys.sleep(1) - print(paste0('Offset #', offset)) - url <- get_me_url(collection, offset) - response <- GET(url) - content <- rawToChar(response$content) - content <- fromJSON(content) - if( typeof(content) == 'list' ) { - content <- rbindlist(content, fill=T) - } - has_more <- nrow(content) >= 20 - if(nrow(content) > 0 && length(content) > 0) { - df <- merge(content, mints, by=c('tokenMint')) %>% as.data.table() - df <- df[, list(collection, token_id, price)] - offset <- offset + 20 - solana_listings <- rbind(solana_listings, df) - } else { - has_more <- FALSE - } +############################### +# Upload NFT Metadata # +############################### +files <- list.files(paste0(base_dir, 'nft_labels/metadata/results/')) +it <- 0 +for(f in files) { + print(f) + results <- read.csv(paste0(base_dir,'/nft_labels/metadata/results/',f)) + for(r in results$results) { + it <- it + 1 + print(paste0('#',it)) + out <- tryCatch( + { + # s <- readChar(fileName, file.info(fileName)$size) + s <- r + .body <- paste0( + '{"records": [{"key": "',.key,'","value":',s,'}]}', + collapse = "" + ) + r <- httr::POST(url = paste(.url,"topics",.topic,sep = "/"), + add_headers('Content-Type' = "application/vnd.kafka.json.v2+json", + 'Accept' = "application/vnd.kafka.v2+json, application/vnd.kafka+json, application/json"), + body = .body) + print(r) + }, + error=function(cond) { + print(cond) + return(NA) + }, + warning=function(cond) { + print(cond) + return(NULL) + }, + finally={ + } + ) } } -for(collection in c('Solana Monkey Business')) { - print(paste0('Working on ', collection, '...')) - has_more <- TRUE - page <- 1 - while(has_more) { - Sys.sleep(1) - print(paste0('Page #', page)) - url <- get_smb_url(page) - response <- GET(url) - content <- rawToChar(response$content) - content <- fromJSON(content) - # content <- rbindlist(content, fill=T) - content <- content %>% as.data.table() - has_more <- nrow(content) > 0 && 'price' %in% colnames(content) - if(has_more) { - content <- content[, list(mint, price)] - content <- unique(content) - content$price <- as.numeric(content$price) / (10^9) - has_more <- nrow(content) >= 40 - colnames(content)[1] <- 'tokenMint' - df <- merge(content, mints, by=c('tokenMint')) %>% as.data.table() - df <- df[, list(collection, token_id, price)] - page <- page + 1 - solana_listings <- rbind(solana_listings, df) - } - } -} - -head(solana_listings) -head(terra_listings) -new_listings <- rbind(solana_listings, terra_listings) -new_listings <- unique(new_listings) - -# listings <- read.csv('./data/listings.csv') %>% as.data.table() -rem <- unique(new_listings$collection) -rem -listings <- listings[ !(collection %in% eval(rem)), ] -listings <- listings[, list(collection, token_id, price)] -listings <- rbind(listings, new_listings) -listings <- listings[order(collection, price)] -listings[, token_id := as.integer(token_id)] - -save( - listings - , file = listings_file -) - diff --git a/viz/update_nft_deal_score_data.RMD b/viz/update_nft_deal_score_data.RMD index 32522e9b..2707a5d0 100644 --- a/viz/update_nft_deal_score_data.RMD +++ b/viz/update_nft_deal_score_data.RMD @@ -51,16 +51,19 @@ library(reticulate) # py_install('cloudscraper', pip = TRUE) # r reticulate python ModuleNotFoundError # print('54') -use_python('/opt/python/3.10.4/bin/python') -py_install('pandas', pip = TRUE) -py_install('snowflake-connector-python', pip = TRUE) SD_MULT = 3 SD_SCALE = 1.95 user <- Sys.info()[['user']] -isRstudio <- user == 'rstudio-connect' +# isRstudio <- user == 'rstudio-connect' +isRstudio <- user != 'kellenblumberg' # isRstudio <- TRUE +if (isRstudio) { + use_python('/opt/python/3.10.4/bin/python') + py_install('pandas', pip = TRUE) + py_install('snowflake-connector-python', pip = TRUE) +} base_dir <- ifelse( isRstudio @@ -109,6 +112,7 @@ add_ethereum_sales(usr, pwd, base_dir) # read sales data from nft_deal_score_sales.csv raw_sales <- read.csv(paste0(base_dir,'nft_deal_score_sales.csv')) %>% as.data.table() raw_sales <- raw_sales[order(collection, sale_date, price)] +unique(raw_sales$collection) # calculate the floor price raw_sales <- raw_sales %>% @@ -149,7 +153,7 @@ query <- ' ' mints <- QuerySnowflake(query) colnames(mints) <- c('collection','tokenMint','token_id') -mints[ collection == 'Cets On Creck', collection := 'Cets on Creck'] +# mints[ collection == 'Cets On Creck', collection := 'Cets on Creck'] # pull terra listings # terra_listings <- scrape_randomearth(base_dir) @@ -168,10 +172,75 @@ get_smb_url <- function(page) { solana_listings <- data.table() solana_collections <- c( - 'okay_bears','the_catalina_whale_mixer','meerkat_millionaires_country_club','solgods','cets_on_creck','stoned_ape_crew','degods','aurory','thugbirdz','solana_monkey_business','degenerate_ape_academy','pesky_penguins' + 'famous_fox_federation' ) solana_collections <- c( - 'okay_bears','the_catalina_whale_mixer','meerkat_millionaires_country_club','solgods','cets_on_creck','stoned_ape_crew','degods','aurory','thugbirdz','solana_monkey_business','degenerate_ape_academy','pesky_penguins' + 'blocksmith_labs' + , 'dazedducks_metagalactic_club' + , 'degenerate_trash_pandas' + , 'famous_fox_federation' + , 'generous_robots_dao' + , 'ghostface' + , 'ghostface' + , 'ghostface_gen_2' + , 'portals' + , 'smokeheads' + , 'theorcs' +) + +solana_collections <- c( + + # 'blocksmith_labs' + # , 'dazedducks_metagalactic_club' + # , 'degenerate_trash_pandas' + 'famous_fox_federation', + # , 'generous_robots_dao' + # , 'ghostface' + # , 'ghostface_gen_2' + # , 'portals' + # , 'smokeheads' + # , 'theorcs', + # 'astrals', + 'aurory', + # 'bohemia_', + # 'bothead', + 'bubblegoose_ballers', + # 'cat_cartel', + 'cets_on_creck', + # 'citizens_by_solsteads', + # 'communi3', + # 'defi_pirates', + # 'degendojonft', + 'degenerate_ape_academy', + # 'degenerate_ape_kindergarten', + 'degods', + # 'doge_capital', + # 'galactic_gecko_space_garage', + # 'justape', + # 'looties', + # 'marinadechefs', + 'meerkat_millionaires_country_club', + # 'monkey_baby_business', + 'okay_bears', + 'pesky_penguins', + 'portals', + 'primates', + # 'psykerhideouts', + # 'quantum_traders', + # 'solana_monke_rejects', + 'solana_monkey_business', + # 'solanauts', + 'solgods', + # 'solstein', + 'stoned_ape_crew', + # 'taiyo_infants_incubators', + 'the_catalina_whale_mixer', + # 'the_remnants_', + # 'the_tower', + # 'the_vaultx_dao', + 'thugbirdz' + # 'trippin_ape_tribe', + # 'visionary_studios' ) # headers = c( # 'Authorization': 'Bearer 9c39e05c-db3c-4f3f-ac48-84099111b813' @@ -244,8 +313,6 @@ for(collection in solana_collections) { } } -solana_listings[order(token_id)] - for(collection in c('Solana Monkey Business')) { print(paste0('Working on ', collection, '...')) has_more <- TRUE @@ -292,6 +359,9 @@ listings <- listings[ !(collection %in% c('LunaBulls','Galactic Punks','Galactic listings <- listings[!is.na(price)] listings <- listings %>% as.data.table() +sort(unique(listings$collection)) +# write.csv(unique(listings[, collection]), '~/Downloads/tmp.csv', row.names=F) + floors <- listings %>% group_by(collection) %>% summarize(cur_floor = min(price)) %>% @@ -312,7 +382,7 @@ get_fmp <- function(data, coefsdf, pred_price) { fmp[, fair_market_price := pred_price + abs_chg + (pct_chg * pred_price / floor_price) ] } -if(TRUE) { +if(FALSE) { coefsdf[, tot := lin_coef + log_coef ] coefsdf[, lin_coef := lin_coef / tot] coefsdf[, log_coef := log_coef / tot] @@ -434,7 +504,7 @@ if(TRUE) { tmp <- tmp[order(-pts)] content <- tmp[ (price < 0.9 * fair_market_price) , head(.SD, 2), by = collection] - content <- content[order(-pts)] + content <- head(content[order(-pts)], 15) # content <- paste(c(header, content$label, collapse='\n')) content <- paste(c(header, content$label), collapse='\n') @@ -459,15 +529,17 @@ if(TRUE) { colnames(fmp)[3] <- 'rarity_rank' colnames(fmp)[4] <- 'deal_score_rank' - for( cur_collection in unique(fmp$collection)) { - print(paste0('Working on ',cur_collection, '...')) - data <- fmp[collection == eval(cur_collection)] - KafkaGeneric( - .topic = 'prod-data-science-uploads' - , .url = 'https://kafka-rest-proxy.flipside.systems' - , .project = paste0('nft-deal-score-rankings-', cur_collection) - , .data = data - ) + if (FALSE) { + for( cur_collection in unique(fmp$collection)) { + print(paste0('Working on ',cur_collection, '...')) + data <- fmp[collection == eval(cur_collection)] + KafkaGeneric( + .topic = 'prod-data-science-uploads' + , .url = 'https://kafka-rest-proxy.flipside.systems' + , .project = paste0('nft-deal-score-rankings-', cur_collection) + , .data = data + ) + } } } @@ -480,6 +552,7 @@ KafkaGeneric( , .data = data ) +sort(unique(listings$collection)) save( listings diff --git a/viz/update_nft_labels.R b/viz/update_nft_labels.R new file mode 100644 index 00000000..f6b624c2 --- /dev/null +++ b/viz/update_nft_labels.R @@ -0,0 +1,566 @@ +--- +title: "Update NFT Deal Score Data" +author: "Kellen" +date: "2022-04-20" +output: html_document +--- + +```{r setup, include=FALSE} +knitr::opts_chunk$set(echo = TRUE) +``` + +## Run Updates + +Hello I am running this at `r Sys.time()` + +```{r update} +#include all required libraries here +#EVEN IF YOU SOURCE util_functions.R +#YOU HAVE TO PUT THE LIBRARIES HERE I KNOW SORRY +#BUT HERE THEY ALL ARE TO SAVE YOU TIME +# install.packages('RCurl') +library(RCurl) +library(fasttime) +library(gridExtra) +library(ggplot2) +library(data.table) +library(reshape2) +library(dplyr) +library(dbplyr) +library(RJSONIO) +library(magrittr) +library(RJSONIO) +library(xts) +library(quantmod) +library(fTrading) +library(curl) +library(stringr) +library(aws.s3) +library(RPostgres) +library(odbc) +library(httr) +library(jsonlite) +library(reticulate) + +#NOW COPY EVERYTHING ELSE FROM YOUR CURRENT +#update_data.R FILE HERE ----------> +# virtualenv_create('pyvenv') +# use_virtualenv('pyvenv') +# virtualenv_install('pyvenv', 'pandas') +# virtualenv_install('pyvenv', 'pandas') +# py_install('cloudscraper', pip = TRUE) +# r reticulate python ModuleNotFoundError +# print('54') + +SD_MULT = 3 +SD_SCALE = 1.95 + +user <- Sys.info()[['user']] +# isRstudio <- user == 'rstudio-connect' +isRstudio <- user != 'kellenblumberg' +# isRstudio <- TRUE +if (isRstudio) { + use_python('/opt/python/3.10.4/bin/python') + py_install('pandas', pip = TRUE) + py_install('snowflake-connector-python', pip = TRUE) +} + +base_dir <- ifelse( + isRstudio + , '/rstudio-data/' + , ifelse(user == 'fcaster' + , '/srv/shiny-server/nft-deal-score/' + , '~/git/nft-deal-score/viz/' + ) +) + +if(isRstudio) { + source('/home/data-science/data_science/util/util_functions.R') + source('/home/data-science/data_science/util/kafka_utils.R') + source_python('/home/data-science/data_science/viz/nft-deal-score/add_sales.py') + source_python('~/upload_solana_nft_labels.py') +} else { + source('~/data_science/util/util_functions.R') + source('~/data_science/util/kafka_utils.R') + source_python(paste0(base_dir, 'scrape_terra_nfts.py')) + source_python(paste0(base_dir, 'add_sales.py')) +} + +usr <- readLines(file.path(base.path,"data_science/util/snowflake.usr")) +pwd <- readLines(file.path(base.path,"data_science/util/snowflake.pwd")) + +load(paste0(base_dir,'nft_deal_score_data.RData')) + +listings_file <- paste0(base_dir,'nft_deal_score_listings_data.RData') +sales_file <- paste0(base_dir,'nft_deal_score_sales_data.RData') +load(listings_file) + + +coefsdf[, tot := lin_coef + log_coef ] +coefsdf[, lin_coef := lin_coef / tot] +coefsdf[, log_coef := log_coef / tot] +sum(coefsdf$log_coef) + sum(coefsdf$lin_coef) + +# write sales data to nft_deal_score_sales.csv +add_solana_sales(usr, pwd, base_dir) +add_ethereum_sales(usr, pwd, base_dir) +# add_terra_sales(usr, pwd, base_dir) + +# read sales data from nft_deal_score_sales.csv +raw_sales <- read.csv(paste0(base_dir,'nft_deal_score_sales.csv')) %>% as.data.table() +raw_sales <- raw_sales[order(collection, sale_date, price)] +unique(raw_sales$collection) + +# calculate the floor price +raw_sales <- raw_sales %>% + group_by(collection) %>% + mutate(mn_20=lag(price, 1)) %>% + as.data.table() + +raw_sales <- raw_sales %>% + group_by(collection) %>% + mutate(rolling_floor=rollapply(mn_20, width = 20, FUN = "quantile", p = .0575, na.pad = TRUE, align = 'right')) %>% + as.data.table() + +raw_sales[, rolling_floor := nafill(rolling_floor, type = "nocb")] + + +# calculate the fair market price +tmp <- merge( raw_sales[, list(collection, token_id, sale_date, price, tx_id, rolling_floor)], coefsdf, by=c('collection') ) +tmp <- merge( tmp, pred_price, by=c('collection','token_id') ) +tmp[, abs_chg := (rolling_floor - floor_price) * lin_coef ] +tmp[, pct_chg := (rolling_floor - floor_price) * log_coef ] +tmp[, fair_market_price := pred_price + abs_chg + (pct_chg * pred_price / floor_price) ] + +# save to an .RData file +sales <- tmp[, list(collection, token_id, sale_date, price, nft_rank, fair_market_price, rolling_floor)] +colnames(sales) <- c('collection', 'token_id', 'block_timestamp', 'price', 'nft_rank', 'pred', 'mn_20') +save( + sales + , file = sales_file +) + + +# load the mints +query <- ' + SELECT DISTINCT project_name AS collection + , mint AS tokenMint + , token_id + FROM solana.dim_nft_metadata +' +mints <- QuerySnowflake(query) +colnames(mints) <- c('collection','tokenMint','token_id') +# mints[ collection == 'Cets On Creck', collection := 'Cets on Creck'] + +# pull terra listings +# terra_listings <- scrape_randomearth(base_dir) +# head(terra_listings) +# unique(terra_listings$collection) + + +# 9c39e05c-db3c-4f3f-ac48-84099111b813 +get_me_url <- function(collection, offset) { + return(paste0('https://api-mainnet.magiceden.dev/v2/collections/',collection,'/listings?offset=',offset,'&limit=20')) +} +get_smb_url <- function(page) { + return(paste0('https://market.solanamonkey.business/api/items?limit=40&page=',page)) +} + +solana_listings <- data.table() + +solana_collections <- c( + 'famous_fox_federation' +) +solana_collections <- c( + 'blocksmith_labs' + , 'dazedducks_metagalactic_club' + , 'degenerate_trash_pandas' + , 'famous_fox_federation' + , 'generous_robots_dao' + , 'ghostface' + , 'ghostface' + , 'ghostface_gen_2' + , 'portals' + , 'smokeheads' + , 'theorcs' +) + +solana_collections <- c( + + # 'blocksmith_labs' + # , 'dazedducks_metagalactic_club' + # , 'degenerate_trash_pandas' + 'famous_fox_federation', + # , 'generous_robots_dao' + # , 'ghostface' + # , 'ghostface_gen_2' + # , 'portals' + # , 'smokeheads' + # , 'theorcs', + # 'astrals', + 'aurory', + # 'bohemia_', + # 'bothead', + 'bubblegoose_ballers', + # 'cat_cartel', + 'cets_on_creck', + # 'citizens_by_solsteads', + # 'communi3', + # 'defi_pirates', + # 'degendojonft', + 'degenerate_ape_academy', + # 'degenerate_ape_kindergarten', + 'degods', + # 'doge_capital', + # 'galactic_gecko_space_garage', + # 'justape', + # 'looties', + # 'marinadechefs', + 'meerkat_millionaires_country_club', + # 'monkey_baby_business', + 'okay_bears', + 'pesky_penguins', + 'portals', + 'primates', + # 'psykerhideouts', + # 'quantum_traders', + # 'solana_monke_rejects', + 'solana_monkey_business', + # 'solanauts', + 'solgods', + # 'solstein', + 'stoned_ape_crew', + # 'taiyo_infants_incubators', + 'the_catalina_whale_mixer', + # 'the_remnants_', + # 'the_tower', + # 'the_vaultx_dao', + 'thugbirdz' + # 'trippin_ape_tribe', + # 'visionary_studios' +) +# headers = c( +# 'Authorization': 'Bearer 9c39e05c-db3c-4f3f-ac48-84099111b813' +# ) +for(collection in solana_collections) { + print(paste0('Working on ', collection, '...')) + has_more <- TRUE + has_err <- FALSE + offset <- 0 + while(has_more) { + Sys.sleep(1) + out <- tryCatch( + { + print(paste0('Offset #', offset)) + url <- get_me_url(collection, offset) + response <- GET( + url = url + # , add_headers(.headers = c('Authorization'= 'Bearer 9c39e05c-db3c-4f3f-ac48-84099111b813')) + , add_headers('Authorization'= 'Bearer 9c39e05c-db3c-4f3f-ac48-84099111b813') + ) + # r <- content(response, as = 'parsed') + content <- rawToChar(response$content) + content <- fromJSON(content) + if( !is.data.frame(content) ) { + content <- rbindlist(content, fill=T) + } + has_more <- nrow(content) > 0 + if(nrow(content) > 0 && length(content) > 0) { + # content <- data.table(content) + df <- merge(content, mints, by=c('tokenMint')) %>% as.data.table() + # if(nrow(df) > 0) { + # print(min(df$price)) + # } + df <- df[, list(collection, token_id, price)] + solana_listings <- rbind(solana_listings, df) + } else { + has_more <- FALSE + } + offset <- offset + 20 + has_err <- FALSE + }, + error=function(cond) { + print(paste0('Error: ', cond)) + return(TRUE) + # has_more <- FALSE + # if(has_err) { + # has_err <- FALSE + # has_more <- FALSE + # return(TRUE) + # } else { + # Sys.sleep(15) + # has_err <- TRUE + # return(FALSE) + # } + # return(TRUE) + }, + warning=function(cond) { + print(paste0('Warning: ', cond)) + return(TRUE) + }, + finally={ + # return(TRUE) + # print(paste0('Finally')) + } + ) + if(out) { + offset <- offset + 20 + # has_more <- FALSE + } + } +} + +for(collection in c('Solana Monkey Business')) { + print(paste0('Working on ', collection, '...')) + has_more <- TRUE + page <- 1 + while(has_more) { + Sys.sleep(1) + print(paste0('Page #', page)) + url <- get_smb_url(page) + response <- GET(url) + content <- rawToChar(response$content) + content <- fromJSON(content) + # content <- rbindlist(content, fill=T) + content <- content %>% as.data.table() + has_more <- nrow(content) > 0 && 'price' %in% colnames(content) + if(has_more) { + content <- content[, list(mint, price)] + content <- unique(content) + content$price <- as.numeric(content$price) / (10^9) + has_more <- nrow(content) >= 40 + colnames(content)[1] <- 'tokenMint' + df <- merge(content, mints, by=c('tokenMint')) %>% as.data.table() + df <- df[, list(collection, token_id, price)] + page <- page + 1 + solana_listings <- rbind(solana_listings, df) + } + } +} + +head(solana_listings) +# head(terra_listings) +# new_listings <- rbind(solana_listings, terra_listings) +new_listings <- unique(solana_listings) + +# listings <- read.csv('./data/listings.csv') %>% as.data.table() +rem <- unique(new_listings$collection) +sort(rem) +listings <- listings[ !(collection %in% eval(rem)), ] +listings <- listings[, list(collection, token_id, price)] +listings <- rbind(listings, new_listings) +listings <- listings[order(collection, price)] +listings[, token_id := as.integer(token_id)] +listings <- listings[ !(collection %in% c('LunaBulls','Galactic Punks','Galactic Angels','Levana Dragon Eggs')) ] + +listings <- listings[!is.na(price)] +listings <- listings %>% as.data.table() + +sort(unique(listings$collection)) +# write.csv(unique(listings[, collection]), '~/Downloads/tmp.csv', row.names=F) + +floors <- listings %>% + group_by(collection) %>% + summarize(cur_floor = min(price)) %>% + as.data.table() + + +get_fmp <- function(data, coefsdf, pred_price) { + coefsdf[, tot := lin_coef + log_coef ] + coefsdf[, lin_coef := lin_coef / tot] + coefsdf[, log_coef := log_coef / tot] + sum(coefsdf$log_coef) + sum(coefsdf$lin_coef) + + fmp <- merge( pred_price, coefsdf, by=c('collection') ) + fmp <- merge( fmp, data[, list(token_id, collection, block_timestamp, price, mn_20)], by=c('token_id','collection') ) + # fmp <- merge( fmp, floors, by=c('collection') ) + fmp[, abs_chg := (mn_20 - floor_price) * lin_coef ] + fmp[, pct_chg := (mn_20 - floor_price) * log_coef ] + fmp[, fair_market_price := pred_price + abs_chg + (pct_chg * pred_price / floor_price) ] +} + +if(FALSE) { + coefsdf[, tot := lin_coef + log_coef ] + coefsdf[, lin_coef := lin_coef / tot] + coefsdf[, log_coef := log_coef / tot] + sum(coefsdf$log_coef) + sum(coefsdf$lin_coef) + + fmp <- merge( pred_price, coefsdf, by=c('collection') ) + fmp <- merge( fmp, floors, by=c('collection') ) + fmp[, abs_chg := (cur_floor - floor_price) * lin_coef ] + fmp[, pct_chg := (cur_floor - floor_price) * log_coef ] + fmp[, fair_market_price := pred_price + abs_chg + (pct_chg * pred_price / floor_price) ] + + mn <- fmp %>% group_by(collection, cur_floor) %>% summarize(mn = min(fair_market_price)) %>% as.data.table() + mn[, ratio := cur_floor / mn] + fmp <- merge(fmp, mn[, list(collection, ratio)]) + fmp[ratio < 1, fair_market_price := fair_market_price * ratio ] + + fmp[, cur_sd := pred_sd * (cur_floor / floor_price) * SD_SCALE ] + fmp[, price_low := qnorm(.2, fair_market_price, cur_sd) ] + fmp[, price_high := qnorm(.8, fair_market_price, cur_sd) ] + + fmp[, price_low := pmax(price_low, cur_floor * 0.975) ] + fmp[, price_high := pmax(price_high, cur_floor * 1.025) ] + + fmp[, price_low := round(price_low, 2) ] + fmp[, price_high := round(price_high, 2) ] + fmp[, fair_market_price := pmax(cur_floor, fair_market_price) ] + fmp[, fair_market_price := round(fair_market_price, 2) ] + fmp[, cur_sd := round(cur_sd, 2) ] + head(fmp[collection == 'SOLGods'][order(fair_market_price)]) + head(fmp[(collection == 'SOLGods') & (rk <= 4654)][order(fair_market_price)]) + head(fmp[(collection == 'SOLGods') & (rk == 4654)][order(fair_market_price)]) + + tmp <- merge(listings, fmp, by = c('collection','token_id')) %>% as.data.table() + tmp[, deal_score := pnorm(price, fair_market_price, cur_sd) ] + tmp[, deal_score := 100 * (1 - deal_score) ] + tmp[, vs_floor := (price / cur_floor) - 1 ] + tmp[, vs_floor_grp := ifelse(vs_floor < .1, '<10%', ifelse(vs_floor < .25, '<25%', '>25%')) ] + tmp[, vs_floor := (price - cur_floor) ] + tmp <- tmp[ !(collection %in% c('Levana Dragon Eggs','Galactic Punks','LunaBulls','Galactic Angels','MAYC')) ] + + + t2 <- tmp[order(-deal_score),.SD[2], list(vs_floor_grp, collection)] %>% as.data.table() + t2 <- t2[, list(collection, vs_floor_grp, deal_score)][order(collection, vs_floor_grp)] + t3 <- tmp[order(-deal_score),.SD[3], list(vs_floor_grp, collection)] %>% as.data.table() + t3 <- t3[, list(collection, vs_floor_grp, deal_score)][order(collection, vs_floor_grp)] + colnames(t2) <- c('collection','vs_floor_grp','deal_score_g2') + colnames(t3) <- c('collection','vs_floor_grp','deal_score_g3') + tmp <- merge(tmp, t2, by=c('collection','vs_floor_grp')) + tmp <- merge(tmp, t3, by=c('collection','vs_floor_grp')) + + + t2 <- tmp[order(-deal_score),.SD[2], list(collection)] %>% as.data.table() + t2 <- t2[, list(collection, deal_score)][order(collection)] + t3 <- tmp[order(-deal_score),.SD[3], list(collection)] %>% as.data.table() + t3 <- t3[, list(collection, deal_score)][order(collection)] + colnames(t2) <- c('collection','deal_score_2') + colnames(t3) <- c('collection','deal_score_3') + + tmp <- merge(tmp, t2, by=c('collection')) + tmp <- merge(tmp, t3, by=c('collection')) + + tmp[, pts := (deal_score * 5 - deal_score_g2 - deal_score_g3 - deal_score_2 - deal_score_3) * ((cur_floor / price)**0.75) + (100 * (1 - (( price - cur_floor ) / (fair_market_price - cur_floor)))) ] + url <- 'https://discord.com/api/webhooks/976332557996150826/8KZqD0ov5OSj1w4PjjLWJtmgnCM9bPWaCkZUUEDMeC27Z0iqiA-ZU5U__rYU9tQI_ijA' + unique(tmp$collection) + for(col in c('price','pred_price','fair_market_price','vs_floor','deal_score','deal_score_2','deal_score_3','pts')) { + if(!'price' %in% col) { + tmp[, eval(col) := round(get(col)) ] + } else { + tmp[, eval(col) := ifelse( + get(col) < 10 + , round(get(col), 2) + , ifelse( + get(col) < 100 + , round(get(col), 1) + , round(get(col))) + ) + ] + } + } + tmp <- tmp[order(-pts)] + head(tmp[, list(collection, token_id, price, nft_rank, rk, pred_price, cur_floor, fair_market_price, deal_score, deal_score_2, deal_score_3, pts)], 20) + head(tmp[, list(collection, token_id, price, nft_rank, rk, pred_price, cur_floor, fair_market_price, deal_score, deal_score_2, deal_score_3, pts)], 20) + paste(head(tmp$label), collapse='\n') + + tmp[, l := nchar(collection)] + mx <- max(tmp$l) + # tmp$clean_collection <- str_pad(collection, eval(mx) - l, side = 'right', pad = '-') ] + tmp$n_pad <- mx - tmp$l + tmp$clean_collection <- str_pad(tmp$collection, mx - tmp$l, side = 'right', pad = '-') + tmp[, clean_collection := str_pad(collection, eval(mx), pad='-', side='right')] + tmp[, clean_collection := str_pad(collection, eval(mx), pad='-', side='both')] + tmp$clean_collection <- str_pad(tmp$collection, mx, pad='-', ) + tmp[, label := paste(clean_collection, str_pad(token_id, 4, side='left'), price, fair_market_price, deal_score, sep='\t')] + tmp[, label := paste( + clean_collection + , str_pad(token_id, 4, side='left') + , str_pad(rk, 4, side='left') + , str_pad(price, 4, side='left') + , str_pad(vs_floor, 5, side='left') + , str_pad(fair_market_price, 4, side='left') + , str_pad(deal_score, 2, side='left') + , str_pad(deal_score_2, 2, side='left') + , str_pad(deal_score_3, 2, side='left') + , str_pad(pts, 3, side='left') + , sep='\t') + ] + header <- paste( + str_pad('collection', mx, side='both', pad='-') + , str_pad('id', 4, side='left') + , str_pad('rk', 4, side='left') + , str_pad('$', 3, side='left') + , str_pad('floor', 5, side='left') + , str_pad('fmp', 3, side='left') + , str_pad('ds', 2, side='left') + , str_pad('ds2', 2, side='left') + , str_pad('ds3', 2, side='left') + , str_pad('pts', 3, side='left') + , sep='\t') + + tmp <- tmp[order(-pts)] + content <- tmp[ (price < 0.9 * fair_market_price) , head(.SD, 2), by = collection] + content <- head(content[order(-pts)], 15) + # content <- paste(c(header, content$label, collapse='\n')) + + content <- paste(c(header, content$label), collapse='\n') + # content <- paste(c(header, head(tmp$label, 10)), collapse='\n') + data <- list( + content = paste0('```',content,'```') + ) + res <- POST(url, body = data, encode = "form", verbose()) + + # tmp <- tmp[order(-deal_score)] + # head(tmp) + # plot_data[, deal_score := round(100 * (1 - y))] + # y <- pnorm(x, mu, sd) + # tmp[, deal_score := ((fair_market_price / price) - 1) ] + # tmp[, deal_score := ((fair_market_price / price) - 0) ] + # tmp <- tmp[order(-deal_score)] + # tmp <- tmp[, list(collection, token_id, fair_market_price, price, deal_score)] + # tmp[, .SD[1:3], collection] + + # fmp <- fmp[, list(collection, token_id, nft_rank, rk, fair_market_price, price_low, price_high)] + fmp <- fmp[, list(collection, token_id, nft_rank, rk, fair_market_price, cur_floor, cur_sd, lin_coef, log_coef)] + colnames(fmp)[3] <- 'rarity_rank' + colnames(fmp)[4] <- 'deal_score_rank' + + if (FALSE) { + for( cur_collection in unique(fmp$collection)) { + print(paste0('Working on ',cur_collection, '...')) + data <- fmp[collection == eval(cur_collection)] + KafkaGeneric( + .topic = 'prod-data-science-uploads' + , .url = 'https://kafka-rest-proxy.flipside.systems' + , .project = paste0('nft-deal-score-rankings-', cur_collection) + , .data = data + ) + } + } +} + +# write the floor prices to snowflake +data <- floors +KafkaGeneric( + .topic = 'prod-data-science-uploads' + , .url = 'https://kafka-rest-proxy.flipside.systems' + , .project = 'nft-deal-score-floors' + , .data = data +) + +sort(unique(listings$collection)) + +save( + listings + , file = listings_file +) +if(!isRstudio) { + write.csv(listings, paste0(base_dir, 'nft_deal_score_listings.csv')) +} + +``` + +Done updating at `r Sys.time()` + +The end. Byeeeee. \ No newline at end of file diff --git a/viz/upload_solana_nft_labels.py b/viz/upload_solana_nft_labels.py new file mode 100644 index 00000000..a5e62fba --- /dev/null +++ b/viz/upload_solana_nft_labels.py @@ -0,0 +1,655 @@ +import re +import os +import json +import time +# import math +import requests +import pandas as pd +# import urllib.request +import snowflake.connector +from time import sleep + +# from solana_model import just_float +# from utils import clean_name, clean_token_id, format_num, merge + +############################ +# Define Constants # +############################ +BASE_PATH = '/home/data-science' +DATA_FOLDER = '/rstudio-data/nft_labels' +RPC = 'https://red-cool-wildflower.solana-mainnet.quiknode.pro/a1674d4ab875dd3f89b34863a86c0f1931f57090/' + + + +############################## +# Load DB Connection # +############################## +with open('{}/data_science/util/snowflake.pwd'.format(BASE_PATH), 'r') as f: + pwd = f.readlines()[0].strip() +with open('{}/data_science/util/snowflake.usr'.format(BASE_PATH), 'r') as f: + usr = f.readlines()[0].strip() + +ctx = snowflake.connector.connect( + user=usr, + password=pwd, + account='vna27887.us-east-1' +) + + +############################ +# Helper Functions # +############################ +def read_csv(data_folder, fname): + return(pd.read_csv('{}/{}.csv'.format(data_folder, fname))) + +def write_csv(data_folder, fname, df, verbose = True): + df.to_csv('{}/{}.csv'.format(data_folder, fname), index=False) + if verbose: + print('Wrote {} rows to {}'.format(len(df), fname)) + +def clean_colnames(df): + names = [ x.lower() for x in df.columns ] + df.columns = names + return(df) + +def clean_collection_name(x): + x = re.sub('\|', '-', x).strip() + x = re.sub('\)', '', x).strip() + x = re.sub('\(', '', x).strip() + x = re.sub('\'', '', x).strip() + return(x) + +def merge(left, right, on=None, how='inner', ensure=True, verbose=True): + df = left.merge(right, on=on, how=how) + if len(df) != len(left) and (ensure or verbose): + print('{} -> {}'.format(len(left), len(df))) + cur = left.merge(right, on=on, how='left') + cols = set(right.columns).difference(set(left.columns)) + print(cols) + col = list(cols)[0] + missing = cur[cur[col].isnull()] + print(missing.head()) + if ensure: + assert(False) + return(df) + +def Convert(tup, di): + di = dict(tup) + return di + + +#################################### +# Metadata From HowRare.Is # +#################################### +def how_rare_is_api(): + query = ''' + SELECT DISTINCT LOWER(project_name) AS lower_collection + FROM solana.core.dim_nft_metadata + ''' + df = ctx.cursor().execute(query) + df = pd.DataFrame.from_records(iter(df), columns=[x[0] for x in df.description]) + + url = 'https://api.howrare.is/v0.1/collections' + r = requests.get(url) + j = r.json() + c_df = pd.DataFrame(j['result']['data']).sort_values('floor_marketcap', ascending=0) + c_df['lower_collection'] = c_df.url.apply(lambda x: x.lower().strip() ) + seen = sorted(df.LOWER_COLLECTION.apply(lambda x: re.sub(' |_|\'', '', x) ).values) + c_df['seen_1'] = c_df.url.apply(lambda x: re.sub(' |_|\'', '', x[1:]).lower() in seen ).astype(int) + c_df['seen_2'] = c_df.name.apply(lambda x: re.sub(' |_|\'', '', x).lower() in seen ).astype(int) + c_df['seen'] = (c_df.seen_1 + c_df.seen_2 > 0).astype(int) + seen = seen + [ 'smb','aurory','degenapes','thugbirdz','degods','okay_bears','catalinawhalemixer','cetsoncreck','stonedapecrew','solgods' ] + c_df = c_df[-(c_df.url.isin([ '/'+x for x in seen]))] + c_df = c_df[c_df.seen == 0] + it = 0 + tot = len(c_df) + m_data = [] + print('Pulling metadata for {} collections'.format(tot)) + for row in c_df.iterrows(): + it += 1 + row = row[1] + collection = row['name'] + print('#{} / {}: {}'.format(it, tot, collection)) + url = row['url'][1:] + if it > 1: + assert(len(m_data)) + url = 'https://api.howrare.is/v0.1/collections/'+url + r = requests.get(url) + j = r.json() + n_errors = 0 + for i in j['result']['data']['items']: + try: + token_id = int(i['id']) + mint = i['mint'] + image = i['image'] + for d in i['attributes']: + d['token_id'] = token_id + d['collection'] = collection + d['mint_address'] = mint + d['image_url'] = image + m_data += [ d ] + except: + # print('Error') + n_errors += 1 + pass + if n_errors: + print('{} errors'.format(n_errors)) + metadata = pd.DataFrame(m_data).rename(columns={'name':'feature_name', 'value':'feature_value'}) + + write_csv(DATA_FOLDER, 'howrare_labels', metadata[['collection','mint_address']]) + + a = metadata.groupby(['collection','mint_address','token_id','image_url'])[[ 'feature_name','feature_value' ]].apply(lambda g: Convert(list(map(tuple, g.values.tolist())), {}) ).reset_index() + a.columns = ['collection','mint_address','token_id','image_url', 'token_metadata'] + a['commission_rate'] = None + a['contract_address'] = a.mint_address + a['contract_name'] = a.collection + a['created_at_block_id'] = 0 + a['created_at_timestamp'] = '2021-01-01' + a['created_at_tx_id'] = '' + a['creator_address'] = a.mint_address + a['creator_name'] = a.collection + a['project_name'] = a.collection + a['token_metadata_uri'] = a.image_url + a['token_name'] = a.collection + a['n'] = range(len(a)) + a['n'] = a.n.apply(lambda x: int(x/50) ) + a['token_id'] = a.token_id.astype(int) + + # remove existing files + fnames = os.listdir(DATA_FOLDER+'/metadata/results/') + print('fnames') + print(fnames) + for f in fnames: + os.remove(DATA_FOLDER+'/metadata/results/'+f) + + # write new metadata incrementally to upload to solana.core.dim_nft_metadata + n = 100000 + tot = int(len(a) / n) + 1 + for i in range(0, len(a), n): + ind = int(i/n) + print('#{} / {}'.format(ind, tot)) + g = a.head(i+n).tail(n).to_dict('records') + txt = [ + { + "model": { + "blockchain": "solana", + "sinks": [ + { + "destination": "{database_name}.silver.nft_metadata", + "type": "snowflake", + "unique_key": "blockchain || contract_address || token_id" + } + ], + }, + "results": g[x:x+50] + } + for x in range(0, len(g), 50) + ] + w = pd.DataFrame({'ind': range(len(txt)), 'results':[json.dumps(x) for x in txt] }) + write_csv( DATA_FOLDER, 'metadata/results/{}'.format(ind), w ) + return + + +################################# +# Load Data From ME API # +################################# +def mints_from_me(): + ################################## + # Get All ME Collections # + ################################## + headers = { + # 'Authorization': 'Bearer 9c39e05c-db3c-4f3f-ac48-84099111b813' + } + data = [] + has_more = 1 + offset = 0 + while has_more: + sleep(1) + print(offset) + url = 'https://api-mainnet.magiceden.dev/v2/collections?offset={}&limit=500'.format(offset) + r = requests.get(url) + j = r.json() + data = data + j + has_more = len(j) + offset += 500 + df = pd.DataFrame(data) + write_csv(DATA_FOLDER, 'me_collections', df) + # df.to_csv('{}/me_collections.csv'.format(DATA_FOLDER), index=False) + df = read_csv(DATA_FOLDER, 'me_collections') + # df = pd.read_csv('./data/me_collections.csv') + + ########################################### + # Get 1 Mint From Each Collection # + ########################################### + it = 0 + l_data = [] + # old_l_df = pd.read_csv('./data/me_mints.csv') + old_l_df = read_csv(DATA_FOLDER, 'me_mints') + seen = list(old_l_df.symbol.unique()) + print('We\'ve already seen {} / {} mints from ME'.format(len(seen), len(df))) + df = df[ -df.symbol.isin(seen) ] + df = df[ (df.symbol.notnull()) & (df.symbol != '') ] + df = df.sort_values('symbol') + tot = len(df) + start = time.time() + for row in df.iterrows(): + sleep(0.5) + it += 1 + row = row[1] + # print('Listings on {}...'.format(row['symbol'])) + url = 'https://api-mainnet.magiceden.dev/v2/collections/{}/activities?offset=0&limit=1'.format(row['symbol']) + if row['symbol'] in seen: + print('Seen') + continue + try: + r = requests.get(url, headers=headers) + j = r.json() + except: + try: + print('Re-trying in 10s') + sleep(10) + r = requests.get(url, headers=headers) + j = r.json() + except: + try: + print('Re-trying in 60s') + sleep(60) + r = requests.get(url, headers=headers) + j = r.json() + except: + print('Re-trying in 60s (again!)') + sleep(60) + r = requests.get(url, headers=headers) + j = r.json() + if len(j): + l_data += [[ row['symbol'], row['name'], j[0]['tokenMint'] ]] + if it == 1 or it % 10 == 0: + print('#{} / {} ({} records in {} secs)'.format(it, tot, len(l_data), round(time.time() - start))) + # l_df = pd.DataFrame(l_data, columns=['symbol','name','mint_address']) + # l_df.to_csv('./data/me_mints.csv', index=False) + l_df = pd.DataFrame(l_data, columns=['symbol','name','mint_address']) + l_df = pd.concat([l_df, old_l_df]).drop_duplicates(subset=['symbol']) + print('Adding {} rows to me_mints'.format(len(l_df) - len(old_l_df))) + # l_df.to_csv('./data/me_mints.csv', index=False) + write_csv(DATA_FOLDER, 'me_mints', l_df) + + + ###################################################### + # Get Update Authorities For All Collections # + ###################################################### + # l_df = pd.read_csv('./data/me_mints.csv') + # m_old = pd.read_csv('./data/me_update_authorities.csv') + m_old = read_csv(DATA_FOLDER, 'me_update_authorities') + m_old['seen'] = 1 + m_data = list(m_old[['symbol','name','update_authority','seen']].values) + seen = [ x[0] for x in m_data ] + print('Seen {} m_data'.format(len(seen))) + l_df = l_df[-l_df.symbol.isin(seen)] + l_df = l_df.sort_values('symbol') + it = 0 + for row in l_df.iterrows(): + sleep(.5) + it += 1 + row = row[1] + symbol = row['symbol'] + print('Working on {}...'.format(symbol)) + if symbol in seen: + print('Seen') + continue + url = 'https://api-mainnet.magiceden.dev/v2/tokens/{}'.format(row['mint_address']) + try: + r = requests.get(url, headers=headers) + j = r.json() + except: + print('Re-trying in 10s') + sleep(10) + try: + r = requests.get(url, headers=headers) + j = r.json() + except: + print('Re-trying in 60s') + sleep(60) + r = requests.get(url, headers=headers) + j = r.json() + if 'updateAuthority' in j.keys(): + m_data += [[ row['symbol'], row['name'], j['updateAuthority'], 0 ]] + if it % 10 == 0: + print('it#{}: {}'.format(it, len(m_data))) + # m_df = pd.DataFrame(m_data, columns=['symbol','name','update_authority']) + # m_df.to_csv('./data/me_update_authorities.csv', index=False) + m_df = pd.DataFrame(m_data, columns=['symbol','name','update_authority','seen']) + m_df = m_df.drop_duplicates() + print('Adding {} rows to me_update_authorities'.format(len(m_df) - len(m_old))) + write_csv(DATA_FOLDER, 'me_update_authorities', m_df) + # m_df.to_csv('./data/me_update_authorities.csv', index=False) + +def pull_from_metaboss(): + + ###################################################### + # Get Update Authorities For All Collections # + ###################################################### + # m_df = pd.read_csv('./data/me_update_authorities.csv') + m_df = read_csv(DATA_FOLDER, 'me_update_authorities') + n_auth = m_df.groupby('update_authority').name.count().reset_index().rename(columns={'name':'n_auth'}) + m_df = m_df.merge(n_auth) + l1 = len(m_df[ (m_df.seen == 0) & (m_df.n_auth == 1)]) + l2 = len(m_df[ (m_df.seen == 0) & (m_df.n_auth > 1)]) + print('{} with 1 update_authority; {} with 2+ update_authority'.format(l1, l2)) + + need = list(m_df[ (m_df.seen == 0) & (m_df.n_auth == 1) ].update_authority.unique()) + need = m_df[m_df.update_authority.isin(need)] + # l_df = pd.read_csv('./data/me_mints.csv') + l_df = read_csv(DATA_FOLDER, 'me_mints') + fix = need.merge(l_df[[ 'name','mint_address' ]]) + need = fix.copy().rename(columns={'name':'collection'}) + # need = need.drop_duplicates(subset=['update_authority']).sort_values('collection').head(7).tail(1) + need['collection'] = need.collection.apply(lambda x: clean_collection_name(x) ) + need = need.drop_duplicates(subset=['update_authority']).sort_values('collection') + # need = need.head(2) + + mfiles = ['/data/mints/{}/{}_mint_accounts.json'.format(re.sub(' |-', '_', collection), update_authority) for collection, update_authority in zip(need.collection.values, need.update_authority.values) ] + seen = [ x for x in mfiles if os.path.exists(x) ] + seen = [] + + # for update authorities that have only 1 collection, we can just check metaboss once + mfolder = '{}/mints/'.format(DATA_FOLDER) + it = 0 + tot = len(need) + for row in need.iterrows(): + it += 1 + row = row[1] + collection = row['collection'] + print('#{} / {}: {}'.format(it, tot, collection)) + # if collection in seen: + # continue + update_authority = row['update_authority'] + # print('Working on {}...'.format(collection)) + collection_dir = re.sub(' |-', '_', collection) + + dir = '{}{}/'.format(mfolder, collection_dir) + mfile = '{}{}_mint_accounts.json'.format(dir, update_authority) + if not os.path.exists(dir): + print(collection) + os.makedirs(dir) + # elif len(os.listdir(dir)) and os.path.exists(mfile): + # print('Already have {}.'.format(collection)) + # print('Seen') + # continue + seen.append(update_authority) + os.system('metaboss -r {} -T 300 snapshot mints --update-authority {} --output {}'.format(RPC, update_authority, dir)) + + # write the mints to csv + data = [] + for path in os.listdir(mfolder): + if os.path.isdir('{}{}'.format(mfolder, path)): + collection = re.sub('_', ' ', path).strip() + for fname in os.listdir(mfolder+path): + f = mfolder+path+'/'+fname + if os.path.isfile(f) and '.json' in f: + with open(f) as file: + j = json.load(file) + for m in j: + data += [[ collection, m ]] + df = pd.DataFrame(data, columns=['collection','mint_address']) + df.collection.unique() + write_csv(DATA_FOLDER, 'single_update_auth_labels', df) + # df.to_csv('./data/single_update_auth_labels.csv', index=False) + + ################################ + # Multiple Authorities # + ################################ + need = list(m_df[ (m_df.seen == 0) & (m_df.n_auth > 1) ].update_authority.unique()) + need = m_df[m_df.update_authority.isin(need)] + fix = need.merge(l_df[[ 'name','mint_address' ]]) + need = fix.copy().rename(columns={'name':'collection'}) + need['collection'] = need.collection.apply(lambda x: clean_collection_name(x) ) + need = need.sort_values('collection').drop_duplicates(subset=['update_authority'], keep='first') + # need = need.head(2) + it = 0 + a = [] + for row in need.iterrows(): + it += 1 + print('#{}/{}'.format(it, len(need))) + row = row[1] + collection = row['collection'] + update_authority = row['update_authority'] + print('Working on {}...'.format(collection)) + collection_dir = re.sub(' |-', '_', collection) + + dir = '{}{}/'.format(mfolder, collection_dir) + mfile = '{}{}_mint_accounts.json'.format(dir, update_authority) + if not os.path.exists(dir): + print(collection) + os.makedirs(dir) + a.append(update_authority) + os.system('metaboss -r {} -T 300 snapshot mints --update-authority {} --output {}'.format(RPC, update_authority, dir)) + + odir = dir+'output/' + if not os.path.exists(odir): + print('Making dir {}'.format(odir)) + os.makedirs(odir) + os.system('metaboss -r {} -T 300 decode mint --list-file {} --output {}'.format(RPC, mfile, odir )) + + ################################################## + # Load All The Mints for Each Collection # + ################################################## + # now that we have the mints, create a data frame with the info for each mint in each collection + mfolder = '{}/mints/'.format(DATA_FOLDER) + data = [] + seen = [ x[1] for x in data ] + it = 0 + dirs = sorted(os.listdir(mfolder)) + dirs = [ x for x in dirs if not x in ['3D_Sniping_Demons']] + tot = len(dirs) + for path in dirs: + print('{} / {} ({} records)'.format(it, tot, len(data))) + it += 1 + if os.path.isdir(mfolder+path): + collection = re.sub('_', ' ', path).strip() + print('Found {}'.format(collection)) + if not os.path.exists(mfolder+path+'/output/'): + print('No output') + continue + fnames = os.listdir(mfolder+path+'/output/') + print('{} files found'.format(len(fnames))) + for fname in fnames: + f = mfolder+path+'/output/'+fname + if fname[:-5] in seen: + continue + if os.path.isfile(f) and '.json' in f: + try: + with open(f) as file: + j = json.load(file) + data += [[ collection, fname, j['name'], j['symbol'], j['uri'] ]] + except: + print('Error {}'.format(fname[:-5])) + + ################################################## + # Load All The Mints for Each Collection # + ################################################## + new_mints = pd.DataFrame(data, columns=['collection','mint_address','name','symbol','uri']) + # tmp = tmp[-(tmp.collection.isin(['Dskullys','Decimusdynamics']))] + n = len(new_mints[(new_mints.uri.isnull()) | (new_mints.uri == '')]) + tot = len(new_mints) + pct = round(n * 100 / tot, 1) + print('{} ({}%) rows have no uri'.format(n, pct)) + new_mints = new_mints[new_mints.uri != ''] + + # function to clean the name of each NFT (remove the number) + def f_cn(x): + if not x or x != x: + return(x) + if '#' in x[-6:]: + x = ''.join(re.split('#', x)[:-1]).strip() + elif bool(re.match('.+\s+[0-9]+', x)): + x = ' '.join(re.split(' ', x)[:-1]).strip() + return(x) + new_mints['clean_name'] = new_mints.name.apply(lambda x: f_cn(x) ) + + # determine for each collection if we should look at collection-name-symbol, collection-symbol, or just collection to determine what collection it actuallly belongs to + # this logic is because e.g. some only have a few names in the collection so we can iterate, but some have a different name for each NFT, so we assume its the same collection for all + a = new_mints.drop_duplicates(subset=['collection','clean_name','symbol']).groupby(['collection']).uri.count().reset_index().sort_values('uri', ascending=0) + symbol_only = a[a.uri > 10].collection.unique() + b = new_mints[new_mints.collection.isin(symbol_only)].drop_duplicates(subset=['collection','symbol']).groupby(['collection']).uri.count().reset_index().sort_values('uri', ascending=0) + collection_only = b[b.uri > 10].collection.unique() + symbol_only = [x for x in symbol_only if not x in collection_only] + + # now get the info for each collection-name-symbol combo + g1 = new_mints[ (-(new_mints.collection.isin(symbol_only))) & (-(new_mints.collection.isin(collection_only))) ].groupby(['collection','clean_name','symbol']).head(1).reset_index() + g2 = new_mints[ ((new_mints.collection.isin(symbol_only))) & (-(new_mints.collection.isin(collection_only))) ].groupby(['collection','symbol']).head(1).reset_index() + g3 = new_mints[ (-(new_mints.collection.isin(symbol_only))) & ((new_mints.collection.isin(collection_only))) ].groupby(['collection']).head(1).reset_index() + g = pd.concat([g1, g2, g3]).drop_duplicates(subset=['mint_address']) + print('{} Total: {} all, {} collection-symbol {} collection'.format(len(g), len(g1), len(g2), len(g3))) + # g.to_csv('~/Downloads/tmp-g.csv', index=False) + + # iterate over each row to get what collection they are actually in + # by pulling data from the uri + uri_data = [] + it = 0 + tot = len(g) + print(tot) + errs = [] + seen = [ x['uri'] for x in uri_data ] + # for row in g[ -(g.uri.isin(seen)) ].iterrows(): + for row in g.iterrows(): + row = row[1] + it += 1 + # if it % 100 == 0: + # uri_df = pd.DataFrame(uri_data)[[ 'collection','name','symbol','row_symbol','row_collection','uri','row_clean_name','mint_address' ]] + # uri_df.to_csv('~/Downloads/uri_df.csv', index=False) + print('#{} / {}: {}'.format(it, tot, row['collection'])) + try: + r = requests.get(row['uri']) + j = r.json() + j['uri'] = row['uri'] + j['row_collection'] = row['collection'] + j['row_clean_name'] = row['clean_name'] + j['row_symbol'] = row['symbol'] + j['mint_address'] = row['mint_address'] + uri_data += [j] + except: + print('Error') + errs.append(row) + uri_df = pd.DataFrame(uri_data)[[ 'collection','name','symbol','row_symbol','row_collection','uri','row_clean_name','mint_address' ]] + write_csv(DATA_FOLDER, 'uri_df', uri_df) + # uri_df.to_csv('~/Downloads/uri_df.csv', index=False) + + # for each row, parse the json from the uri + # uri_df = pd.read_csv('~/Downloads/uri_df.csv') + # read_csv(DATA_FOLDER, 'uri_df') + def f(x, c): + x = str(x) + try: + n = json.loads(re.sub("'", "\"", x))[c] + if type(n) == list: + return(n[0]) + return(n) + except: + try: + return(json.loads(re.sub("'", "\"", x))[c]) + except: + try: + return(json.loads(re.sub("'", "\"", x))[0][c]) + except: + try: + return(json.loads(re.sub("'", "\"", x))[0]) + except: + return(x) + # parse the json more + uri_df['parsed_collection'] = uri_df.collection.apply(lambda x: f(x, 'name') ) + uri_df['parsed_family'] = uri_df.collection.apply(lambda x: f(x, 'family') ) + uri_df['clean_name'] = uri_df.name.apply( lambda x: f_cn(x) ) + # calculate what the collection name is + uri_df['use_collection'] = uri_df.parsed_collection.replace('', None).fillna( uri_df.clean_name )#.fillna( uri_df.row_symbol ) + # uri_df[uri_df.use_collection == 'nan'][['use_collection','parsed_collection','parsed_family','clean_name','name','collection','symbol','row_symbol','row_collection']].head() + # uri_df[uri_df.use_collection == 'nan'][['use_collection','parsed_collection','parsed_family','clean_name','name','collection','symbol','row_symbol','row_collection']].to_csv('~/Downloads/tmp.csv', index=False) + len(uri_df) + + # clean the collection name + def f1(x): + try: + if len(x['use_collection']) == 1: + return(x['clean_name']) + if bool(re.match('.+\s+#[0-9]+', x['use_collection'])): + return(''.join(re.split('#', x['use_collection'])[:-1]).strip()) + if '{' in x['use_collection']: + return(x['clean_name']) + return(x['use_collection'].strip().title()) + except: + return(x['use_collection'].strip().title()) + uri_df['tmp'] = uri_df.apply(lambda x: f1(x), 1 ) + uri_df['use_collection'] = uri_df.apply(lambda x: f1(x), 1 ) + + # clean the mint_address + uri_df['mint_address'] = uri_df.mint_address.apply(lambda x: re.sub('.json','', x)) + uri_df = uri_df.fillna('None') + + for i in range(2): + # for each collection-name-symbol combo, see how many have multiple mappings + a = uri_df.copy().fillna('None') + a = a[['row_collection','row_clean_name','row_symbol','use_collection']].drop_duplicates().groupby(['row_collection','row_clean_name','row_symbol']).use_collection.count().reset_index().rename(columns={'use_collection':'n_1'}) + uri_df = merge(uri_df, a, ensure=True) + + # for each collection-symbol combo, see how many have multiple mappings + a = uri_df.copy().fillna('None') + a = a[['row_collection','row_symbol','use_collection']].drop_duplicates().groupby(['row_collection','row_symbol']).use_collection.count().reset_index().rename(columns={'use_collection':'n_2'}) + uri_df = merge(uri_df, a, ensure=True) + + # for each collection combo, see how many have multiple mappings + a = uri_df.copy().fillna('None') + a = a[['row_collection','use_collection']].drop_duplicates().groupby(['row_collection']).use_collection.count().reset_index().rename(columns={'use_collection':'n_3'}) + uri_df = merge(uri_df, a, ensure=True) + + uri_df['n'] = uri_df.apply(lambda x: x['n_3'] if x['row_collection'] in collection_only else x['n_2'] if x['row_collection'] in symbol_only else x['n_1'], 1 ) + print('{} / {} ({}%) have multiple collection-name-symbol mappings'.format(len(uri_df[uri_df.n > 1]), len(uri_df), round( 100.0 * len(uri_df[uri_df.n > 1]) / len(uri_df)))) + + # if there is multiple, use the parsed_family instead of the use_collection + uri_df['use_collection'] = uri_df.apply(lambda x: x['use_collection'] if x['n'] == 1 else x['parsed_family'], 1 ) + del uri_df['n_1'] + del uri_df['n_2'] + del uri_df['n_3'] + + # only take rows where there is a single mapping + m = uri_df[uri_df.n==1][[ 'use_collection','row_collection','row_clean_name','row_symbol' ]].dropna().drop_duplicates() + m.columns = [ 'use_collection','collection','clean_name','symbol' ] + + m_1 = new_mints[ (-(new_mints.collection.isin(symbol_only))) & (-(new_mints.collection.isin(collection_only))) ].fillna('').merge(m.fillna(''), how='left') + m_2 = new_mints[ ((new_mints.collection.isin(symbol_only))) & (-(new_mints.collection.isin(collection_only))) ][[ 'collection','mint_address','symbol' ]].fillna('').merge(m.fillna(''), how='left') + m_3 = new_mints[ (-(new_mints.collection.isin(symbol_only))) & ((new_mints.collection.isin(collection_only))) ][[ 'collection','mint_address' ]].fillna('').merge(m.fillna(''), how='left') + len(m_1) + len(m_2) + len(m_3) + len(new_mints) + # m = new_mints.fillna('').merge(m.fillna(''), how='left') + m = pd.concat( [m_1, m_2, m_3] ) + print('After all this, we have {}% of the mints'.format( round(len(m) * 100 / len(new_mints)) )) + len(new_mints) + len(m) + m['mint_address'] = m.mint_address.apply(lambda x: re.sub('.json', '', x) ) + m = m[['mint_address','use_collection']].dropna().drop_duplicates() + m.columns = ['mint_address','collection'] + + m[m.collection.isnull()].head() + m[m.collection=='Nan'].head() + + m = m[m.collection != 'Nan'] + + tmp = m.groupby('collection').mint_address.count().reset_index().sort_values('mint_address', ascending=0) + tmp.head() + + # m.to_csv('./data/mult_update_auth_labels.csv', index=False) + write_csv(DATA_FOLDER, 'mult_update_auth_labels', m) + +def compile(): + single_update_auth_labels = read_csv(DATA_FOLDER, 'single_update_auth_labels') + mult_update_auth_labels = read_csv(DATA_FOLDER, 'mult_update_auth_labels') + howrare_labels = read_csv(DATA_FOLDER, 'howrare_labels') + df = pd.concat([howrare_labels, single_update_auth_labels, mult_update_auth_labels]) + df = df[ (df.collection != 'Nan') & (df.collection != 'nan') & (df.collection.notnull()) ] + df = df.drop_duplicates(subset=['mint_address'], keep='first') + print(len(df[df.collection == 'DegenTown'])) + write_csv(DATA_FOLDER, 'solana_nft_labels', df[['mint_address','collection']]) + +# print('Loaded!') +# mints_from_me() +# pull_from_metaboss() +# compile() +# how_rare_is_api() \ No newline at end of file diff --git a/viz/utils.py b/viz/utils.py index 86a199ca..73a98c6c 100644 --- a/viz/utils.py +++ b/viz/utils.py @@ -16,9 +16,12 @@ clean_names = { ,'boredapeyachtclub': 'BAYC' ,'mutantapeyachtclub': 'MAYC' ,'bayc': 'BAYC' + ,'bakc': 'BAKC' ,'mayc': 'MAYC' ,'solgods': 'SOLGods' ,'meerkatmillionairescc': 'Meerkat Millionaires' + ,'ggsg:galacticgeckos': 'Galactic Geckos' + ,'solstein': 'SolStein' # ,'stonedapecrew': 'Stoned Ape Crew' } @@ -47,6 +50,7 @@ def clean_name(name): name = re.sub('-', ' ', name) name = re.sub(' On ', ' on ', name) name = re.sub('Defi ', 'DeFi ', name) + # name = re.sub(r'[^a-zA-Z0-9\s]', '', name) return(name) From 8f8820d2f722a5a13de5bd5fda303cbb809411b0 Mon Sep 17 00:00:00 2001 From: flipside-kellen Date: Thu, 14 Jul 2022 15:13:46 -0700 Subject: [PATCH 2/2] automated --- viz/update_data.R | 1 + viz/upload_solana_nft_labels.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/viz/update_data.R b/viz/update_data.R index 49b1394f..661bc0c9 100644 --- a/viz/update_data.R +++ b/viz/update_data.R @@ -32,6 +32,7 @@ if(isRstudio) { mints_from_me() pull_from_metaboss() how_rare_is_api() +# saves labels to '/rstudio-data/nft_labels/solana_nft_labels.csv' compile() diff --git a/viz/upload_solana_nft_labels.py b/viz/upload_solana_nft_labels.py index a5e62fba..98efb648 100644 --- a/viz/upload_solana_nft_labels.py +++ b/viz/upload_solana_nft_labels.py @@ -644,8 +644,8 @@ def compile(): howrare_labels = read_csv(DATA_FOLDER, 'howrare_labels') df = pd.concat([howrare_labels, single_update_auth_labels, mult_update_auth_labels]) df = df[ (df.collection != 'Nan') & (df.collection != 'nan') & (df.collection.notnull()) ] + df = df[ (df.mint_address != 'Nan') & (df.mint_address != 'nan') & (df.mint_address.notnull()) ] df = df.drop_duplicates(subset=['mint_address'], keep='first') - print(len(df[df.collection == 'DegenTown'])) write_csv(DATA_FOLDER, 'solana_nft_labels', df[['mint_address','collection']]) # print('Loaded!')