diff --git a/commands.txt b/commands.txt new file mode 100644 index 00000000..5fc34bd7 --- /dev/null +++ b/commands.txt @@ -0,0 +1,32 @@ + +sudo cp ~/nft-deal-score/viz/ui.R /srv/shiny-server/nft-deal-score/ +sudo cp ~/nft-deal-score/viz/server.R /srv/shiny-server/nft-deal-score/ +sudo cp ~/nft-deal-score/viz/www/styles.css /srv/shiny-server/nft-deal-score/www/ +sudo cp ~/nft-deal-score/viz/data.Rdata /srv/shiny-server/nft-deal-score + +sudo touch /srv/shiny-server/nft-deal-score/ui.R +sudo touch /srv/shiny-server/nft-deal-score/server.R +sudo touch /srv/shiny-server/nft-deal-score/www/styles.css +sudo touch /srv/shiny-server/nft-deal-score/data.Rdata + +sudo cp ~/data_science/viz/thorchain-console/ui.R /srv/shiny-server/thorchain-console/ +sudo cp ~/data_science/viz/thorchain-console/server.R /srv/shiny-server/thorchain-console/ +sudo cp ~/data_science/viz/thorchain-console/www/styles.css /srv/shiny-server/thorchain-console/www/ +sudo cp ~/data_science/viz/thorchain-console/data.RData /srv/shiny-server/thorchain-console +sudo cp ~/data_science/viz/native-token-recycling/data.RData /srv/shiny-server/native-token-recycling + +sudo cp ~/dfk_calculator_data.RData /rstudio-data/ + +sudo cp ~/nft_deal_score_data.RData /rstudio-data/ +sudo cp ~/test.py /rstudio-data/ +python /rstudio-data/test.py +sudo touch /rstudio-data/dfk_calculator_data.RData + +sudo touch /srv/shiny-server/thorchain-console/ui.R +sudo touch /srv/shiny-server/thorchain-console/server.R +sudo touch /srv/shiny-server/thorchain-console/www/styles.css +sudo touch /srv/shiny-server/thorchain-console/data.RData +sudo touch /srv/shiny-server/native-token-recycling/data.RData + +sudo cp ~/data_science/viz/thorchain-console/server.R /srv/shiny-server/thorchain-console +sudo touch /srv/shiny-server/thorchain-console/server.R \ No newline at end of file diff --git a/convert_to_rdata.R b/convert_to_rdata.R index eef89f82..771faf61 100644 --- a/convert_to_rdata.R +++ b/convert_to_rdata.R @@ -2,26 +2,77 @@ library(data.table) library(dplyr) library(plotly) -pred_price <- read.csv('~/nft-deal-score/data/pred_price.csv') %>% as.data.table() +isRstudio <- Sys.info()[["user"]] == 'rstudio-connect' + +file.location <- ifelse( + isRstudio + , "/rstudio-data/" + , '~/git/nft-deal-score/viz/' +) + +read_csv <- function(fname) { + dir <- ifelse(isRstudio, '/rstudio-data/', '~/git/nft-deal-score/data/') + fname <- paste0(dir, fname) + dt <- read.csv(fname) %>% as.data.table() +} + +# load all csvs +pred_price <- read_csv('pred_price.csv') pred_price[, token_id := as.numeric(token_id) ] pred_price <- pred_price[ collection != 'meerkatmillionaires' ] pred_price <- pred_price[order(token_id)] -attributes <- read.csv('~/nft-deal-score/data/attributes.csv') %>% as.data.table() -feature_values <- read.csv('~/nft-deal-score/data/feature_values.csv') %>% as.data.table() -sales <- read.csv('~/nft-deal-score/data/model_sales.csv') %>% as.data.table() -listings <- read.csv('~/nft-deal-score/data/listings.csv') %>% as.data.table() -coefsdf <- read.csv('~/nft-deal-score/data/coefsdf.csv') %>% as.data.table() -tokens <- read.csv('~/nft-deal-score/data/tokens.csv') %>% as.data.table() +attributes <- read_csv('attributes.csv') +attributes[, feature_name := trimws(feature_name) ] +attributes[, feature_value := trimws(as.character(feature_value)) ] +feature_values <- read_csv('feature_values.csv') +sales <- read_csv('model_sales.csv') +listings <- read_csv('listings.csv') +coefsdf <- read_csv('coefsdf.csv') +tokens <- read_csv('tokens.csv') +tokens[, token_id := clean_token_id] +sales[, price := as.numeric(price)] +sales[, token_id := as.numeric(token_id)] +listings[, token_id := as.numeric(token_id)] +listings <- listings[ !(collection == 'Stoned Ape Crew' & token_id == 764) ] +listings <- listings[ !(collection == 'Solana Monkey Business' & token_id == 953) ] +tokens[, token_id := as.numeric(token_id)] +# manual adjustments to price +ids_1 <- attributes[ (collection == 'Aurory') & (feature_value == 'Solana Blob') ]$token_id +pred_price[ collection == 'Aurory' & token_id %in% eval(ids_1), pred_price := (pred_price * 0.8) ] -# save(pred_price, attributes, feature_values, sales, listings, coefsdf, tokens, file='data.Rdata') -save(pred_price, attributes, feature_values, sales, listings, coefsdf, tokens, file='~/nft-deal-score/viz/data.Rdata') -load('~/git/nft-deal-score/viz/data.Rdata') -write.csv(listings, '~/git/nft-deal-score/data/listings.csv', row.names=F) -write.csv(attributes, '~/git/nft-deal-score/data/attributes.csv', row.names=F) -write.csv(sales, '~/git/nft-deal-score/data/model_sales.csv', row.names=F) -write.csv(sales, '~/git/nft-deal-score/data/model_sales.csv', row.names=F) -write.csv(coefsdf, '~/git/nft-deal-score/data/coefsdf.csv', row.names=F) -write.csv(tokens, '~/git/nft-deal-score/data/tokens.csv', row.names=F) -write.csv(pred_price, '~/git/nft-deal-score/data/pred_price.csv', row.names=F) \ No newline at end of file +ids_2 <- attributes[ (collection == 'Aurory') & (feature_value == 'Long Blob Hair ') ]$token_id +pred_price[ collection == 'Aurory' & token_id %in% eval(ids_2), pred_price := (pred_price * 0.90) ] + +ids_3 <- attributes[ (collection == 'Aurory') & (grepl( 'Mask', feature_value, fixed = TRUE)) ]$token_id +pred_price[ collection == 'Aurory' & token_id %in% eval(ids_3), pred_price := (pred_price * 0.975) ] + +# filter for only collections that have all data +a <- unique(pred_price[, list(collection)]) +b <- unique(sales[, list(collection)]) +c <- unique(listings[, list(collection)]) +d <- merge(merge(a, b), c) + +pred_price <- merge(pred_price, d, by=c('collection')) +attributes <- merge(attributes, d, by=c('collection')) +feature_values <- merge(feature_values, d, by=c('collection')) +sales <- merge(sales, d, by=c('collection')) +listings <- merge(listings, d, by=c('collection')) +coefsdf <- merge(coefsdf, d, by=c('collection')) +tokens <- merge(tokens, d, by=c('collection')) + +save( + pred_price + , attributes + , feature_values + , sales + , listings + , coefsdf + , tokens + , file = paste0(file.location,'nft_deal_score_data.Rdata') +) +save( + listings + , file = paste0(file.location,'nft_deal_score_listings_data.Rdata') +) diff --git a/format_data.py b/format_data.py index 48c045b8..f2b71923 100644 --- a/format_data.py +++ b/format_data.py @@ -361,32 +361,46 @@ def solana(): collection = 'Cets On Creck' collection = 'Astrals' + + + metadata = pd.read_csv('./data/metadata.csv') - print(sorted(metadata.collection.unique())) - metadata = metadata[metadata.collection == collection] - print(sorted(metadata.collection.unique())) + # print(sorted(metadata.collection.unique())) + # metadata = metadata[metadata.collection == collection] + # print(sorted(metadata.collection.unique())) metadata = metadata[-(metadata.feature_name.isin(['adj_nft_rank_0','adj_nft_rank_1','adj_nft_rank_2','nft_rank']))] len(metadata.token_id.unique()) id_map = pd.read_csv('./data/mint_to_token_id_map.csv') id_map = pd.read_csv('./data/tokens.csv') - id_map = id_map[id_map.collection == collection] + cs = ['SOLGods'] + id_map = id_map[id_map.collection.isin(cs)] + metadata = metadata[metadata.collection.isin(cs)] + sorted(id_map.collection.unique()) + sorted(metadata.collection.unique()) - id_map['token_id'] = id_map.token_id.astype(int) - metadata['token_id'] = metadata.token_id.astype(int) + # id_map['token_id'] = id_map.token_id.astype(int) + # metadata['token_id'] = metadata.token_id.astype(int) + id_map['token_id'] = id_map.token_id.astype(str) + metadata['token_id'] = metadata.token_id.astype(str) - metadata = merge(metadata, id_map[['collection','token_id','mint_address','image_url']], ensure = True) + metadata = merge(metadata, id_map[['collection','token_id','mint_address','image_url']], ensure = False) + metadata = metadata[metadata.collection.isin(cs)] metadata['feature_name'] = metadata.feature_name.apply(lambda x: x.title() ) # metadata['image_url'] = metadata.token_id.apply(lambda x: 'https://metadata.degods.com/g/{}.png'.format(x - 1) ) metadata.head() + metadata = metadata[-(metadata.feature_name.isin(['Nft_Rank','Adj_Nft_Rank_0','Adj_Nft_Rank_1','Adj_Nft_Rank_2']))] + # print(metadata.groupby('feature_name').token_id.count().reset_index().sort_values('token_id', ascending=0).head(10)) metadata = metadata[metadata.feature_name != 'L3G3Nd4Ry'] - print(sorted(metadata.collection.unique())) - sorted(metadata[metadata.collection == collection].feature_name.unique()) + # print(sorted(metadata.collection.unique())) + # sorted(metadata[metadata.collection == collection].feature_name.unique()) + # sorted(metadata.feature_name.unique()) # metadata[['collection']].drop_duplicates().to_csv('~/Downloads/tmp.csv', index=False) + metadata['token_id'] = metadata.token_id.astype(int) for collection in metadata.collection.unique(): print(collection) mdf = metadata[metadata.collection == collection] diff --git a/load_data.py b/load_data.py index ebb85525..b6e27ee9 100644 --- a/load_data.py +++ b/load_data.py @@ -138,6 +138,7 @@ def mints_from_me(): offset += 500 lp_df = pd.DataFrame(lp_data) lp_df.to_csv('./data/me_lp_collections.csv', index=False) + lp_df = pd.read_csv('./data/me_lp_collections.csv') it = 0 l_data = [] @@ -249,10 +250,30 @@ def mints_from_me(): m_df.to_csv('./data/me_update_authorities.csv', index=False) m_df = pd.read_csv('./data/me_update_authorities.csv') + def f(x): + x = re.sub('\(|\)', '', x) + x = re.sub(' ', '_', x) + x = re.sub('\'', '', x) + return(x) + m_df['collection'] = m_df.name.apply(lambda x: f(x) ) + + x = 'asf (asf)' + f(x) + + seen = [ x for x in m_df.collection.unique() if os.path.exists('./data/mints/{}/'.format(x)) and len(os.listdir('./data/mints/{}/'.format(x))) ] + print(len(seen)) + # m_df = m_df.merge(lp_df) + len(m_df) + it = 0 rpc = 'https://red-cool-wildflower.solana-mainnet.quiknode.pro/a1674d4ab875dd3f89b34863a86c0f1931f57090/' - for row in m_df.iterrows(): + for row in m_df.sort_values('collection').iterrows(): + it += 1 + if it % 100 == 0: + print('#{}/{}'.format(it, len(m_df))) row = row[1] - collection = row['name'] + collection = row['collection'] + if collection in seen: + continue update_authority = row['update_authority'] print('Working on {}...'.format(collection)) collection_dir = re.sub(' ', '_', collection) @@ -260,7 +281,7 @@ def mints_from_me(): dir = './data/mints/{}/'.format(collection_dir) if not os.path.exists(dir): os.makedirs(dir) - else: + elif len(os.listdir(dir)): # print('Already have {}.'.format(collection)) print('Seen') continue @@ -278,7 +299,48 @@ def mints_from_me(): os.system('metaboss -r {} -t 300 decode mint --list-file {} --output {}'.format(rpc, fname, dir_mints)) + data = [] + for path in os.listdir('./data/mints/'): + if os.path.isdir('./data/mints/'+path): + collection = re.sub('_', ' ', path).strip() + for fname in os.listdir('./data/mints/'+path): + f = './data/mints/'+path+'/'+fname + if os.path.isfile(f) and '.json' in f: + with open(f) as file: + j = json.load(file) + for m in j: + data += [[ collection, m ]] + df = pd.DataFrame(data, columns=['collection','mint_address']) + df = df[df.collection != 'etc'] + df = df.drop_duplicates() + df['n'] = 1 + g = df.groupby(['mint_address']).n.sum().reset_index() + g = g[g.n > 1] + tmp_0 = g[['mint_address']].merge(df).groupby('collection').n.count().reset_index().sort_values('n', ascending=0) + tmp_0.tail(20) + tmp = g.merge(df[[ 'collection','mint_address' ]]) + tmp = tmp.sort_values(['mint_address','collection']) + tmp[tmp.collection == 'Fractals'] + df[df.mint_address == '11gATLu654HjcVhkuarVy9YVm11CL74vjEmi1RhojRi'] + # tmp.sort_values(['mint_address','collection']).head() + rem = tmp.collection.unique() + # len(rem) + # len(df.collection.unique()) + # tmp.head() + # tmp = g[['mint_address']].merge(df).groupby('collection').n.count().reset_index().sort_values('n', ascending=0) + # tmp[tmp.collection == 'Fractals'] + # tmp.head(20) + 'Fractals' in rem + 'Fractals' in df[-df.collection.isin(rem)].collection.unique() + print(sorted(df.collection.unique())) + len(df.collection.unique()) + sorted(df.collection.unique) + df[['collection']].drop_duplicates().sort_values('collection').to_csv('~/Downloads/tmp.csv', index=False) + len(df[-df.collection.isin(rem)]) + len(df[-df.collection.isin(rem)].drop_duplicates(subset=['mint_address'])) + df[-df.collection.isin(rem)].to_csv('./data/collection_mints.csv', index=False) len(df) + len(df.drop_duplicates(subset=['mint_address'])) len(df[df.twitter.notnull()]) len(df[ (df.twitter.notnull()) & (df.website.notnull())]) diff --git a/objects/saved_params.pickle b/objects/saved_params.pickle index e69de29b..44c19886 100644 Binary files a/objects/saved_params.pickle and b/objects/saved_params.pickle differ diff --git a/requirements.txt b/requirements.txt index fb4a06e2..4cf7a642 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,14 @@ -beautifulsoup4==4.10.0 +beautifulsoup4==4.11.1 +kutils==0.3.0 matplotlib==3.3.2 numpy==1.19.2 pandas==1.1.3 requests==2.24.0 -scikit_learn==1.0.1 +scikit_learn==1.0.2 +scipy==1.5.2 seaborn==0.11.0 selenium==3.141.0 snowflake==0.0.3 -snowflake_connector_python==2.4.3 +snowflake_connector_python==2.7.2 statsmodels==0.12.0 -tensorflow==2.6.0 +tensorflow==2.4.1 diff --git a/scrape_listings.R b/scrape_listings.R new file mode 100644 index 00000000..bfb2d5c5 --- /dev/null +++ b/scrape_listings.R @@ -0,0 +1,95 @@ + +isRstudio <- Sys.info()[["user"]] == 'rstudio-connect' +if(isRstudio) { + source("/home/data-science/data_science/util/util_functions.R") +} else { + source("~/data_science/util/util_functions.R") + setwd('~/git/nft-deal-score') +} + +library(httr) +library(jsonlite) + + +query <- " + SELECT DISTINCT project_name AS collection + , mint AS tokenMint + , token_id + FROM solana.dim_nft_metadata +" +mints <- QuerySnowflake(query) +colnames(mints) <- c('collection','tokenMint','token_id') + +collections <- c( + 'meerkat_millionaires_country_club','solgods','cets_on_creck','stoned_ape_crew','degods','aurory','thugbirdz','solana_monkey_business','degenerate_ape_academy','pesky_penguins' +) + +get_me_url <- function(collection, offset) { + return(paste0('https://api-mainnet.magiceden.dev/v2/collections/',collection,'/listings?offset=',offset,'&limit=20')) +} +get_smb_url <- function(page) { + return(paste0('https://market.solanamonkey.business/api/items?limit=40&page=',page)) +} + +new_listings <- data.table() +collection <- collections[1] +for(collection in collections) { + print(paste0('Working on ', collection, '...')) + has_more <- TRUE + offset <- 0 + while(has_more) { + print(paste0('Offset #', offset)) + url <- get_me_url(collection, offset) + response <- GET(url) + content <- rawToChar(response$content) + content <- fromJSON(content) + # content <- rbindlist(content, fill=T) + has_more <- nrow(content) >= 20 + if(nrow(content) > 0 && length(content) > 0) { + df <- merge(content, mints, by=c('tokenMint')) %>% as.data.table() + df <- df[, list(collection, token_id, price)] + offset <- offset + 20 + new_listings <- rbind(new_listings, df) + } else { + has_more <- FALSE + } + } +} + +for(collection in c('Solana Monkey Business')) { + print(paste0('Working on ', collection, '...')) + has_more <- TRUE + page <- 1 + while(has_more) { + print(paste0('Page #', page)) + url <- get_smb_url(page) + response <- GET(url) + content <- rawToChar(response$content) + content <- fromJSON(content) + # content <- rbindlist(content, fill=T) + content <- content %>% as.data.table() + has_more <- nrow(content) > 0 && 'price' %in% colnames(content) + if(has_more) { + content <- content[, list(mint, price)] + content <- unique(content) + content$price <- as.numeric(content$price) / (10^9) + has_more <- nrow(content) >= 40 + colnames(content)[1] <- 'tokenMint' + df <- merge(content, mints, by=c('tokenMint')) %>% as.data.table() + df <- df[, list(collection, token_id, price)] + page <- page + 1 + new_listings <- rbind(new_listings, df) + } + } +} +new_listings <- unique(new_listings) + +listings <- read.csv('./data/listings.csv') %>% as.data.table() +rem <- unique(new_listings$collection) +listings <- listings[ !(collection %in% eval(rem)), ] +listings <- listings[, list(collection, token_id, price)] +listings <- rbind(listings, new_listings) +listings <- listings[order(collection, price)] + +write.csv(listings, './data/listings.csv', row.names=F) + diff --git a/scrape_sol_nfts.py b/scrape_sol_nfts.py index db45b90c..fcfaee02 100644 --- a/scrape_sol_nfts.py +++ b/scrape_sol_nfts.py @@ -40,14 +40,22 @@ def how_rare_is_api(): j.keys() t_data = [] metadata = pd.DataFrame() - for d in j['result']['data'][8:]: - collection = 'Cets on Creck' - collection = 'SOLGods' - collection = 'Meerkat Millionaires' - collection = d['url'][1:] + d = { + 'Degen Apes': 'degenapes' + , 'Pesky Penguins': 'peskypenguinclub' + , 'Aurory': 'aurory' + , 'Solana Monkey Business': 'smb' + , 'Thugbirdz': 'thugbirdz' + } + for collection, url in d.items(): + # collection = 'Cets on Creck' + # collection = 'SOLGods' + # collection = 'Meerkat Millionaires' + # collection = d['url'][1:] print('Working on collection {}, {}, {}'.format(collection, len(t_data), len(metadata))) - url = 'https://api.howrare.is/v0.1/collections'+d['url'] - url = 'https://api.howrare.is/v0.1/collections/meerkatmillionaires' + # url = 'https://api.howrare.is/v0.1/collections'+d['url'] + # url = 'https://api.howrare.is/v0.1/collections/meerkatmillionaires' + url = 'https://api.howrare.is/v0.1/collections/'+url r = requests.get(url) j = r.json() for i in j['result']['data']['items']: @@ -64,27 +72,50 @@ def how_rare_is_api(): old = pd.read_csv('./data/tokens.csv') sorted(old.collection.unique()) l0 = len(old) + do_merge = True tokens = pd.DataFrame(t_data, columns=['collection','token_id','nft_rank','mint_address','image_url']) - # old = old.merge(tokens, how='left', on=['collection','token_id']) - old = old.append(tokens) - # old['nft_rank'] = old.nft_rank_y.fillna(old.nft_rank_y) - # del old['nft_rank_x'] - # del old['nft_rank_y'] + if do_merge: + old['token_id'] = old.token_id.astype(str) + tokens['token_id'] = tokens.token_id.astype(str) + old = old.merge(tokens, how='left', on=['collection','token_id']) + old[old.collection == 'Solana Monkey Business'] + for c in [ 'nft_rank','mint_address','image_url' ]: + old[c] = old[c+'_x'].fillna(old[c+'_y']) + del old[c+'_x'] + del old[c+'_y'] + old['clean_token_id'] = old.clean_token_id.fillna(old.token_id) + old['chain'] = old.chain.fillna('Solana') + else: + old = old.append(tokens) print('Adding {} rows'.format(len(old) - l0)) + old[old.nft_rank.isnull()].groupby('collection').token_id.count() old.to_csv('./data/tokens.csv', index=False) old = pd.read_csv('./data/metadata.csv') - old = old[-(old.collection == 'Meerkat Millionaires Cc')] + a = old[['collection','token_id']].drop_duplicates() + a['exclude'] = 0 + a['token_id'] = a.token_id.astype(str) + metadata['token_id'] = metadata.token_id.astype(str) + m = metadata.merge(a, how='left') + m = m[m.exclude.isnull()] + len(m[m.exclude.isnull()].token_id.unique()) + del m['exclude'] + # old = old[-(old.collection == 'Meerkat Millionaires Cc')] print(sorted(old.collection.unique())) l0 = len(old) metadata.collection.unique() # metadata = pd.DataFrame(t_data, columns=['collection','token_id','nft_rank','mint_address','image_url']) # old = old.merge(tokens, how='left', on=['collection','token_id']) - old = old.append(metadata) + old = old.append(m[['collection','token_id','name','value']].rename(columns={'name':'feature_name','value':'feature_value'}) ) + old['token_id'] = old.token_id.astype(str) + old = old.drop_duplicates(subset=['collection','token_id','feature_name']) # old['nft_rank'] = old.nft_rank_y.fillna(old.nft_rank_y) # del old['nft_rank_x'] # del old['nft_rank_y'] print('Adding {} rows'.format(len(old) - l0)) + print(old.groupby('collection').token_id.count()) + old[old.collection.isin(metadata.collection)] + old[(old.collection == 'Thugbirdz') & (old.token_id == '1206')] old.to_csv('./data/metadata.csv', index=False) @@ -580,7 +611,7 @@ def scrape_opensea_listings(browser, collections=['BAYC','MAYC']): old.groupby('collection').token_id.count() old.to_csv('./data/listings.csv', index=False) -def scrape_listings(browser, collections = [ 'solgods','cets-on-creck','stoned-ape-crew','degods','aurory','thugbirdz','smb','degenapes','peskypenguinclub' ], alerted = [], is_listings = True): +def scrape_listings(browser, collections = [ 'meerkat-millionaires-cc','solgods','cets-on-creck','stoned-ape-crew','degods','aurory','thugbirdz','smb','degenapes','peskypenguinclub' ], alerted = [], is_listings = True): print('Scraping solanafloor listings...') data = [] m_data = [] @@ -775,6 +806,7 @@ def scrape_listings(browser, collections = [ 'solgods','cets-on-creck','stoned-a old = pd.read_csv('./data/listings.csv') listings = pd.DataFrame(data, columns=['collection','token_id','price']).drop_duplicates() + listings.groupby('collection').price.min() # others = scrape_magic_eden() # listings = listings.append(others).drop_duplicates() # d = { @@ -788,10 +820,13 @@ def scrape_listings(browser, collections = [ 'solgods','cets-on-creck','stoned-a # ,'degods': 'DeGods' # } listings['collection'] = listings.collection.apply(lambda x: clean_name(x)) + listings.groupby('collection').price.min() + listings.groupby('collection').price.count() listings[listings.token_id=='1656'] listings[listings.token_id==484] old = old[ -(old.collection.isin(listings.collection.unique())) ] + old = old[old.collection != 'Meerkat Millionaires Cc'] pred_price = pd.read_csv('./data/pred_price.csv') listings.token_id.values[:3] pred_price.token_id.values[:3] diff --git a/scratch.sql b/scratch.sql index 93f1d9a9..df84125a 100644 --- a/scratch.sql +++ b/scratch.sql @@ -1,3 +1,380 @@ + + +WITH base AS ( + SELECT native_to_address, COUNT(1) AS n + FROM thorchain.swaps + WHERE block_timestamp >= '2022-04-01' + GROUP BY 1 +) +select b.n, s.* from thorchain.swaps s +JOIN base b ON b.native_to_address = s.native_to_address +where block_timestamp >= '2022-04-10' +ORDER BY block_timestamp, n, tx_id +LIMIT 100 + + + +SELECT MIN(block_timestamp) AS mn +, MAX(block_timestamp) AS mx +, COUNT(1) AS n +FROM flipside_dev_db.thorchain.liquidity_actions + + +SELECT MIN(block_timestamp) AS mn +, MAX(block_timestamp) AS mx +, COUNT(1) AS n +FROM flipside_dev_db.thorchain.liquidity_actions + + +SELECT MIN(block_timestamp) AS mn +, MAX(block_timestamp) AS mx +, COUNT(1) AS n +FROM flipside_dev_db.thorchain.bond_actions + +SELECT + + + +https://app.flipsidecrypto.com/dashboard/small-lp-actions-LD1XQ9 + + +https://app.flipsidecrypto.com/dashboard/pool-ranks-UbtLg9 +https://app.flipsidecrypto.com/dashboard/price-shift-_-JpTq +https://app.flipsidecrypto.com/dashboard/tho-rchain-average-age-of-synth-holders-Z2AXIx +https://app.flipsidecrypto.com/dashboard/thor-64-standardized-tvl-over-time-all-pools-Zf6w-L +https://app.flipsidecrypto.com/dashboard/tho-rchain-pool-ranks-RNNzza +https://app.flipsidecrypto.com/dashboard/thorchain-synth-mints-burns-aH0lCY + +https://discord.com/channels/889577356681945098/889577399308656662/951960381411192842 + +solana-keygen recover 'prompt:?key=3/0' --outfile ~/.config/solana/tmp.json + + +SELECT * +, instructions[0]:parsed:info:lamports / POWER(10, 9) AS sol_amount +FROM solana.fact_transactions +WHERE block_timestamp >= '2022-01-02' +AND tx_id = '5H6UQqbxa2wtryax6SAZgjXB9B6Za4ip6GsheqopAsbLCrLMPvYf35H551SaAKNy6bi6BceRGtkwwP9LRoN7RiVo' + + +SELECT * +FROM solana.fact_transfers +WHERE block_timestamp >= '2022-01-02' +AND tx_id = '393xRouisz4DuMxzARAqPy7FVYQZtkfpmAMuDXXj39uvASFYtMijHM9hyVzXSocsB4fk2woLNfnWTM4qXJxJsWBw' + +SELECT lp.tx_id +, lp.signers[0] as signer +, t.instructions[0]:parsed:info:lamports / POWER(10, 9) AS sol_amount +FROM solana.fact_staking_lp_actions lp +JOIN solana.fact_transactions t ON t.tx_id = lp.tx_id +WHERE lp.block_timestamp >= '2022-01-01' +AND lp.event_type = 'delegate' +AND tx_id = '393xRouisz4DuMxzARAqPy7FVYQZtkfpmAMuDXXj39uvASFYtMijHM9hyVzXSocsB4fk2woLNfnWTM4qXJxJsWBw' +LIMIT 10 + +sudo apt-get build-dep python E: You must put some 'deb-src' URIs in your sources.list + + +SELECT project_name +, COUNT(1) AS n +, SUM( CASE WHEN address IS NULL THEN 1 ELSE 0 END) AS n_nulls +FROM crosschain.address_labels +WHERE blockchain = 'solana' +AND label_subtype = 'nf_token_contract' +GROUP BY 1 + +-- MEv1 De-Listing +WITH mints AS ( + SELECT DISTINCT project_name + , mint + , token_id + FROM solana.dim_nft_metadata +) +SELECT pre_token_balances[0]:mint::string AS mint +, t.* +, m.project_name +, m.token_id +FROM solana.fact_transactions t +JOIN mints m ON m.mint = t.pre_token_balances[0]:mint::string +WHERE block_timestamp >= CURRENT_DATE - 30 +AND tx_id = '3CxhnTCXYX1zH6HbNESFsZLwdHfTe7RUYF8tAgB168hciVjUGggp2PwVEsnDvpd2kNqMha7kH2be7NtSTppAnXzn' +AND instructions[0]:data = 'TE6axTojnpk' +AND instructions[0]:programId = 'MEisE1HzehtrDpAAT8PnLHjpSSkRYakotTuJRPjTpo8' +AND succeeded = TRUE +LIMIT 100 + +ENwHiaH9NA9veUqRzGozjWnTuR9xcNvHcPZVFMi3Ca9L + + +WITH mints AS ( + SELECT DISTINCT project_name + , mint + , token_id + FROM solana.dim_nft_metadata +) +SELECT instructions[0]:data AS data +, COUNT(1) AS n +, COUNT(1) AS nn +FROM solana.fact_transactions t +JOIN mints m ON m.mint = t.pre_token_balances[0]:mint::string +WHERE block_timestamp >= '2022-04-17' +AND instructions[0]:programId = 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K' +AND succeeded = TRUE +GROUP BY 1 +ORDER BY 2 DESC + + +-- MEv2 De-Listing +WITH mints AS ( + SELECT DISTINCT project_name + , mint + , token_id + FROM solana.dim_nft_metadata +), rem AS ( + SELECT pre_token_balances[0]:mint::string AS mint + , m.project_name + , m.token_id + , t.tx_id AS remove_tx + , block_timestamp AS remove_time + , ROW_NUMBER() OVER (PARTITION BY mint ORDER BY block_timestamp DESC) AS rn + FROM solana.fact_transactions t + JOIN mints m ON m.mint = t.pre_token_balances[0]:mint::string + WHERE block_timestamp >= CURRENT_DATE - 3 + AND LEFT(instructions[0]:data::string, 4) IN ('ENwH','3GyW') + AND instructions[0]:programId = 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K' +), add AS ( + SELECT pre_token_balances[0]:mint::string AS mint + , m.project_name + , m.token_id + , t.tx_id AS listing_tx + , block_timestamp AS listing_time + , ROW_NUMBER() OVER (PARTITION BY mint ORDER BY block_timestamp DESC) AS rn + FROM solana.fact_transactions t + JOIN mints m ON m.mint = t.pre_token_balances[0]:mint::string + WHERE block_timestamp >= CURRENT_DATE - 3 + AND LEFT(instructions[0]:data::string, 4) IN ('2B3v') + AND instructions[0]:programId = 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K' + AND succeeded = TRUE +) +SELECT a.* +, r.remove_tx +, r.remove_time +, CASE WHEN r.remove_time IS NULL OR a.listing_time > r.remove_time THEN 1 ELSE 0 END AS is_listed +FROM add a +LEFT JOIN rem r ON r.mint = a.mint AND r.rn = 1 +WHERE a.rn = 1 + +thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj + +WITH base AS ( + SELECT from_address AS address + , block_timestamp::date AS date + , -from_amount AS amount + , 'From Swap' AS tx_type + FROM thorchain.swaps + WHERE from_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj' + AND from_asset = 'THOR.RUNE' + + UNION ALL + + SELECT native_to_address AS address + , block_timestamp::date AS date + , to_amount AS amount + , 'To Swap' AS tx_type + FROM thorchain.swaps + WHERE native_to_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj' + AND to_asset = 'THOR.RUNE' + + UNION ALL + + SELECT from_address AS address + , block_timestamp::date AS date + , -rune_amount AS amount + , 'From Transfer' AS tx_type + FROM thorchain.transfers + WHERE from_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj' + + UNION ALL + + SELECT to_address AS address + , block_timestamp::date AS date + , rune_amount AS amount + , 'To Transfer' AS tx_type + FROM thorchain.transfers + WHERE to_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj' +) +SELECT * +FROM base +ORDER BY date DESC + +68. [Hard] $RUNE Distribution +Calculate the distribution of $RUNE holdings by address. Include RUNE that is in liquidity pools. (e.g. if I have 100 $RUNE in my wallet and LP 50, I should still be considered to be holding 100). In terms of charts, feel free to create a histogram or whichever visual you think works best! + +Hint: use thorchain.transfers and thorchain.liquidity_actions + +70. [Hard] Weighted-average LP duration +What is the average duration of liquidity held in each pool, weighted by the size of the LP? + +Hint: use thorchain.liquidity_actions + +69. [Medium] LP Size Distribution +What is the current distribution of LP size for each pool? You will have to use both 'add_liquidity' and 'remove_liquidity' to determine which LPs are stil current. + +Hint: use thorchain.liquidity_actions + +71. [Easy] Block Rewards vs Swap Fees +Breakdown the yield from block rewards vs swap fees, both total and by pool. Show how the proportions have changed over time since the network started. + +Hint: use thorchain.block_rewards + + +72. [Medium] Swap Volume vs LP Rewards +Chart the weekly swap volume and LP rewards on the same chart. What is the relationship between the two? + +Hint: use thorchain.daily_pool_stats to get swap volume and earnings_to_pools from thorchain.daily_earnings to get LP rewards + +73. [Hard] Realized APY +Visualize the realized APY of LP-ers that have removed liquidity from THORChain. What was their actual APY vs HODL (accounting for impermanent loss) when you pro-rate it for how long they were LP-ing for? Are certain pools out-performing others? + +Hint: use thorchain.liquidity_actions + + +SELECT * +FROM THORCHAIN.DAILY_POOL_STATS +WHERE pool_name ilike '%luna%' +ORDER BY day DESC + + +WITH base AS ( + SELECT from_address AS address + , block_timestamp::date AS date + , rune_amount AS lp_amount + , 'Add LP' AS tx_type + FROM thorchain.liquidity_actions + WHERE from_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj' + AND lp_action = 'add_liquidity' + + UNION ALL + + SELECT from_address AS address + , block_timestamp::date AS date + , -rune_amount AS lp_amount + , 'Rem LP' AS tx_type + FROM thorchain.liquidity_actions + WHERE from_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj' + AND lp_action = 'remove_liquidity' + + UNION ALL + + SELECT native_to_address AS address + , block_timestamp::date AS date + , to_amount AS amount + , 'To Swap' AS tx_type + FROM thorchain.swaps + WHERE native_to_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj' + AND to_asset = 'THOR.RUNE' + + UNION ALL + + SELECT from_address AS address + , block_timestamp::date AS date + , -rune_amount AS amount + , 'From Transfer' AS tx_type + FROM thorchain.transfers + WHERE from_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj' + + UNION ALL + + SELECT to_address AS address + , block_timestamp::date AS date + , rune_amount AS amount + , 'To Transfer' AS tx_type + FROM thorchain.transfers + WHERE to_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj' +) +SELECT * +FROM base +ORDER BY date DESC + +-- 2B3v: listing +-- ENwH: de-listing +-- 3GyW: sale + +WITH mints AS ( + SELECT DISTINCT project_name + , mint + , token_id + FROM solana.dim_nft_metadata +) +SELECT pre_token_balances[0]:mint::string AS mint +, t.* +, m.project_name +, m.token_id +FROM solana.fact_transactions t +JOIN mints m ON m.mint = t.pre_token_balances[0]:mint::string +WHERE block_timestamp >= CURRENT_DATE - 2 +AND tx_id = '3CxhnTCXYX1zH6HbNESFsZLwdHfTe7RUYF8tAgB168hciVjUGggp2PwVEsnDvpd2kNqMha7kH2be7NtSTppAnXzn' +AND LEFT(instructions[0]:data::string, 4) IN ('2B3v') +AND instructions[0]:programId = 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K' +AND succeeded = TRUE +LIMIT 100 + + + + + + +SELECT MIN(block_timestamp) AS mn +, MAX(block_timestamp) AS mx +, COUNT(1) AS n +FROM flipside_prod_db.thorchain.liquidity_actions + + +SELECT MIN(block_timestamp) AS mn +, MAX(block_timestamp) AS mx +, COUNT(1) AS n +FROM flipside_prod_db.thorchain.prices + +liquidity_actions +bond_actions + + +SELECT MIN(block_timestamp) AS mn +, MAX(block_timestamp) AS mx +, COUNT(1) AS n +FROM flipside_prod_db.thorchain.bond_actions + + +SELECT MIN(block_timestamp) AS mn +, MAX(block_timestamp) AS mx +, COUNT(1) AS n +FROM flipside_prod_db.thorchain.pool_block_balances +WHERE rune_amount > 0 and COALESCE(rune_amount_usd, 0) = 0 + +SELECT MIN(block_timestamp) AS mn +, MAX(block_timestamp) AS mx +, COUNT(1) AS n +FROM flipside_prod_db.thorchain.pool_block_balances +WHERE asset_amount > 0 and COALESCE(asset_amount_usd, 0) = 0 + +SELECT * +FROM flipside_prod_db.thorchain.pool_block_balances +WHERE (asset_amount > 0 and COALESCE(asset_amount_usd, 0) = 0) +OR (rune_amount > 0 and COALESCE(rune_amount_usd, 0) = 0) +ORDER BY block_timestamp DESC +LIMIT 10000 + +SELECT block_timestamp::date AS date +, COUNT(1) AS n +FROM flipside_prod_db.thorchain.pool_block_balances +WHERE (asset_amount > 0 and COALESCE(asset_amount_usd, 0) = 0) +OR (rune_amount > 0 and COALESCE(rune_amount_usd, 0) = 0) +ORDER BY block_timestamp DESC +GROUP BY 1 +ORDER BY 1 + + + WITH active_vault_events_cte AS (SELECT max(block_timestamp)::string AS recency, min(block_timestamp)::string AS start_time, 'active_vault_events' AS "table" diff --git a/solana_model.py b/solana_model.py index 74cc3d7e..1e14c027 100644 --- a/solana_model.py +++ b/solana_model.py @@ -364,12 +364,12 @@ def train_model(check_exclude=False, supplement_with_listings=True, use_saved_pa collections = [ x for x in collections if not x in ['Bakc','BAKC','MAYC'] ] collections = [ x for x in collections if not x in ['Astrals','Cets on Cleck','DeFi Pirates'] ] collections = ['Cets on Creck'] - collections = list(s_df[['collection']].drop_duplicates().merge(m_df[['collection']].drop_duplicates()).collection.unique()) s_df.groupby('collection').block_timestamp.max() collections = ['Meerkat Millionaires'] + collections = list(s_df[['collection']].drop_duplicates().merge(m_df[['collection']].drop_duplicates()).collection.unique()) print(sorted(collections)) for collection in collections: - if collection in ['Astrals','BAYC','MAYC']: + if collection in ['Astrals','Bakc','BAYC','MAYC']: continue if not collection in saved_params.keys(): saved_params[collection] = {} @@ -381,7 +381,7 @@ def train_model(check_exclude=False, supplement_with_listings=True, use_saved_pa print('\nWorking on collection {}'.format(collection)) sales = s_df[ s_df.collection == collection ] sales[sales.sim==0].block_timestamp.max() - metadata = m_df[ m_df.collection == collection ].drop_duplicates() + metadata = m_df[ m_df.collection == collection ].drop_duplicates(subset=['token_id','feature_name'], keep='last') metadata = metadata[metadata.feature_name != 'Genesis Role?'] metadata[metadata.token_id=='1'] metadata[metadata.feature_name=='Genesis Role?'].feature_value.unique() @@ -629,7 +629,14 @@ def train_model(check_exclude=False, supplement_with_listings=True, use_saved_pa mn = coefs.val.min() if mn >= 0: df, bst_p, bst_r = ku.get_bst_params( model, df, X_new, y, target_col, col, verbose = True, wt_col='wt', params = [bst_p] ) + coefs['col'] = coefs.col.apply(lambda x: re.sub('std_', '', x) ) + coefs['n'] = 0 + n = pd.DataFrame() + for c in cat_metadata.columns: + if not c in [ 'collection','token_id' ]: + coefs.loc[ coefs.col == c, 'n' ] = len(cat_metadata[cat_metadata[c] == 1]) coefs.to_csv('./data/coefs/{}_{}_{}.csv'.format(collection, model, it), index=False) + test = ku.apply_model( model, bst_p, df, test, cur_std_pred_cols, target_col, col) if model in ['rfr']: df[col] = df[col] + df['rarity_value_'+it] diff --git a/update.py b/update.py index 6693a07e..a0b9941c 100644 --- a/update.py +++ b/update.py @@ -52,13 +52,16 @@ ld.add_solana_sales() ld.add_eth_sales() # update listings -# ssn.scrape_listings(browser, ['smb']) -# ssn.scrape_listings(browser, ['cets-on-creck']) +# ssn.scrape_listings(browser, ['meerkat-millionaires-cc']) +# ssn.scrape_listings(browser, ['thugbirdz']) ssn.scrape_listings(browser) ssn.scrape_randomearth(browser) ssn.scrape_opensea_listings(browser) # ssn.scrape_listings(browser, ['smb','aurory']) +listings = pd.read_csv('./data/listings.csv') +listings[listings.collection == 'Solana Monkey Business'].sort_values('price').head(20) + # update model # ssn.convert_collection_names() # sm.train_model(True, False) diff --git a/update_data.R b/update_data.R new file mode 100644 index 00000000..8bcfc110 --- /dev/null +++ b/update_data.R @@ -0,0 +1,14 @@ + +install.packages('reticulate') +library(reticulate) + +switch( + Sys.info()[["user"]], + "rstudio-connect" = source("/home/data-science/data_science/util/util_functions.R"), + source("~/data_science/util/util_functions.R") +) + +# use_python("/usr/local/bin/python") + +py_run_file("script.py") + diff --git a/utils.py b/utils.py index 5c1f0c77..86a199ca 100644 --- a/utils.py +++ b/utils.py @@ -40,7 +40,7 @@ def clean_token_id(df): def clean_name(name): x = re.sub('-', '', name).lower() - x = re.sub(' ', '', name).lower() + x = re.sub(' ', '', x).lower() if x in clean_names.keys(): return(clean_names[x]) name = name.title() diff --git a/viz/global.R b/viz/global.R index 97c086c9..a0264728 100644 --- a/viz/global.R +++ b/viz/global.R @@ -1,5 +1,5 @@ # setwd('~/git/nft-deal-score/viz') -source("~/data_science/util/util_functions.R") +# source("~/data_science/util/util_functions.R") library(data.table) library(shiny) library(ggplot2) diff --git a/viz/server.R b/viz/server.R index 39a33bc5..61c36667 100644 --- a/viz/server.R +++ b/viz/server.R @@ -1,5 +1,14 @@ server <- function(input, output, session) { - load('data.Rdata') + # load('data.Rdata') + + file.location <- ifelse( + Sys.info()[["user"]] == "rstudio-connect" + , "/rstudio-data/" + , '~/git/nft-deal-score/viz/' + ) + load(paste0(file.location, 'nft_deal_score_data.RData')) + load(paste0(file.location, 'nft_deal_score_listings_data.RData')) + # load(paste0(file.location, 'data.RData')) metadata <- unique(attributes[, list(collection, feature_name, feature_value)]) @@ -81,6 +90,20 @@ server <- function(input, output, session) { ) }) + output$minfloorinput <- renderUI({ + textInput( + inputId = 'minfloorinput' + , label = NULL + , width = "100%" + ) + }) + output$maxfloorinput <- renderUI({ + textInput( + inputId = 'maxfloorinput' + , label = NULL + , width = "100%" + ) + }) output$maxrarityrankinput2 <- renderUI({ textInput( inputId = 'maxrarityrank2' @@ -509,10 +532,12 @@ server <- function(input, output, session) { } t <- '' if (nrow(data)) { - p <- format(round(mean(head(data$price, 100)), 1), big.mark=',') - f <- format(round(mean(head(data$vs_floor, 100)), 1), big.mark=',') + data <- head(data, 100) + p <- format(round(mean(data$price), 1), big.mark=',') + f <- format(round(mean(data$vs_floor), 1), big.mark=',') data[, pct_vs_floor := (vs_floor + price) / price ] - pct <- format(round(mean(head(data$pct_vs_floor, 100)), 1), big.mark=',') + pct <- sum(data$price) / sum(data$mn_20) + pct <- format(round(pct, 1), big.mark=',') chain <- getChain() currency <- ifelse( chain == 'Solana', 'SOL', ifelse(chain == 'Ethereum', 'ETH', 'LUNA') ) t <- paste0(p, ' $',currency,' (+',f,' / ',pct,'x vs the floor)') @@ -823,6 +848,18 @@ server <- function(input, output, session) { r <- as.numeric(input$maxrarityrank2) data <- data[ nft_rank <= eval(r) ] } + if(input$minfloorinput != '') { + r <- as.numeric(input$minfloorinput) + data <- data[ minfloorinput >= eval(r) ] + } + if(input$maxfloorinput != '') { + r <- as.numeric(input$maxfloorinput) + data <- data[ maxfloorinput <= eval(r) ] + } + if(input$maxrarityrank2 != '') { + r <- as.numeric(input$maxrarityrank2) + data <- data[ nft_rank <= eval(r) ] + } if(input$minrarityrank2 != '') { data <- data[ nft_rank >= eval(as.numeric(input$minrarityrank2)) ] } diff --git a/viz/ui.R b/viz/ui.R index a1d553e1..f01cbc02 100644 --- a/viz/ui.R +++ b/viz/ui.R @@ -146,34 +146,48 @@ fluidPage( , bsTooltip(id = "historical-sales-tooltip", title = "This app is still in beta - sales data may be incomplete or delayed", placement = "bottom", trigger = "hover") , fluidRow( class = 'filters' - , column(3 + , column(2 , div( class = "inputtitle" , "Min Deal Score Rank" ) , fluidRow(uiOutput("minnftrankinput2")) ) - , column(3 + , column(2 , div( class = "inputtitle" , "Max Deal Score Rank" ) , fluidRow(uiOutput("maxnftrankinput2")) ) - , column(3 + , column(2 , div( class = "inputtitle" , "Min Rarity Rank" ) , fluidRow(uiOutput("minrarityrankinput2")) ) - , column(3 + , column(2 , div( class = "inputtitle" , "Max Rarity Rank" ) , fluidRow(uiOutput("maxrarityrankinput2")) ) + , column(2 + , div( + class = "inputtitle" + , "Min Floor" + ) + , fluidRow(uiOutput("minfloorinput")) + ) + , column(2 + , div( + class = "inputtitle" + , "Max Floor" + ) + , fluidRow(uiOutput("maxfloorinput")) + ) , column(3 , div( class = "inputtitle" diff --git a/viz/update_data.R b/viz/update_data.R new file mode 100644 index 00000000..e72b7acd --- /dev/null +++ b/viz/update_data.R @@ -0,0 +1,8 @@ + +isRstudio <- Sys.info()[["user"]] == 'rstudio-connect' +if(isRstudio) { + source("/home/data-science/data_science/util/util_functions.R") +} else { + source('~/git/nft-deal-score/scrape_listings.R') +} +source("/home/data-science/data_science/util/util_functions.R") \ No newline at end of file