mirror of
https://github.com/FlipsideCrypto/nft-deal-score.git
synced 2026-02-06 10:56:58 +00:00
updating data struture
This commit is contained in:
flipside-kellen
parent
2a1fbdfd04
commit
4d2df03ede
32
commands.txt
Normal file
32
commands.txt
Normal file
@ -0,0 +1,32 @@
|
||||
|
||||
sudo cp ~/nft-deal-score/viz/ui.R /srv/shiny-server/nft-deal-score/
|
||||
sudo cp ~/nft-deal-score/viz/server.R /srv/shiny-server/nft-deal-score/
|
||||
sudo cp ~/nft-deal-score/viz/www/styles.css /srv/shiny-server/nft-deal-score/www/
|
||||
sudo cp ~/nft-deal-score/viz/data.Rdata /srv/shiny-server/nft-deal-score
|
||||
|
||||
sudo touch /srv/shiny-server/nft-deal-score/ui.R
|
||||
sudo touch /srv/shiny-server/nft-deal-score/server.R
|
||||
sudo touch /srv/shiny-server/nft-deal-score/www/styles.css
|
||||
sudo touch /srv/shiny-server/nft-deal-score/data.Rdata
|
||||
|
||||
sudo cp ~/data_science/viz/thorchain-console/ui.R /srv/shiny-server/thorchain-console/
|
||||
sudo cp ~/data_science/viz/thorchain-console/server.R /srv/shiny-server/thorchain-console/
|
||||
sudo cp ~/data_science/viz/thorchain-console/www/styles.css /srv/shiny-server/thorchain-console/www/
|
||||
sudo cp ~/data_science/viz/thorchain-console/data.RData /srv/shiny-server/thorchain-console
|
||||
sudo cp ~/data_science/viz/native-token-recycling/data.RData /srv/shiny-server/native-token-recycling
|
||||
|
||||
sudo cp ~/dfk_calculator_data.RData /rstudio-data/
|
||||
|
||||
sudo cp ~/nft_deal_score_data.RData /rstudio-data/
|
||||
sudo cp ~/test.py /rstudio-data/
|
||||
python /rstudio-data/test.py
|
||||
sudo touch /rstudio-data/dfk_calculator_data.RData
|
||||
|
||||
sudo touch /srv/shiny-server/thorchain-console/ui.R
|
||||
sudo touch /srv/shiny-server/thorchain-console/server.R
|
||||
sudo touch /srv/shiny-server/thorchain-console/www/styles.css
|
||||
sudo touch /srv/shiny-server/thorchain-console/data.RData
|
||||
sudo touch /srv/shiny-server/native-token-recycling/data.RData
|
||||
|
||||
sudo cp ~/data_science/viz/thorchain-console/server.R /srv/shiny-server/thorchain-console
|
||||
sudo touch /srv/shiny-server/thorchain-console/server.R
|
||||
@ -2,26 +2,77 @@ library(data.table)
|
||||
library(dplyr)
|
||||
library(plotly)
|
||||
|
||||
pred_price <- read.csv('~/nft-deal-score/data/pred_price.csv') %>% as.data.table()
|
||||
isRstudio <- Sys.info()[["user"]] == 'rstudio-connect'
|
||||
|
||||
file.location <- ifelse(
|
||||
isRstudio
|
||||
, "/rstudio-data/"
|
||||
, '~/git/nft-deal-score/viz/'
|
||||
)
|
||||
|
||||
read_csv <- function(fname) {
|
||||
dir <- ifelse(isRstudio, '/rstudio-data/', '~/git/nft-deal-score/data/')
|
||||
fname <- paste0(dir, fname)
|
||||
dt <- read.csv(fname) %>% as.data.table()
|
||||
}
|
||||
|
||||
# load all csvs
|
||||
pred_price <- read_csv('pred_price.csv')
|
||||
pred_price[, token_id := as.numeric(token_id) ]
|
||||
pred_price <- pred_price[ collection != 'meerkatmillionaires' ]
|
||||
pred_price <- pred_price[order(token_id)]
|
||||
|
||||
attributes <- read.csv('~/nft-deal-score/data/attributes.csv') %>% as.data.table()
|
||||
feature_values <- read.csv('~/nft-deal-score/data/feature_values.csv') %>% as.data.table()
|
||||
sales <- read.csv('~/nft-deal-score/data/model_sales.csv') %>% as.data.table()
|
||||
listings <- read.csv('~/nft-deal-score/data/listings.csv') %>% as.data.table()
|
||||
coefsdf <- read.csv('~/nft-deal-score/data/coefsdf.csv') %>% as.data.table()
|
||||
tokens <- read.csv('~/nft-deal-score/data/tokens.csv') %>% as.data.table()
|
||||
attributes <- read_csv('attributes.csv')
|
||||
attributes[, feature_name := trimws(feature_name) ]
|
||||
attributes[, feature_value := trimws(as.character(feature_value)) ]
|
||||
feature_values <- read_csv('feature_values.csv')
|
||||
sales <- read_csv('model_sales.csv')
|
||||
listings <- read_csv('listings.csv')
|
||||
coefsdf <- read_csv('coefsdf.csv')
|
||||
tokens <- read_csv('tokens.csv')
|
||||
tokens[, token_id := clean_token_id]
|
||||
sales[, price := as.numeric(price)]
|
||||
sales[, token_id := as.numeric(token_id)]
|
||||
listings[, token_id := as.numeric(token_id)]
|
||||
listings <- listings[ !(collection == 'Stoned Ape Crew' & token_id == 764) ]
|
||||
listings <- listings[ !(collection == 'Solana Monkey Business' & token_id == 953) ]
|
||||
tokens[, token_id := as.numeric(token_id)]
|
||||
|
||||
# manual adjustments to price
|
||||
ids_1 <- attributes[ (collection == 'Aurory') & (feature_value == 'Solana Blob') ]$token_id
|
||||
pred_price[ collection == 'Aurory' & token_id %in% eval(ids_1), pred_price := (pred_price * 0.8) ]
|
||||
|
||||
# save(pred_price, attributes, feature_values, sales, listings, coefsdf, tokens, file='data.Rdata')
|
||||
save(pred_price, attributes, feature_values, sales, listings, coefsdf, tokens, file='~/nft-deal-score/viz/data.Rdata')
|
||||
load('~/git/nft-deal-score/viz/data.Rdata')
|
||||
write.csv(listings, '~/git/nft-deal-score/data/listings.csv', row.names=F)
|
||||
write.csv(attributes, '~/git/nft-deal-score/data/attributes.csv', row.names=F)
|
||||
write.csv(sales, '~/git/nft-deal-score/data/model_sales.csv', row.names=F)
|
||||
write.csv(sales, '~/git/nft-deal-score/data/model_sales.csv', row.names=F)
|
||||
write.csv(coefsdf, '~/git/nft-deal-score/data/coefsdf.csv', row.names=F)
|
||||
write.csv(tokens, '~/git/nft-deal-score/data/tokens.csv', row.names=F)
|
||||
write.csv(pred_price, '~/git/nft-deal-score/data/pred_price.csv', row.names=F)
|
||||
ids_2 <- attributes[ (collection == 'Aurory') & (feature_value == 'Long Blob Hair ') ]$token_id
|
||||
pred_price[ collection == 'Aurory' & token_id %in% eval(ids_2), pred_price := (pred_price * 0.90) ]
|
||||
|
||||
ids_3 <- attributes[ (collection == 'Aurory') & (grepl( 'Mask', feature_value, fixed = TRUE)) ]$token_id
|
||||
pred_price[ collection == 'Aurory' & token_id %in% eval(ids_3), pred_price := (pred_price * 0.975) ]
|
||||
|
||||
# filter for only collections that have all data
|
||||
a <- unique(pred_price[, list(collection)])
|
||||
b <- unique(sales[, list(collection)])
|
||||
c <- unique(listings[, list(collection)])
|
||||
d <- merge(merge(a, b), c)
|
||||
|
||||
pred_price <- merge(pred_price, d, by=c('collection'))
|
||||
attributes <- merge(attributes, d, by=c('collection'))
|
||||
feature_values <- merge(feature_values, d, by=c('collection'))
|
||||
sales <- merge(sales, d, by=c('collection'))
|
||||
listings <- merge(listings, d, by=c('collection'))
|
||||
coefsdf <- merge(coefsdf, d, by=c('collection'))
|
||||
tokens <- merge(tokens, d, by=c('collection'))
|
||||
|
||||
save(
|
||||
pred_price
|
||||
, attributes
|
||||
, feature_values
|
||||
, sales
|
||||
, listings
|
||||
, coefsdf
|
||||
, tokens
|
||||
, file = paste0(file.location,'nft_deal_score_data.Rdata')
|
||||
)
|
||||
save(
|
||||
listings
|
||||
, file = paste0(file.location,'nft_deal_score_listings_data.Rdata')
|
||||
)
|
||||
|
||||
@ -361,32 +361,46 @@ def solana():
|
||||
|
||||
collection = 'Cets On Creck'
|
||||
collection = 'Astrals'
|
||||
|
||||
|
||||
|
||||
metadata = pd.read_csv('./data/metadata.csv')
|
||||
print(sorted(metadata.collection.unique()))
|
||||
metadata = metadata[metadata.collection == collection]
|
||||
print(sorted(metadata.collection.unique()))
|
||||
# print(sorted(metadata.collection.unique()))
|
||||
# metadata = metadata[metadata.collection == collection]
|
||||
# print(sorted(metadata.collection.unique()))
|
||||
metadata = metadata[-(metadata.feature_name.isin(['adj_nft_rank_0','adj_nft_rank_1','adj_nft_rank_2','nft_rank']))]
|
||||
len(metadata.token_id.unique())
|
||||
id_map = pd.read_csv('./data/mint_to_token_id_map.csv')
|
||||
id_map = pd.read_csv('./data/tokens.csv')
|
||||
id_map = id_map[id_map.collection == collection]
|
||||
cs = ['SOLGods']
|
||||
id_map = id_map[id_map.collection.isin(cs)]
|
||||
metadata = metadata[metadata.collection.isin(cs)]
|
||||
sorted(id_map.collection.unique())
|
||||
sorted(metadata.collection.unique())
|
||||
|
||||
id_map['token_id'] = id_map.token_id.astype(int)
|
||||
metadata['token_id'] = metadata.token_id.astype(int)
|
||||
# id_map['token_id'] = id_map.token_id.astype(int)
|
||||
# metadata['token_id'] = metadata.token_id.astype(int)
|
||||
id_map['token_id'] = id_map.token_id.astype(str)
|
||||
metadata['token_id'] = metadata.token_id.astype(str)
|
||||
|
||||
metadata = merge(metadata, id_map[['collection','token_id','mint_address','image_url']], ensure = True)
|
||||
metadata = merge(metadata, id_map[['collection','token_id','mint_address','image_url']], ensure = False)
|
||||
metadata = metadata[metadata.collection.isin(cs)]
|
||||
|
||||
metadata['feature_name'] = metadata.feature_name.apply(lambda x: x.title() )
|
||||
# metadata['image_url'] = metadata.token_id.apply(lambda x: 'https://metadata.degods.com/g/{}.png'.format(x - 1) )
|
||||
metadata.head()
|
||||
metadata = metadata[-(metadata.feature_name.isin(['Nft_Rank','Adj_Nft_Rank_0','Adj_Nft_Rank_1','Adj_Nft_Rank_2']))]
|
||||
# print(metadata.groupby('feature_name').token_id.count().reset_index().sort_values('token_id', ascending=0).head(10))
|
||||
|
||||
metadata = metadata[metadata.feature_name != 'L3G3Nd4Ry']
|
||||
|
||||
print(sorted(metadata.collection.unique()))
|
||||
sorted(metadata[metadata.collection == collection].feature_name.unique())
|
||||
# print(sorted(metadata.collection.unique()))
|
||||
# sorted(metadata[metadata.collection == collection].feature_name.unique())
|
||||
# sorted(metadata.feature_name.unique())
|
||||
|
||||
# metadata[['collection']].drop_duplicates().to_csv('~/Downloads/tmp.csv', index=False)
|
||||
|
||||
metadata['token_id'] = metadata.token_id.astype(int)
|
||||
for collection in metadata.collection.unique():
|
||||
print(collection)
|
||||
mdf = metadata[metadata.collection == collection]
|
||||
|
||||
68
load_data.py
68
load_data.py
@ -138,6 +138,7 @@ def mints_from_me():
|
||||
offset += 500
|
||||
lp_df = pd.DataFrame(lp_data)
|
||||
lp_df.to_csv('./data/me_lp_collections.csv', index=False)
|
||||
lp_df = pd.read_csv('./data/me_lp_collections.csv')
|
||||
|
||||
it = 0
|
||||
l_data = []
|
||||
@ -249,10 +250,30 @@ def mints_from_me():
|
||||
m_df.to_csv('./data/me_update_authorities.csv', index=False)
|
||||
|
||||
m_df = pd.read_csv('./data/me_update_authorities.csv')
|
||||
def f(x):
|
||||
x = re.sub('\(|\)', '', x)
|
||||
x = re.sub(' ', '_', x)
|
||||
x = re.sub('\'', '', x)
|
||||
return(x)
|
||||
m_df['collection'] = m_df.name.apply(lambda x: f(x) )
|
||||
|
||||
x = 'asf (asf)'
|
||||
f(x)
|
||||
|
||||
seen = [ x for x in m_df.collection.unique() if os.path.exists('./data/mints/{}/'.format(x)) and len(os.listdir('./data/mints/{}/'.format(x))) ]
|
||||
print(len(seen))
|
||||
# m_df = m_df.merge(lp_df)
|
||||
len(m_df)
|
||||
it = 0
|
||||
rpc = 'https://red-cool-wildflower.solana-mainnet.quiknode.pro/a1674d4ab875dd3f89b34863a86c0f1931f57090/'
|
||||
for row in m_df.iterrows():
|
||||
for row in m_df.sort_values('collection').iterrows():
|
||||
it += 1
|
||||
if it % 100 == 0:
|
||||
print('#{}/{}'.format(it, len(m_df)))
|
||||
row = row[1]
|
||||
collection = row['name']
|
||||
collection = row['collection']
|
||||
if collection in seen:
|
||||
continue
|
||||
update_authority = row['update_authority']
|
||||
print('Working on {}...'.format(collection))
|
||||
collection_dir = re.sub(' ', '_', collection)
|
||||
@ -260,7 +281,7 @@ def mints_from_me():
|
||||
dir = './data/mints/{}/'.format(collection_dir)
|
||||
if not os.path.exists(dir):
|
||||
os.makedirs(dir)
|
||||
else:
|
||||
elif len(os.listdir(dir)):
|
||||
# print('Already have {}.'.format(collection))
|
||||
print('Seen')
|
||||
continue
|
||||
@ -278,7 +299,48 @@ def mints_from_me():
|
||||
os.system('metaboss -r {} -t 300 decode mint --list-file {} --output {}'.format(rpc, fname, dir_mints))
|
||||
|
||||
|
||||
data = []
|
||||
for path in os.listdir('./data/mints/'):
|
||||
if os.path.isdir('./data/mints/'+path):
|
||||
collection = re.sub('_', ' ', path).strip()
|
||||
for fname in os.listdir('./data/mints/'+path):
|
||||
f = './data/mints/'+path+'/'+fname
|
||||
if os.path.isfile(f) and '.json' in f:
|
||||
with open(f) as file:
|
||||
j = json.load(file)
|
||||
for m in j:
|
||||
data += [[ collection, m ]]
|
||||
df = pd.DataFrame(data, columns=['collection','mint_address'])
|
||||
df = df[df.collection != 'etc']
|
||||
df = df.drop_duplicates()
|
||||
df['n'] = 1
|
||||
g = df.groupby(['mint_address']).n.sum().reset_index()
|
||||
g = g[g.n > 1]
|
||||
tmp_0 = g[['mint_address']].merge(df).groupby('collection').n.count().reset_index().sort_values('n', ascending=0)
|
||||
tmp_0.tail(20)
|
||||
tmp = g.merge(df[[ 'collection','mint_address' ]])
|
||||
tmp = tmp.sort_values(['mint_address','collection'])
|
||||
tmp[tmp.collection == 'Fractals']
|
||||
df[df.mint_address == '11gATLu654HjcVhkuarVy9YVm11CL74vjEmi1RhojRi']
|
||||
# tmp.sort_values(['mint_address','collection']).head()
|
||||
rem = tmp.collection.unique()
|
||||
# len(rem)
|
||||
# len(df.collection.unique())
|
||||
# tmp.head()
|
||||
# tmp = g[['mint_address']].merge(df).groupby('collection').n.count().reset_index().sort_values('n', ascending=0)
|
||||
# tmp[tmp.collection == 'Fractals']
|
||||
# tmp.head(20)
|
||||
'Fractals' in rem
|
||||
'Fractals' in df[-df.collection.isin(rem)].collection.unique()
|
||||
print(sorted(df.collection.unique()))
|
||||
len(df.collection.unique())
|
||||
sorted(df.collection.unique)
|
||||
df[['collection']].drop_duplicates().sort_values('collection').to_csv('~/Downloads/tmp.csv', index=False)
|
||||
len(df[-df.collection.isin(rem)])
|
||||
len(df[-df.collection.isin(rem)].drop_duplicates(subset=['mint_address']))
|
||||
df[-df.collection.isin(rem)].to_csv('./data/collection_mints.csv', index=False)
|
||||
len(df)
|
||||
len(df.drop_duplicates(subset=['mint_address']))
|
||||
len(df[df.twitter.notnull()])
|
||||
len(df[ (df.twitter.notnull()) & (df.website.notnull())])
|
||||
|
||||
|
||||
Binary file not shown.
@ -1,12 +1,14 @@
|
||||
beautifulsoup4==4.10.0
|
||||
beautifulsoup4==4.11.1
|
||||
kutils==0.3.0
|
||||
matplotlib==3.3.2
|
||||
numpy==1.19.2
|
||||
pandas==1.1.3
|
||||
requests==2.24.0
|
||||
scikit_learn==1.0.1
|
||||
scikit_learn==1.0.2
|
||||
scipy==1.5.2
|
||||
seaborn==0.11.0
|
||||
selenium==3.141.0
|
||||
snowflake==0.0.3
|
||||
snowflake_connector_python==2.4.3
|
||||
snowflake_connector_python==2.7.2
|
||||
statsmodels==0.12.0
|
||||
tensorflow==2.6.0
|
||||
tensorflow==2.4.1
|
||||
|
||||
95
scrape_listings.R
Normal file
95
scrape_listings.R
Normal file
@ -0,0 +1,95 @@
|
||||
|
||||
isRstudio <- Sys.info()[["user"]] == 'rstudio-connect'
|
||||
if(isRstudio) {
|
||||
source("/home/data-science/data_science/util/util_functions.R")
|
||||
} else {
|
||||
source("~/data_science/util/util_functions.R")
|
||||
setwd('~/git/nft-deal-score')
|
||||
}
|
||||
|
||||
library(httr)
|
||||
library(jsonlite)
|
||||
|
||||
|
||||
query <- "
|
||||
SELECT DISTINCT project_name AS collection
|
||||
, mint AS tokenMint
|
||||
, token_id
|
||||
FROM solana.dim_nft_metadata
|
||||
"
|
||||
mints <- QuerySnowflake(query)
|
||||
colnames(mints) <- c('collection','tokenMint','token_id')
|
||||
|
||||
collections <- c(
|
||||
'meerkat_millionaires_country_club','solgods','cets_on_creck','stoned_ape_crew','degods','aurory','thugbirdz','solana_monkey_business','degenerate_ape_academy','pesky_penguins'
|
||||
)
|
||||
|
||||
get_me_url <- function(collection, offset) {
|
||||
return(paste0('https://api-mainnet.magiceden.dev/v2/collections/',collection,'/listings?offset=',offset,'&limit=20'))
|
||||
}
|
||||
get_smb_url <- function(page) {
|
||||
return(paste0('https://market.solanamonkey.business/api/items?limit=40&page=',page))
|
||||
}
|
||||
|
||||
new_listings <- data.table()
|
||||
collection <- collections[1]
|
||||
for(collection in collections) {
|
||||
print(paste0('Working on ', collection, '...'))
|
||||
has_more <- TRUE
|
||||
offset <- 0
|
||||
while(has_more) {
|
||||
print(paste0('Offset #', offset))
|
||||
url <- get_me_url(collection, offset)
|
||||
response <- GET(url)
|
||||
content <- rawToChar(response$content)
|
||||
content <- fromJSON(content)
|
||||
# content <- rbindlist(content, fill=T)
|
||||
has_more <- nrow(content) >= 20
|
||||
if(nrow(content) > 0 && length(content) > 0) {
|
||||
df <- merge(content, mints, by=c('tokenMint')) %>% as.data.table()
|
||||
df <- df[, list(collection, token_id, price)]
|
||||
offset <- offset + 20
|
||||
new_listings <- rbind(new_listings, df)
|
||||
} else {
|
||||
has_more <- FALSE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for(collection in c('Solana Monkey Business')) {
|
||||
print(paste0('Working on ', collection, '...'))
|
||||
has_more <- TRUE
|
||||
page <- 1
|
||||
while(has_more) {
|
||||
print(paste0('Page #', page))
|
||||
url <- get_smb_url(page)
|
||||
response <- GET(url)
|
||||
content <- rawToChar(response$content)
|
||||
content <- fromJSON(content)
|
||||
# content <- rbindlist(content, fill=T)
|
||||
content <- content %>% as.data.table()
|
||||
has_more <- nrow(content) > 0 && 'price' %in% colnames(content)
|
||||
if(has_more) {
|
||||
content <- content[, list(mint, price)]
|
||||
content <- unique(content)
|
||||
content$price <- as.numeric(content$price) / (10^9)
|
||||
has_more <- nrow(content) >= 40
|
||||
colnames(content)[1] <- 'tokenMint'
|
||||
df <- merge(content, mints, by=c('tokenMint')) %>% as.data.table()
|
||||
df <- df[, list(collection, token_id, price)]
|
||||
page <- page + 1
|
||||
new_listings <- rbind(new_listings, df)
|
||||
}
|
||||
}
|
||||
}
|
||||
new_listings <- unique(new_listings)
|
||||
|
||||
listings <- read.csv('./data/listings.csv') %>% as.data.table()
|
||||
rem <- unique(new_listings$collection)
|
||||
listings <- listings[ !(collection %in% eval(rem)), ]
|
||||
listings <- listings[, list(collection, token_id, price)]
|
||||
listings <- rbind(listings, new_listings)
|
||||
listings <- listings[order(collection, price)]
|
||||
|
||||
write.csv(listings, './data/listings.csv', row.names=F)
|
||||
|
||||
@ -40,14 +40,22 @@ def how_rare_is_api():
|
||||
j.keys()
|
||||
t_data = []
|
||||
metadata = pd.DataFrame()
|
||||
for d in j['result']['data'][8:]:
|
||||
collection = 'Cets on Creck'
|
||||
collection = 'SOLGods'
|
||||
collection = 'Meerkat Millionaires'
|
||||
collection = d['url'][1:]
|
||||
d = {
|
||||
'Degen Apes': 'degenapes'
|
||||
, 'Pesky Penguins': 'peskypenguinclub'
|
||||
, 'Aurory': 'aurory'
|
||||
, 'Solana Monkey Business': 'smb'
|
||||
, 'Thugbirdz': 'thugbirdz'
|
||||
}
|
||||
for collection, url in d.items():
|
||||
# collection = 'Cets on Creck'
|
||||
# collection = 'SOLGods'
|
||||
# collection = 'Meerkat Millionaires'
|
||||
# collection = d['url'][1:]
|
||||
print('Working on collection {}, {}, {}'.format(collection, len(t_data), len(metadata)))
|
||||
url = 'https://api.howrare.is/v0.1/collections'+d['url']
|
||||
url = 'https://api.howrare.is/v0.1/collections/meerkatmillionaires'
|
||||
# url = 'https://api.howrare.is/v0.1/collections'+d['url']
|
||||
# url = 'https://api.howrare.is/v0.1/collections/meerkatmillionaires'
|
||||
url = 'https://api.howrare.is/v0.1/collections/'+url
|
||||
r = requests.get(url)
|
||||
j = r.json()
|
||||
for i in j['result']['data']['items']:
|
||||
@ -64,27 +72,50 @@ def how_rare_is_api():
|
||||
old = pd.read_csv('./data/tokens.csv')
|
||||
sorted(old.collection.unique())
|
||||
l0 = len(old)
|
||||
do_merge = True
|
||||
tokens = pd.DataFrame(t_data, columns=['collection','token_id','nft_rank','mint_address','image_url'])
|
||||
# old = old.merge(tokens, how='left', on=['collection','token_id'])
|
||||
old = old.append(tokens)
|
||||
# old['nft_rank'] = old.nft_rank_y.fillna(old.nft_rank_y)
|
||||
# del old['nft_rank_x']
|
||||
# del old['nft_rank_y']
|
||||
if do_merge:
|
||||
old['token_id'] = old.token_id.astype(str)
|
||||
tokens['token_id'] = tokens.token_id.astype(str)
|
||||
old = old.merge(tokens, how='left', on=['collection','token_id'])
|
||||
old[old.collection == 'Solana Monkey Business']
|
||||
for c in [ 'nft_rank','mint_address','image_url' ]:
|
||||
old[c] = old[c+'_x'].fillna(old[c+'_y'])
|
||||
del old[c+'_x']
|
||||
del old[c+'_y']
|
||||
old['clean_token_id'] = old.clean_token_id.fillna(old.token_id)
|
||||
old['chain'] = old.chain.fillna('Solana')
|
||||
else:
|
||||
old = old.append(tokens)
|
||||
print('Adding {} rows'.format(len(old) - l0))
|
||||
old[old.nft_rank.isnull()].groupby('collection').token_id.count()
|
||||
old.to_csv('./data/tokens.csv', index=False)
|
||||
|
||||
old = pd.read_csv('./data/metadata.csv')
|
||||
old = old[-(old.collection == 'Meerkat Millionaires Cc')]
|
||||
a = old[['collection','token_id']].drop_duplicates()
|
||||
a['exclude'] = 0
|
||||
a['token_id'] = a.token_id.astype(str)
|
||||
metadata['token_id'] = metadata.token_id.astype(str)
|
||||
m = metadata.merge(a, how='left')
|
||||
m = m[m.exclude.isnull()]
|
||||
len(m[m.exclude.isnull()].token_id.unique())
|
||||
del m['exclude']
|
||||
# old = old[-(old.collection == 'Meerkat Millionaires Cc')]
|
||||
print(sorted(old.collection.unique()))
|
||||
l0 = len(old)
|
||||
metadata.collection.unique()
|
||||
# metadata = pd.DataFrame(t_data, columns=['collection','token_id','nft_rank','mint_address','image_url'])
|
||||
# old = old.merge(tokens, how='left', on=['collection','token_id'])
|
||||
old = old.append(metadata)
|
||||
old = old.append(m[['collection','token_id','name','value']].rename(columns={'name':'feature_name','value':'feature_value'}) )
|
||||
old['token_id'] = old.token_id.astype(str)
|
||||
old = old.drop_duplicates(subset=['collection','token_id','feature_name'])
|
||||
# old['nft_rank'] = old.nft_rank_y.fillna(old.nft_rank_y)
|
||||
# del old['nft_rank_x']
|
||||
# del old['nft_rank_y']
|
||||
print('Adding {} rows'.format(len(old) - l0))
|
||||
print(old.groupby('collection').token_id.count())
|
||||
old[old.collection.isin(metadata.collection)]
|
||||
old[(old.collection == 'Thugbirdz') & (old.token_id == '1206')]
|
||||
old.to_csv('./data/metadata.csv', index=False)
|
||||
|
||||
|
||||
@ -580,7 +611,7 @@ def scrape_opensea_listings(browser, collections=['BAYC','MAYC']):
|
||||
old.groupby('collection').token_id.count()
|
||||
old.to_csv('./data/listings.csv', index=False)
|
||||
|
||||
def scrape_listings(browser, collections = [ 'solgods','cets-on-creck','stoned-ape-crew','degods','aurory','thugbirdz','smb','degenapes','peskypenguinclub' ], alerted = [], is_listings = True):
|
||||
def scrape_listings(browser, collections = [ 'meerkat-millionaires-cc','solgods','cets-on-creck','stoned-ape-crew','degods','aurory','thugbirdz','smb','degenapes','peskypenguinclub' ], alerted = [], is_listings = True):
|
||||
print('Scraping solanafloor listings...')
|
||||
data = []
|
||||
m_data = []
|
||||
@ -775,6 +806,7 @@ def scrape_listings(browser, collections = [ 'solgods','cets-on-creck','stoned-a
|
||||
|
||||
old = pd.read_csv('./data/listings.csv')
|
||||
listings = pd.DataFrame(data, columns=['collection','token_id','price']).drop_duplicates()
|
||||
listings.groupby('collection').price.min()
|
||||
# others = scrape_magic_eden()
|
||||
# listings = listings.append(others).drop_duplicates()
|
||||
# d = {
|
||||
@ -788,10 +820,13 @@ def scrape_listings(browser, collections = [ 'solgods','cets-on-creck','stoned-a
|
||||
# ,'degods': 'DeGods'
|
||||
# }
|
||||
listings['collection'] = listings.collection.apply(lambda x: clean_name(x))
|
||||
listings.groupby('collection').price.min()
|
||||
listings.groupby('collection').price.count()
|
||||
listings[listings.token_id=='1656']
|
||||
listings[listings.token_id==484]
|
||||
|
||||
old = old[ -(old.collection.isin(listings.collection.unique())) ]
|
||||
old = old[old.collection != 'Meerkat Millionaires Cc']
|
||||
pred_price = pd.read_csv('./data/pred_price.csv')
|
||||
listings.token_id.values[:3]
|
||||
pred_price.token_id.values[:3]
|
||||
|
||||
377
scratch.sql
377
scratch.sql
@ -1,3 +1,380 @@
|
||||
|
||||
|
||||
WITH base AS (
|
||||
SELECT native_to_address, COUNT(1) AS n
|
||||
FROM thorchain.swaps
|
||||
WHERE block_timestamp >= '2022-04-01'
|
||||
GROUP BY 1
|
||||
)
|
||||
select b.n, s.* from thorchain.swaps s
|
||||
JOIN base b ON b.native_to_address = s.native_to_address
|
||||
where block_timestamp >= '2022-04-10'
|
||||
ORDER BY block_timestamp, n, tx_id
|
||||
LIMIT 100
|
||||
|
||||
|
||||
|
||||
SELECT MIN(block_timestamp) AS mn
|
||||
, MAX(block_timestamp) AS mx
|
||||
, COUNT(1) AS n
|
||||
FROM flipside_dev_db.thorchain.liquidity_actions
|
||||
|
||||
|
||||
SELECT MIN(block_timestamp) AS mn
|
||||
, MAX(block_timestamp) AS mx
|
||||
, COUNT(1) AS n
|
||||
FROM flipside_dev_db.thorchain.liquidity_actions
|
||||
|
||||
|
||||
SELECT MIN(block_timestamp) AS mn
|
||||
, MAX(block_timestamp) AS mx
|
||||
, COUNT(1) AS n
|
||||
FROM flipside_dev_db.thorchain.bond_actions
|
||||
|
||||
SELECT
|
||||
|
||||
|
||||
|
||||
https://app.flipsidecrypto.com/dashboard/small-lp-actions-LD1XQ9
|
||||
|
||||
|
||||
https://app.flipsidecrypto.com/dashboard/pool-ranks-UbtLg9
|
||||
https://app.flipsidecrypto.com/dashboard/price-shift-_-JpTq
|
||||
https://app.flipsidecrypto.com/dashboard/tho-rchain-average-age-of-synth-holders-Z2AXIx
|
||||
https://app.flipsidecrypto.com/dashboard/thor-64-standardized-tvl-over-time-all-pools-Zf6w-L
|
||||
https://app.flipsidecrypto.com/dashboard/tho-rchain-pool-ranks-RNNzza
|
||||
https://app.flipsidecrypto.com/dashboard/thorchain-synth-mints-burns-aH0lCY
|
||||
|
||||
https://discord.com/channels/889577356681945098/889577399308656662/951960381411192842
|
||||
|
||||
solana-keygen recover 'prompt:?key=3/0' --outfile ~/.config/solana/tmp.json
|
||||
|
||||
|
||||
SELECT *
|
||||
, instructions[0]:parsed:info:lamports / POWER(10, 9) AS sol_amount
|
||||
FROM solana.fact_transactions
|
||||
WHERE block_timestamp >= '2022-01-02'
|
||||
AND tx_id = '5H6UQqbxa2wtryax6SAZgjXB9B6Za4ip6GsheqopAsbLCrLMPvYf35H551SaAKNy6bi6BceRGtkwwP9LRoN7RiVo'
|
||||
|
||||
|
||||
SELECT *
|
||||
FROM solana.fact_transfers
|
||||
WHERE block_timestamp >= '2022-01-02'
|
||||
AND tx_id = '393xRouisz4DuMxzARAqPy7FVYQZtkfpmAMuDXXj39uvASFYtMijHM9hyVzXSocsB4fk2woLNfnWTM4qXJxJsWBw'
|
||||
|
||||
SELECT lp.tx_id
|
||||
, lp.signers[0] as signer
|
||||
, t.instructions[0]:parsed:info:lamports / POWER(10, 9) AS sol_amount
|
||||
FROM solana.fact_staking_lp_actions lp
|
||||
JOIN solana.fact_transactions t ON t.tx_id = lp.tx_id
|
||||
WHERE lp.block_timestamp >= '2022-01-01'
|
||||
AND lp.event_type = 'delegate'
|
||||
AND tx_id = '393xRouisz4DuMxzARAqPy7FVYQZtkfpmAMuDXXj39uvASFYtMijHM9hyVzXSocsB4fk2woLNfnWTM4qXJxJsWBw'
|
||||
LIMIT 10
|
||||
|
||||
sudo apt-get build-dep python E: You must put some 'deb-src' URIs in your sources.list
|
||||
|
||||
|
||||
SELECT project_name
|
||||
, COUNT(1) AS n
|
||||
, SUM( CASE WHEN address IS NULL THEN 1 ELSE 0 END) AS n_nulls
|
||||
FROM crosschain.address_labels
|
||||
WHERE blockchain = 'solana'
|
||||
AND label_subtype = 'nf_token_contract'
|
||||
GROUP BY 1
|
||||
|
||||
-- MEv1 De-Listing
|
||||
WITH mints AS (
|
||||
SELECT DISTINCT project_name
|
||||
, mint
|
||||
, token_id
|
||||
FROM solana.dim_nft_metadata
|
||||
)
|
||||
SELECT pre_token_balances[0]:mint::string AS mint
|
||||
, t.*
|
||||
, m.project_name
|
||||
, m.token_id
|
||||
FROM solana.fact_transactions t
|
||||
JOIN mints m ON m.mint = t.pre_token_balances[0]:mint::string
|
||||
WHERE block_timestamp >= CURRENT_DATE - 30
|
||||
AND tx_id = '3CxhnTCXYX1zH6HbNESFsZLwdHfTe7RUYF8tAgB168hciVjUGggp2PwVEsnDvpd2kNqMha7kH2be7NtSTppAnXzn'
|
||||
AND instructions[0]:data = 'TE6axTojnpk'
|
||||
AND instructions[0]:programId = 'MEisE1HzehtrDpAAT8PnLHjpSSkRYakotTuJRPjTpo8'
|
||||
AND succeeded = TRUE
|
||||
LIMIT 100
|
||||
|
||||
ENwHiaH9NA9veUqRzGozjWnTuR9xcNvHcPZVFMi3Ca9L
|
||||
|
||||
|
||||
WITH mints AS (
|
||||
SELECT DISTINCT project_name
|
||||
, mint
|
||||
, token_id
|
||||
FROM solana.dim_nft_metadata
|
||||
)
|
||||
SELECT instructions[0]:data AS data
|
||||
, COUNT(1) AS n
|
||||
, COUNT(1) AS nn
|
||||
FROM solana.fact_transactions t
|
||||
JOIN mints m ON m.mint = t.pre_token_balances[0]:mint::string
|
||||
WHERE block_timestamp >= '2022-04-17'
|
||||
AND instructions[0]:programId = 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K'
|
||||
AND succeeded = TRUE
|
||||
GROUP BY 1
|
||||
ORDER BY 2 DESC
|
||||
|
||||
|
||||
-- MEv2 De-Listing
|
||||
WITH mints AS (
|
||||
SELECT DISTINCT project_name
|
||||
, mint
|
||||
, token_id
|
||||
FROM solana.dim_nft_metadata
|
||||
), rem AS (
|
||||
SELECT pre_token_balances[0]:mint::string AS mint
|
||||
, m.project_name
|
||||
, m.token_id
|
||||
, t.tx_id AS remove_tx
|
||||
, block_timestamp AS remove_time
|
||||
, ROW_NUMBER() OVER (PARTITION BY mint ORDER BY block_timestamp DESC) AS rn
|
||||
FROM solana.fact_transactions t
|
||||
JOIN mints m ON m.mint = t.pre_token_balances[0]:mint::string
|
||||
WHERE block_timestamp >= CURRENT_DATE - 3
|
||||
AND LEFT(instructions[0]:data::string, 4) IN ('ENwH','3GyW')
|
||||
AND instructions[0]:programId = 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K'
|
||||
), add AS (
|
||||
SELECT pre_token_balances[0]:mint::string AS mint
|
||||
, m.project_name
|
||||
, m.token_id
|
||||
, t.tx_id AS listing_tx
|
||||
, block_timestamp AS listing_time
|
||||
, ROW_NUMBER() OVER (PARTITION BY mint ORDER BY block_timestamp DESC) AS rn
|
||||
FROM solana.fact_transactions t
|
||||
JOIN mints m ON m.mint = t.pre_token_balances[0]:mint::string
|
||||
WHERE block_timestamp >= CURRENT_DATE - 3
|
||||
AND LEFT(instructions[0]:data::string, 4) IN ('2B3v')
|
||||
AND instructions[0]:programId = 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K'
|
||||
AND succeeded = TRUE
|
||||
)
|
||||
SELECT a.*
|
||||
, r.remove_tx
|
||||
, r.remove_time
|
||||
, CASE WHEN r.remove_time IS NULL OR a.listing_time > r.remove_time THEN 1 ELSE 0 END AS is_listed
|
||||
FROM add a
|
||||
LEFT JOIN rem r ON r.mint = a.mint AND r.rn = 1
|
||||
WHERE a.rn = 1
|
||||
|
||||
thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj
|
||||
|
||||
WITH base AS (
|
||||
SELECT from_address AS address
|
||||
, block_timestamp::date AS date
|
||||
, -from_amount AS amount
|
||||
, 'From Swap' AS tx_type
|
||||
FROM thorchain.swaps
|
||||
WHERE from_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj'
|
||||
AND from_asset = 'THOR.RUNE'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT native_to_address AS address
|
||||
, block_timestamp::date AS date
|
||||
, to_amount AS amount
|
||||
, 'To Swap' AS tx_type
|
||||
FROM thorchain.swaps
|
||||
WHERE native_to_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj'
|
||||
AND to_asset = 'THOR.RUNE'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT from_address AS address
|
||||
, block_timestamp::date AS date
|
||||
, -rune_amount AS amount
|
||||
, 'From Transfer' AS tx_type
|
||||
FROM thorchain.transfers
|
||||
WHERE from_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT to_address AS address
|
||||
, block_timestamp::date AS date
|
||||
, rune_amount AS amount
|
||||
, 'To Transfer' AS tx_type
|
||||
FROM thorchain.transfers
|
||||
WHERE to_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj'
|
||||
)
|
||||
SELECT *
|
||||
FROM base
|
||||
ORDER BY date DESC
|
||||
|
||||
68. [Hard] $RUNE Distribution
|
||||
Calculate the distribution of $RUNE holdings by address. Include RUNE that is in liquidity pools. (e.g. if I have 100 $RUNE in my wallet and LP 50, I should still be considered to be holding 100). In terms of charts, feel free to create a histogram or whichever visual you think works best!
|
||||
|
||||
Hint: use thorchain.transfers and thorchain.liquidity_actions
|
||||
|
||||
70. [Hard] Weighted-average LP duration
|
||||
What is the average duration of liquidity held in each pool, weighted by the size of the LP?
|
||||
|
||||
Hint: use thorchain.liquidity_actions
|
||||
|
||||
69. [Medium] LP Size Distribution
|
||||
What is the current distribution of LP size for each pool? You will have to use both 'add_liquidity' and 'remove_liquidity' to determine which LPs are stil current.
|
||||
|
||||
Hint: use thorchain.liquidity_actions
|
||||
|
||||
71. [Easy] Block Rewards vs Swap Fees
|
||||
Breakdown the yield from block rewards vs swap fees, both total and by pool. Show how the proportions have changed over time since the network started.
|
||||
|
||||
Hint: use thorchain.block_rewards
|
||||
|
||||
|
||||
72. [Medium] Swap Volume vs LP Rewards
|
||||
Chart the weekly swap volume and LP rewards on the same chart. What is the relationship between the two?
|
||||
|
||||
Hint: use thorchain.daily_pool_stats to get swap volume and earnings_to_pools from thorchain.daily_earnings to get LP rewards
|
||||
|
||||
73. [Hard] Realized APY
|
||||
Visualize the realized APY of LP-ers that have removed liquidity from THORChain. What was their actual APY vs HODL (accounting for impermanent loss) when you pro-rate it for how long they were LP-ing for? Are certain pools out-performing others?
|
||||
|
||||
Hint: use thorchain.liquidity_actions
|
||||
|
||||
|
||||
SELECT *
|
||||
FROM THORCHAIN.DAILY_POOL_STATS
|
||||
WHERE pool_name ilike '%luna%'
|
||||
ORDER BY day DESC
|
||||
|
||||
|
||||
WITH base AS (
|
||||
SELECT from_address AS address
|
||||
, block_timestamp::date AS date
|
||||
, rune_amount AS lp_amount
|
||||
, 'Add LP' AS tx_type
|
||||
FROM thorchain.liquidity_actions
|
||||
WHERE from_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj'
|
||||
AND lp_action = 'add_liquidity'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT from_address AS address
|
||||
, block_timestamp::date AS date
|
||||
, -rune_amount AS lp_amount
|
||||
, 'Rem LP' AS tx_type
|
||||
FROM thorchain.liquidity_actions
|
||||
WHERE from_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj'
|
||||
AND lp_action = 'remove_liquidity'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT native_to_address AS address
|
||||
, block_timestamp::date AS date
|
||||
, to_amount AS amount
|
||||
, 'To Swap' AS tx_type
|
||||
FROM thorchain.swaps
|
||||
WHERE native_to_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj'
|
||||
AND to_asset = 'THOR.RUNE'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT from_address AS address
|
||||
, block_timestamp::date AS date
|
||||
, -rune_amount AS amount
|
||||
, 'From Transfer' AS tx_type
|
||||
FROM thorchain.transfers
|
||||
WHERE from_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT to_address AS address
|
||||
, block_timestamp::date AS date
|
||||
, rune_amount AS amount
|
||||
, 'To Transfer' AS tx_type
|
||||
FROM thorchain.transfers
|
||||
WHERE to_address = 'thor12209cxpf4mpm8qxdyzmm4k8mfuyjt4fysnwyjj'
|
||||
)
|
||||
SELECT *
|
||||
FROM base
|
||||
ORDER BY date DESC
|
||||
|
||||
-- 2B3v: listing
|
||||
-- ENwH: de-listing
|
||||
-- 3GyW: sale
|
||||
|
||||
WITH mints AS (
|
||||
SELECT DISTINCT project_name
|
||||
, mint
|
||||
, token_id
|
||||
FROM solana.dim_nft_metadata
|
||||
)
|
||||
SELECT pre_token_balances[0]:mint::string AS mint
|
||||
, t.*
|
||||
, m.project_name
|
||||
, m.token_id
|
||||
FROM solana.fact_transactions t
|
||||
JOIN mints m ON m.mint = t.pre_token_balances[0]:mint::string
|
||||
WHERE block_timestamp >= CURRENT_DATE - 2
|
||||
AND tx_id = '3CxhnTCXYX1zH6HbNESFsZLwdHfTe7RUYF8tAgB168hciVjUGggp2PwVEsnDvpd2kNqMha7kH2be7NtSTppAnXzn'
|
||||
AND LEFT(instructions[0]:data::string, 4) IN ('2B3v')
|
||||
AND instructions[0]:programId = 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K'
|
||||
AND succeeded = TRUE
|
||||
LIMIT 100
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
SELECT MIN(block_timestamp) AS mn
|
||||
, MAX(block_timestamp) AS mx
|
||||
, COUNT(1) AS n
|
||||
FROM flipside_prod_db.thorchain.liquidity_actions
|
||||
|
||||
|
||||
SELECT MIN(block_timestamp) AS mn
|
||||
, MAX(block_timestamp) AS mx
|
||||
, COUNT(1) AS n
|
||||
FROM flipside_prod_db.thorchain.prices
|
||||
|
||||
liquidity_actions
|
||||
bond_actions
|
||||
|
||||
|
||||
SELECT MIN(block_timestamp) AS mn
|
||||
, MAX(block_timestamp) AS mx
|
||||
, COUNT(1) AS n
|
||||
FROM flipside_prod_db.thorchain.bond_actions
|
||||
|
||||
|
||||
SELECT MIN(block_timestamp) AS mn
|
||||
, MAX(block_timestamp) AS mx
|
||||
, COUNT(1) AS n
|
||||
FROM flipside_prod_db.thorchain.pool_block_balances
|
||||
WHERE rune_amount > 0 and COALESCE(rune_amount_usd, 0) = 0
|
||||
|
||||
SELECT MIN(block_timestamp) AS mn
|
||||
, MAX(block_timestamp) AS mx
|
||||
, COUNT(1) AS n
|
||||
FROM flipside_prod_db.thorchain.pool_block_balances
|
||||
WHERE asset_amount > 0 and COALESCE(asset_amount_usd, 0) = 0
|
||||
|
||||
SELECT *
|
||||
FROM flipside_prod_db.thorchain.pool_block_balances
|
||||
WHERE (asset_amount > 0 and COALESCE(asset_amount_usd, 0) = 0)
|
||||
OR (rune_amount > 0 and COALESCE(rune_amount_usd, 0) = 0)
|
||||
ORDER BY block_timestamp DESC
|
||||
LIMIT 10000
|
||||
|
||||
SELECT block_timestamp::date AS date
|
||||
, COUNT(1) AS n
|
||||
FROM flipside_prod_db.thorchain.pool_block_balances
|
||||
WHERE (asset_amount > 0 and COALESCE(asset_amount_usd, 0) = 0)
|
||||
OR (rune_amount > 0 and COALESCE(rune_amount_usd, 0) = 0)
|
||||
ORDER BY block_timestamp DESC
|
||||
GROUP BY 1
|
||||
ORDER BY 1
|
||||
|
||||
|
||||
|
||||
WITH active_vault_events_cte AS
|
||||
(SELECT max(block_timestamp)::string AS recency, min(block_timestamp)::string AS start_time,
|
||||
'active_vault_events' AS "table"
|
||||
|
||||
@ -364,12 +364,12 @@ def train_model(check_exclude=False, supplement_with_listings=True, use_saved_pa
|
||||
collections = [ x for x in collections if not x in ['Bakc','BAKC','MAYC'] ]
|
||||
collections = [ x for x in collections if not x in ['Astrals','Cets on Cleck','DeFi Pirates'] ]
|
||||
collections = ['Cets on Creck']
|
||||
collections = list(s_df[['collection']].drop_duplicates().merge(m_df[['collection']].drop_duplicates()).collection.unique())
|
||||
s_df.groupby('collection').block_timestamp.max()
|
||||
collections = ['Meerkat Millionaires']
|
||||
collections = list(s_df[['collection']].drop_duplicates().merge(m_df[['collection']].drop_duplicates()).collection.unique())
|
||||
print(sorted(collections))
|
||||
for collection in collections:
|
||||
if collection in ['Astrals','BAYC','MAYC']:
|
||||
if collection in ['Astrals','Bakc','BAYC','MAYC']:
|
||||
continue
|
||||
if not collection in saved_params.keys():
|
||||
saved_params[collection] = {}
|
||||
@ -381,7 +381,7 @@ def train_model(check_exclude=False, supplement_with_listings=True, use_saved_pa
|
||||
print('\nWorking on collection {}'.format(collection))
|
||||
sales = s_df[ s_df.collection == collection ]
|
||||
sales[sales.sim==0].block_timestamp.max()
|
||||
metadata = m_df[ m_df.collection == collection ].drop_duplicates()
|
||||
metadata = m_df[ m_df.collection == collection ].drop_duplicates(subset=['token_id','feature_name'], keep='last')
|
||||
metadata = metadata[metadata.feature_name != 'Genesis Role?']
|
||||
metadata[metadata.token_id=='1']
|
||||
metadata[metadata.feature_name=='Genesis Role?'].feature_value.unique()
|
||||
@ -629,7 +629,14 @@ def train_model(check_exclude=False, supplement_with_listings=True, use_saved_pa
|
||||
mn = coefs.val.min()
|
||||
if mn >= 0:
|
||||
df, bst_p, bst_r = ku.get_bst_params( model, df, X_new, y, target_col, col, verbose = True, wt_col='wt', params = [bst_p] )
|
||||
coefs['col'] = coefs.col.apply(lambda x: re.sub('std_', '', x) )
|
||||
coefs['n'] = 0
|
||||
n = pd.DataFrame()
|
||||
for c in cat_metadata.columns:
|
||||
if not c in [ 'collection','token_id' ]:
|
||||
coefs.loc[ coefs.col == c, 'n' ] = len(cat_metadata[cat_metadata[c] == 1])
|
||||
coefs.to_csv('./data/coefs/{}_{}_{}.csv'.format(collection, model, it), index=False)
|
||||
|
||||
test = ku.apply_model( model, bst_p, df, test, cur_std_pred_cols, target_col, col)
|
||||
if model in ['rfr']:
|
||||
df[col] = df[col] + df['rarity_value_'+it]
|
||||
|
||||
@ -52,13 +52,16 @@ ld.add_solana_sales()
|
||||
ld.add_eth_sales()
|
||||
|
||||
# update listings
|
||||
# ssn.scrape_listings(browser, ['smb'])
|
||||
# ssn.scrape_listings(browser, ['cets-on-creck'])
|
||||
# ssn.scrape_listings(browser, ['meerkat-millionaires-cc'])
|
||||
# ssn.scrape_listings(browser, ['thugbirdz'])
|
||||
ssn.scrape_listings(browser)
|
||||
ssn.scrape_randomearth(browser)
|
||||
ssn.scrape_opensea_listings(browser)
|
||||
# ssn.scrape_listings(browser, ['smb','aurory'])
|
||||
|
||||
listings = pd.read_csv('./data/listings.csv')
|
||||
listings[listings.collection == 'Solana Monkey Business'].sort_values('price').head(20)
|
||||
|
||||
# update model
|
||||
# ssn.convert_collection_names()
|
||||
# sm.train_model(True, False)
|
||||
|
||||
14
update_data.R
Normal file
14
update_data.R
Normal file
@ -0,0 +1,14 @@
|
||||
|
||||
install.packages('reticulate')
|
||||
library(reticulate)
|
||||
|
||||
switch(
|
||||
Sys.info()[["user"]],
|
||||
"rstudio-connect" = source("/home/data-science/data_science/util/util_functions.R"),
|
||||
source("~/data_science/util/util_functions.R")
|
||||
)
|
||||
|
||||
# use_python("/usr/local/bin/python")
|
||||
|
||||
py_run_file("script.py")
|
||||
|
||||
2
utils.py
2
utils.py
@ -40,7 +40,7 @@ def clean_token_id(df):
|
||||
|
||||
def clean_name(name):
|
||||
x = re.sub('-', '', name).lower()
|
||||
x = re.sub(' ', '', name).lower()
|
||||
x = re.sub(' ', '', x).lower()
|
||||
if x in clean_names.keys():
|
||||
return(clean_names[x])
|
||||
name = name.title()
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# setwd('~/git/nft-deal-score/viz')
|
||||
source("~/data_science/util/util_functions.R")
|
||||
# source("~/data_science/util/util_functions.R")
|
||||
library(data.table)
|
||||
library(shiny)
|
||||
library(ggplot2)
|
||||
|
||||
45
viz/server.R
45
viz/server.R
@ -1,5 +1,14 @@
|
||||
server <- function(input, output, session) {
|
||||
load('data.Rdata')
|
||||
# load('data.Rdata')
|
||||
|
||||
file.location <- ifelse(
|
||||
Sys.info()[["user"]] == "rstudio-connect"
|
||||
, "/rstudio-data/"
|
||||
, '~/git/nft-deal-score/viz/'
|
||||
)
|
||||
load(paste0(file.location, 'nft_deal_score_data.RData'))
|
||||
load(paste0(file.location, 'nft_deal_score_listings_data.RData'))
|
||||
# load(paste0(file.location, 'data.RData'))
|
||||
|
||||
metadata <- unique(attributes[, list(collection, feature_name, feature_value)])
|
||||
|
||||
@ -81,6 +90,20 @@ server <- function(input, output, session) {
|
||||
)
|
||||
})
|
||||
|
||||
output$minfloorinput <- renderUI({
|
||||
textInput(
|
||||
inputId = 'minfloorinput'
|
||||
, label = NULL
|
||||
, width = "100%"
|
||||
)
|
||||
})
|
||||
output$maxfloorinput <- renderUI({
|
||||
textInput(
|
||||
inputId = 'maxfloorinput'
|
||||
, label = NULL
|
||||
, width = "100%"
|
||||
)
|
||||
})
|
||||
output$maxrarityrankinput2 <- renderUI({
|
||||
textInput(
|
||||
inputId = 'maxrarityrank2'
|
||||
@ -509,10 +532,12 @@ server <- function(input, output, session) {
|
||||
}
|
||||
t <- ''
|
||||
if (nrow(data)) {
|
||||
p <- format(round(mean(head(data$price, 100)), 1), big.mark=',')
|
||||
f <- format(round(mean(head(data$vs_floor, 100)), 1), big.mark=',')
|
||||
data <- head(data, 100)
|
||||
p <- format(round(mean(data$price), 1), big.mark=',')
|
||||
f <- format(round(mean(data$vs_floor), 1), big.mark=',')
|
||||
data[, pct_vs_floor := (vs_floor + price) / price ]
|
||||
pct <- format(round(mean(head(data$pct_vs_floor, 100)), 1), big.mark=',')
|
||||
pct <- sum(data$price) / sum(data$mn_20)
|
||||
pct <- format(round(pct, 1), big.mark=',')
|
||||
chain <- getChain()
|
||||
currency <- ifelse( chain == 'Solana', 'SOL', ifelse(chain == 'Ethereum', 'ETH', 'LUNA') )
|
||||
t <- paste0(p, ' $',currency,' (+',f,' / ',pct,'x vs the floor)')
|
||||
@ -823,6 +848,18 @@ server <- function(input, output, session) {
|
||||
r <- as.numeric(input$maxrarityrank2)
|
||||
data <- data[ nft_rank <= eval(r) ]
|
||||
}
|
||||
if(input$minfloorinput != '') {
|
||||
r <- as.numeric(input$minfloorinput)
|
||||
data <- data[ minfloorinput >= eval(r) ]
|
||||
}
|
||||
if(input$maxfloorinput != '') {
|
||||
r <- as.numeric(input$maxfloorinput)
|
||||
data <- data[ maxfloorinput <= eval(r) ]
|
||||
}
|
||||
if(input$maxrarityrank2 != '') {
|
||||
r <- as.numeric(input$maxrarityrank2)
|
||||
data <- data[ nft_rank <= eval(r) ]
|
||||
}
|
||||
if(input$minrarityrank2 != '') {
|
||||
data <- data[ nft_rank >= eval(as.numeric(input$minrarityrank2)) ]
|
||||
}
|
||||
|
||||
22
viz/ui.R
22
viz/ui.R
@ -146,34 +146,48 @@ fluidPage(
|
||||
, bsTooltip(id = "historical-sales-tooltip", title = "This app is still in beta - sales data may be incomplete or delayed", placement = "bottom", trigger = "hover")
|
||||
, fluidRow(
|
||||
class = 'filters'
|
||||
, column(3
|
||||
, column(2
|
||||
, div(
|
||||
class = "inputtitle"
|
||||
, "Min Deal Score Rank"
|
||||
)
|
||||
, fluidRow(uiOutput("minnftrankinput2"))
|
||||
)
|
||||
, column(3
|
||||
, column(2
|
||||
, div(
|
||||
class = "inputtitle"
|
||||
, "Max Deal Score Rank"
|
||||
)
|
||||
, fluidRow(uiOutput("maxnftrankinput2"))
|
||||
)
|
||||
, column(3
|
||||
, column(2
|
||||
, div(
|
||||
class = "inputtitle"
|
||||
, "Min Rarity Rank"
|
||||
)
|
||||
, fluidRow(uiOutput("minrarityrankinput2"))
|
||||
)
|
||||
, column(3
|
||||
, column(2
|
||||
, div(
|
||||
class = "inputtitle"
|
||||
, "Max Rarity Rank"
|
||||
)
|
||||
, fluidRow(uiOutput("maxrarityrankinput2"))
|
||||
)
|
||||
, column(2
|
||||
, div(
|
||||
class = "inputtitle"
|
||||
, "Min Floor"
|
||||
)
|
||||
, fluidRow(uiOutput("minfloorinput"))
|
||||
)
|
||||
, column(2
|
||||
, div(
|
||||
class = "inputtitle"
|
||||
, "Max Floor"
|
||||
)
|
||||
, fluidRow(uiOutput("maxfloorinput"))
|
||||
)
|
||||
, column(3
|
||||
, div(
|
||||
class = "inputtitle"
|
||||
|
||||
8
viz/update_data.R
Normal file
8
viz/update_data.R
Normal file
@ -0,0 +1,8 @@
|
||||
|
||||
isRstudio <- Sys.info()[["user"]] == 'rstudio-connect'
|
||||
if(isRstudio) {
|
||||
source("/home/data-science/data_science/util/util_functions.R")
|
||||
} else {
|
||||
source('~/git/nft-deal-score/scrape_listings.R')
|
||||
}
|
||||
source("/home/data-science/data_science/util/util_functions.R")
|
||||
Loading…
Reference in New Issue
Block a user