mirror of
https://github.com/FlipsideCrypto/nft-deal-score.git
synced 2026-02-06 10:56:58 +00:00
some good stuff aka degods
This commit is contained in:
flipside-kellen
parent
bbc4fad338
commit
2129a728d0
45
load_data.py
45
load_data.py
@ -49,6 +49,51 @@ def manual_clean():
|
||||
df['clean_token_id'] = df.token_id
|
||||
df.to_csv('./data/{}.csv'.format(c), index=False)
|
||||
|
||||
def add_solana_sales():
|
||||
my_file = open('./scripts/solana-rpc-app/output.txt', 'r')
|
||||
content = my_file.read()
|
||||
my_file.close()
|
||||
content_list = content.split('[')
|
||||
data = []
|
||||
for c in content_list:
|
||||
s = re.split(',', c)
|
||||
if len(s) > 1 and '#' in s[1]:
|
||||
data += [[ re.split('"', s[0])[1], int(re.split('#', re.split('"', s[1])[1])[1]) ]]
|
||||
df = pd.DataFrame(data, columns=['mint','token_id']).drop_duplicates()
|
||||
query = '''
|
||||
SELECT tx_id
|
||||
, n.mint
|
||||
, n.block_timestamp AS sale_date
|
||||
, (inner_instruction:instructions[0]:parsed:info:lamports
|
||||
+ inner_instruction:instructions[1]:parsed:info:lamports
|
||||
+ inner_instruction:instructions[2]:parsed:info:lamports
|
||||
+ inner_instruction:instructions[3]:parsed:info:lamports) / POWER(10, 9) AS price
|
||||
FROM solana.nfts n
|
||||
LEFT JOIN crosschain.address_labels l ON LOWER(n.mint) = LOWER(l.address)
|
||||
WHERE block_timestamp >= CURRENT_DATE - 200
|
||||
AND instruction:data like '3UjLyJvuY4%'
|
||||
AND l.project_name ilike 'degods'
|
||||
'''
|
||||
sales = ctx.cursor().execute(query)
|
||||
sales = pd.DataFrame.from_records(iter(sales), columns=[x[0] for x in sales.description])
|
||||
sales = clean_colnames(sales)
|
||||
print('Queried {} sales'.format(len(sales)))
|
||||
sales['chain'] = 'Solana'
|
||||
sales['collection'] = 'DeGods'
|
||||
m = sales.merge(df, how='left', on=['mint'])
|
||||
s_df = pd.read_csv('./data/sales.csv')
|
||||
l0 = len(s_df)
|
||||
s_df = s_df[-s_df.collection.isin(sales.collection.unique())]
|
||||
s_df = s_df.append(m)
|
||||
print(s_df.groupby('collection').token_id.count())
|
||||
l1 = len(s_df)
|
||||
print('Added {} sales'.format(l1 - l0))
|
||||
for c in [ 'mint','tmp' ]:
|
||||
if c in s_df:
|
||||
del s_df[c]
|
||||
s_df.to_csv('./data/sales.csv', index=False)
|
||||
pass
|
||||
|
||||
def solana_metadata():
|
||||
metadata = pd.read_csv('./data/metadata.csv')
|
||||
metadata[metadata.collection == 'Solana Monkey Business'].feature_name.unique()
|
||||
|
||||
@ -6,8 +6,31 @@ os.chdir('/Users/kellenblumberg/git/nft-deal-score')
|
||||
from solana_model import get_sales
|
||||
from scrape_sol_nfts import clean_name
|
||||
|
||||
|
||||
def add_att_count():
|
||||
m_df = pd.read_csv('./data/metadata.csv')
|
||||
print(len(m_df))
|
||||
collection = 'DeGods'
|
||||
cur = m_df[m_df.collection == collection]
|
||||
cur['feature_value'] = cur.feature_value.fillna('None')
|
||||
sorted(cur.feature_name.unique())
|
||||
g = cur[(cur.feature_value != 'None') & (-cur.feature_name.isin(['nft_rank','adj_nft_rank_0','adj_nft_rank_1','adj_nft_rank_2','attribute_count','Attribute Count']))]
|
||||
g = g.groupby(['collection','token_id']).feature_value.count().reset_index()
|
||||
g.columns = [ 'collection', 'token_id','feature_value' ]
|
||||
g['feature_name'] = 'Attribute Count'
|
||||
g['chain'] = 'Terra' if False else 'Solana'
|
||||
cur = cur[cur.feature_name != 'Attribute Count']
|
||||
m_df = m_df[ -((m_df.collection == collection) & (m_df.feature_name.isin(['attribute_count', 'Attribute Count']))) ]
|
||||
print(len(m_df))
|
||||
m_df = m_df.append(g)
|
||||
print(len(m_df))
|
||||
m_df.to_csv('./data/metadata.csv', index=False)
|
||||
|
||||
|
||||
m_df = pd.read_csv('./data/metadata.csv')
|
||||
|
||||
solana_rarities = pd.read_csv('./data/solana_rarities.csv')
|
||||
tokens = pd.read_csv('./data/tokens.csv')[['collection','token_id','nft_rank']]
|
||||
solana_rarities = pd.read_csv('./data/solana_rarities.csv')
|
||||
lp_ranks = pd.read_csv('./data/lp_ranks.csv')
|
||||
gp_ranks = pd.read_csv('./data/gp_ranks.csv')
|
||||
@ -26,14 +49,15 @@ if False:
|
||||
metadata['chain'] = metadata.collection.apply(lambda x: 'Terra' if x in ['LunaBulls','Galactic Punks','Levana Dragon Eggs'] else 'Solana' )
|
||||
metadata.to_csv('./data/metadata.csv', index=False)
|
||||
|
||||
rarities = solana_rarities.append(lp_ranks).append(gp_ranks)
|
||||
rarities = solana_rarities.append(lp_ranks).append(gp_ranks).append(tokens).drop_duplicates(keep='first')
|
||||
rarities = rarities[[ 'collection','token_id','nft_rank' ]]
|
||||
rarities['collection'] = rarities.collection.apply(lambda x: clean_name(x) )
|
||||
rarities[ (rarities.collection == 'Solana Monkey Business') & (rarities.token_id == 903) ]
|
||||
# rarities[ (rarities.collection == 'Solana Monkey Business') & (rarities.token_id == 903) ]
|
||||
rarities.loc[ (rarities.collection == 'Solana Monkey Business') & (rarities.token_id == 903) , 'nft_rank' ] = 18
|
||||
rarities['adj_nft_rank_0'] = rarities.nft_rank.apply(lambda x: (x+1) ** -0.2 )
|
||||
rarities['adj_nft_rank_1'] = rarities.nft_rank.apply(lambda x: (x+1) ** -0.9 )
|
||||
rarities['adj_nft_rank_2'] = rarities.nft_rank.apply(lambda x: (x+1) ** -1.4 )
|
||||
print(rarities.groupby('collection').count())
|
||||
|
||||
print(solana_rarities.groupby('collection').token_id.count())
|
||||
print(rarities.groupby('collection').token_id.count())
|
||||
@ -49,17 +73,19 @@ print(m_df[(m_df.token_id=='10') & (m_df.collection == 'Aurory')])
|
||||
|
||||
sorted(m_df.feature_name.unique())
|
||||
|
||||
c = 'Solana Monkey Business'
|
||||
for c in m_df.collection.unique():
|
||||
cur = m_df[m_df.collection == c]
|
||||
base = cur[[ 'collection','token_id','chain' ]].drop_duplicates()
|
||||
for f in cur.feature_name.unique():
|
||||
exists = set(cur[cur.feature_name==f].token_id.unique())
|
||||
missing = sorted(list(set(base.token_id).difference(exists)))
|
||||
a = base[ base.token_id.isin(missing) ]
|
||||
a['feature_name'] = f
|
||||
a['feature_value'] = 'None'
|
||||
m_df = m_df.append(a)
|
||||
fill_missing_metadata = False
|
||||
if fill_missing_metadata:
|
||||
c = 'Solana Monkey Business'
|
||||
for c in m_df.collection.unique():
|
||||
cur = m_df[m_df.collection == c]
|
||||
base = cur[[ 'collection','token_id','chain' ]].drop_duplicates()
|
||||
for f in cur.feature_name.unique():
|
||||
exists = set(cur[cur.feature_name==f].token_id.unique())
|
||||
missing = sorted(list(set(base.token_id).difference(exists)))
|
||||
a = base[ base.token_id.isin(missing) ]
|
||||
a['feature_name'] = f
|
||||
a['feature_value'] = 'None'
|
||||
m_df = m_df.append(a)
|
||||
|
||||
print(m_df[(m_df.token_id=='1') & (m_df.collection == 'Solana Monkey Business')])
|
||||
print(m_df[(m_df.token_id=='10') & (m_df.collection == 'Aurory')])
|
||||
@ -77,17 +103,11 @@ print(m_df[(m_df.token_id=='1') & (m_df.collection == 'Solana Monkey Business')]
|
||||
print(m_df[(m_df.token_id=='10') & (m_df.collection == 'Aurory')])
|
||||
|
||||
m_df['feature_value'] = m_df.feature_value.apply(lambda x: x.strip() if type(x) == str else x )
|
||||
m_df['chain'] = m_df.collection.apply(lambda x: 'Terra' if x in ['LunaBulls','Galactic Punks','Levana Dragon Eggs'] else 'Solana' )
|
||||
m_df.to_csv('./data/metadata.csv', index=False)
|
||||
|
||||
|
||||
g = m_df[['collection','token_id']].drop_duplicates().groupby('collection').token_id.count().reset_index()
|
||||
a = m_df.groupby('collection').token_id.count().reset_index().rename(columns={'token_id':'atts'})
|
||||
g = g.merge(a)
|
||||
g['rat'] = g.atts / g.token_id
|
||||
print(g)
|
||||
|
||||
|
||||
m_df.collection.unique()
|
||||
m_df['chain'] = m_df.collection.apply(lambda x: 'Terra' if x in ['LunaBulls','Galactic Punks','Levana Dragon Eggs'] else 'Solana' )
|
||||
|
||||
m_df.to_csv('./data/metadata.csv', index=False)
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
from operator import index
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
from secrets import token_bytes
|
||||
import time
|
||||
import requests
|
||||
import functools
|
||||
@ -17,40 +19,19 @@ from selenium.webdriver.common.keys import Keys
|
||||
os.chdir('/Users/kellenblumberg/git/nft-deal-score')
|
||||
os.environ['PATH'] += os.pathsep + '/Users/kellenblumberg/shared/'
|
||||
|
||||
from utils import merge, clean_name
|
||||
|
||||
# browser = webdriver.Chrome()
|
||||
|
||||
# old = pd.read_csv('./data/tokens.csv')
|
||||
# metadata[(metadata.collection == 'Galactic Punks') & (metadata.feature_name=='attribute_count')].drop_duplicates(subset=['feature_value']).merge(old)
|
||||
|
||||
clean_names = {
|
||||
'aurory': 'Aurory'
|
||||
,'thugbirdz': 'Thugbirdz'
|
||||
,'smb': 'Solana Monkey Business'
|
||||
,'degenapes': 'Degen Apes'
|
||||
,'peskypenguinclub': 'Pesky Penguins'
|
||||
,'meerkatmillionaires': 'Meerkat Millionaires'
|
||||
,'boryokudragonz': 'Boryoku Dragonz'
|
||||
}
|
||||
|
||||
def clean_name(name):
|
||||
if name in clean_names.keys():
|
||||
return(clean_names[name])
|
||||
return(name)
|
||||
|
||||
def convert_collection_names():
|
||||
d = {
|
||||
'aurory': 'Aurory'
|
||||
,'thugbirdz': 'Thugbirdz'
|
||||
,'smb': 'Solana Monkey Business'
|
||||
,'degenapes': 'Degen Apes'
|
||||
,'peskypenguinclub': 'Pesky Penguins'
|
||||
,'meerkatmillionaires': 'Meerkat Millionaires'
|
||||
,'boryokudragonz': 'Boryoku Dragonz'
|
||||
}
|
||||
for c in [ 'pred_price', 'attributes', 'feature_values', 'model_sales', 'listings', 'coefsdf', 'tokens' ]:
|
||||
try:
|
||||
df = pd.read_csv('./data/{}.csv'.format(c))
|
||||
df['collection'] = df.collection.apply(lambda x: clean_name(x) if x in d.keys() else x )
|
||||
df['collection'] = df.collection.apply(lambda x: clean_name(x) if x in clean_names.keys() else x )
|
||||
df.to_csv('./data/{}.csv'.format(c), index=False)
|
||||
except:
|
||||
print('error',c)
|
||||
@ -401,7 +382,7 @@ def scrape_solanafloor():
|
||||
df.to_csv('./data/sf_projects.csv', index=False)
|
||||
|
||||
|
||||
def scrape_listings(browser, collections = [ 'aurory','thugbirdz','smb','degenapes','peskypenguinclub' ], alerted = [], is_listings = True):
|
||||
def scrape_listings(browser, collections = [ 'degods','aurory','thugbirdz','smb','degenapes','peskypenguinclub' ], alerted = [], is_listings = True):
|
||||
print('Scraping solanafloor listings...')
|
||||
data = []
|
||||
m_data = []
|
||||
@ -418,6 +399,7 @@ def scrape_listings(browser, collections = [ 'aurory','thugbirdz','smb','degenap
|
||||
# old = pd.read_csv('./data/solana_rarities.csv')
|
||||
# collections = sf_projects[(sf_projects.to_scrape==1) & (sf_projects.is_lite==0) & (-sf_projects.collection.isin(old.collection.unique()))].collection.unique()
|
||||
collection = 'portals'
|
||||
collection = 'degods'
|
||||
for collection in collections:
|
||||
if collection == 'boryokudragonz':
|
||||
continue
|
||||
@ -528,6 +510,7 @@ def scrape_listings(browser, collections = [ 'aurory','thugbirdz','smb','degenap
|
||||
,'peskypenguinclub': 'Pesky Penguins'
|
||||
,'meerkatmillionaires': 'Meerkat Millionaires'
|
||||
,'boryokudragonz': 'Boryoku Dragonz'
|
||||
,'degods': 'DeGods'
|
||||
}
|
||||
listings['collection'] = listings.collection.apply(lambda x: clean_name(x))
|
||||
listings[listings.token_id=='1656']
|
||||
@ -546,6 +529,7 @@ def scrape_listings(browser, collections = [ 'aurory','thugbirdz','smb','degenap
|
||||
listings = listings.append(old)
|
||||
print(listings.groupby('collection').token_id.count())
|
||||
listings.to_csv('./data/listings.csv', index=False)
|
||||
return
|
||||
|
||||
listings = listings.sort_values('price')
|
||||
t1 = listings.groupby('collection').head(1).rename(columns={'price':'t1'})
|
||||
@ -694,10 +678,17 @@ def scrape_solanafloor(browser):
|
||||
len(tokens.token_id.unique())
|
||||
tokens[ tokens.collection == 'degenapes' ].sort_values('token_id')
|
||||
old = pd.read_csv('./data/tokens.csv')
|
||||
|
||||
old[old.collection == 'degods']
|
||||
old['collection'] = old.collection.apply(lambda x: clean_name(x) )
|
||||
# old['image_url'] = old.apply(lambda x: 'https://metadata.degods.com/g/{}.png'.format(int(x['token_id']) - 1) if x['collection'] == 'DeGods' else x['image_url'], 1 )
|
||||
old.head()
|
||||
old[ old.collection == 'DeGods' ].head()
|
||||
old[ old.collection == 'LunaBulls' ].head()
|
||||
tokens = old.append(tokens).drop_duplicates()
|
||||
print(tokens.groupby('collection').token_id.count())
|
||||
tokens.to_csv('./data/tokens.csv', index=False)
|
||||
# old.to_csv('./data/tokens.csv', index=False)
|
||||
|
||||
def scrape_solana_explorer(browser):
|
||||
url = 'https://explorer.solana.com/address/9uBX3ASjxWvNBAD1xjbVaKA74mWGZys3RGSF7DdeDD3F/tokens'
|
||||
@ -794,6 +785,7 @@ def scrape_how_rare():
|
||||
'meerkatmillionaires': 10000,
|
||||
'peskypenguinclub': 8888,
|
||||
'boryokudragonz': 1111,
|
||||
'degods': 1111,
|
||||
}
|
||||
k = 'degenapes'
|
||||
v = collections[k]
|
||||
@ -933,30 +925,10 @@ def save_img():
|
||||
urllib.request.urlretrieve(src, './viz/www/img/{}/{}.png'.format('smb', i))
|
||||
|
||||
def scratch():
|
||||
o_metadata = pd.read_csv('./data/metadata.csv')
|
||||
o_sales = pd.read_csv('./data/sales.csv')
|
||||
o_sales[ o_sales.collection == 'Solana Monkey Business' ].head()
|
||||
o_sales[ o_sales.collection == 'Solana Monkey Business' ].sort_values('sale_date', ascending=0).head()
|
||||
o_sales['tmp'] = o_sales.sale_date.apply(lambda x: str(x)[:10] )
|
||||
o_sales['n_sales'] = 1
|
||||
o_sales.groupby(['collection','tmp'])[['price','n_sales']].sum().reset_index().to_csv('./data/tableau_sales_data.csv', index=False)
|
||||
o_metadata.head()
|
||||
o_sales.head()
|
||||
o_sales.to_csv('./data/md_sales.csv', index=False)
|
||||
# get metadata
|
||||
# add rank
|
||||
pass
|
||||
|
||||
def create_mint_csv():
|
||||
mints = pd.DataFrame()
|
||||
auth_to_mint = {}
|
||||
for collection, update_authority in d.items():
|
||||
auth_to_mint[update_authority] = collection
|
||||
for fname in [ './data/mints/'+f for f in os.listdir('./data/mints') ]:
|
||||
pass
|
||||
with open(fname, 'r') as f:
|
||||
j = json.load(f)
|
||||
cur = pd.DataFrame(j)
|
||||
cur.columns = ['mint_address']
|
||||
cur['update_authority'] = re.split('/|_', fname)[3]
|
||||
cur['collection'] = re.split('/|_', fname)[3]
|
||||
|
||||
def scrape_how_rare_is():
|
||||
d = {
|
||||
@ -991,10 +963,82 @@ def scrape_how_rare_is():
|
||||
df['clean_token_id'] = df.token_id
|
||||
df['chain'] = 'Solana'
|
||||
tokens = pd.read_csv('./data/tokens.csv')
|
||||
tokens.collection.unique()
|
||||
tokens = tokens[-tokens.collection.isin(df.collection.unique())]
|
||||
tokens = tokens.append(df)
|
||||
tokens.to_csv('./data/tokens.csv', index=False)
|
||||
|
||||
def scrape_howrare_token_id_mint_map():
|
||||
mints = pd.read_csv('./data/solana_mints.csv')
|
||||
collection = 'DeGods'
|
||||
s = requests.Session()
|
||||
s.get('https://magiceden.io/')
|
||||
r = s.get('https://httpbin.org/cookies')
|
||||
browser.manage().getCookies()
|
||||
cookie_list = browser.get_cookies()
|
||||
cookies = {}
|
||||
for c in cookie_list:
|
||||
cookies[c['name']] = c['value']
|
||||
|
||||
|
||||
for collection in [ 'DeGods' ]:
|
||||
it = 0
|
||||
for mint_address in sorted(mints[mints.collection == collection].mint_address.unique()):
|
||||
it += 1
|
||||
url = 'https://api-mainnet.magiceden.io/rpc/getNFTByMintAddress/{}'.format(mint_address)
|
||||
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
|
||||
headers = {'authority': 'api-mainnet.magiceden.io',
|
||||
'cache-control': 'max-age=0',
|
||||
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"macOS"',
|
||||
'upgrade-insecure-requests': '1',
|
||||
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36',
|
||||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
|
||||
'sec-fetch-site': 'none',
|
||||
'sec-fetch-mode': 'navigate',
|
||||
'sec-fetch-user': '?1',
|
||||
'sec-fetch-dest': 'document',
|
||||
'accept-language': 'en-US,en;q=0.9',
|
||||
'cookie': 'ajs_anonymous_id=eba2f8f7-3519-4110-a1c2-4fbb1c350ae4; rl_page_init_referrer=RudderEncrypt%3AU2FsdGVkX19wSK5HkzEPJocowXJThl3gTmPeTtvkaH8%3D; rl_page_init_referring_domain=RudderEncrypt%3AU2FsdGVkX18tu9VUWzMjRFJLT3XFToyFsfZjjQBl4HM%3D; _clck=1xl6xwz|1|ez0|0; __cf_bm=BgwftpyDbLNKxpuLYvLE1ekRTn2MWF7KU5Suu3SRMlo-1644967874-0-ATSK7m2xYwOl+xF+SGhqbInjBZiL4ywBWBVroe1O22PBjZQtMzt8Pno6Q4panpaxXuaj7ys/wWkSRSAqGYih8PCDdzizesQyOA/U9HVidpqTXFRU/ckeufeFfxhesga4Sg==; rl_user_id=RudderEncrypt%3AU2FsdGVkX189KreVOKsVULLvR9eM%2FKE7IkW6qokXvIvedAi9vH71haADdBNKEtqdFIzVlBAeHTR411o5svVVeQ%3D%3D; rl_anonymous_id=RudderEncrypt%3AU2FsdGVkX184cKLxwy%2FrsyLEVzTyifhXO8CNRjMrODDx8Xe%2FukvGqi7V%2FMgmldWtnjIq1VLhmwusKYIZ%2FFAeSQ%3D%3D; rl_group_id=RudderEncrypt%3AU2FsdGVkX1%2FCcC9ITF5hwaucXM%2Bz4tvMxvI0YaIIGa0%3D; rl_trait=RudderEncrypt%3AU2FsdGVkX19PxxarpffnZFAUNe2tgmUmZC6txkvO2Z7ccbw4JrssvPZMZ9w2c1g3%2FUuHawkxfqu5d2FvhirmPn7f0dSP29A9UVJ3eHmXYBliJzuT8XW1B9prqWgRWWz3GpqGGJGXa%2BoqX1MQLCvltunTqQR%2Fxw2CKW%2BZDPsas89XXY0NlmixoT81TgLEPKddSjZDBihP09u9aM1YvCCxuw%3D%3D; rl_group_trait=RudderEncrypt%3AU2FsdGVkX1%2BUocb9WeinM5mlF6NiPC8ISK%2FYcQgmPYk%3D; _clsk=fewr53|1644967907370|3|0|e.clarity.ms/collect',
|
||||
'if-none-match': 'W/"564-Uw9zGORiLyOok/cpoNblXV6SsrI"'}
|
||||
r = s.get(url, headers=headers)
|
||||
browser.get(url)
|
||||
s = json.loads(BeautifulSoup(browser.page_source).text)
|
||||
token_id = re.split('#', s['results']['title'])[-1]
|
||||
data += [[ collection, mint_address, token_id ]]
|
||||
if it % 25 == 0:
|
||||
print(it, len(data))
|
||||
print(data[-2:])
|
||||
|
||||
r = requests.get(url, cookies = dict(cookies), headers=headers)
|
||||
s.get(url, cookies = dict(cookies), headers=headers)
|
||||
r = s.get(url)
|
||||
sorted(mints.collection.unique())
|
||||
data = []
|
||||
s_data = []
|
||||
for i in range(1, 10001):
|
||||
if i % 25 == 0:
|
||||
print(i, len(data), len(s_data))
|
||||
if len(data) > 1:
|
||||
print(data[-2:])
|
||||
if len(s_data) > 1:
|
||||
print(s_data[-2:])
|
||||
url = 'https://howrare.is/{}/{}/'.format(collection, i)
|
||||
browser.get(url)
|
||||
sleep(0.01)
|
||||
soup = BeautifulSoup(browser.page_source)
|
||||
mint_address = re.split('/', soup.find_all('div', class_="nft_title")[0].find_all('div', class_='overflow')[0].find_all('a')[0].attrs['href'])[-1]
|
||||
data += [[ collection, i, mint_address ]]
|
||||
table = soup.find_all('div', class_='sales_history')
|
||||
if len(table) == 0:
|
||||
continue
|
||||
for row in table[0].find_all('div', class_='all_coll_row')[1:]:
|
||||
cells = row.find_all('div', class_='all_coll_col')
|
||||
if len(cells) > 1:
|
||||
s_data += [[ collection, i, cells[0].text.strip(), cells[1].text.strip() ]]
|
||||
df = pd.DataFrame(data, columns=['collection','token_id','mint_address'])
|
||||
df.to_csv('./data/solana_token_id_mint_map.csv', index=False)
|
||||
|
||||
def metadata_from_solscan():
|
||||
collections = [
|
||||
@ -1003,16 +1047,20 @@ def metadata_from_solscan():
|
||||
, [ 'balloonsville', 'https://bafybeih5i7lktx6o7rjceuqvlxmpqzwfh4nhr322wq5hjncxbicf4fbq2e.ipfs.dweb.link/{}.json', 0, 5000 ]
|
||||
]
|
||||
data = []
|
||||
token_data = []
|
||||
collection = 'DeGods'
|
||||
for i in range(0, 5000):
|
||||
if i % 25 == 2:
|
||||
print(i, len(data))
|
||||
print(data[-1])
|
||||
print(token_data[-1])
|
||||
url = 'https://sld-gengo.s3.amazonaws.com/{}.json'.format(i)
|
||||
url = 'https://metadata.degods.com/g/{}.json'.format(i)
|
||||
url = 'https://bafybeih5i7lktx6o7rjceuqvlxmpqzwfh4nhr322wq5hjncxbicf4fbq2e.ipfs.dweb.link/{}.json'.format(i)
|
||||
url = 'https://metadata.degods.com/g/{}.json'.format(i)
|
||||
r = requests.get(url).json()
|
||||
token_data += [[ collection, i, r['image'] ]]
|
||||
for a in r['attributes']:
|
||||
data += [[ 'balloonsville', i, a['trait_type'], a['value'] ]]
|
||||
data += [[ collection, i, a['trait_type'], a['value'] ]]
|
||||
df = pd.DataFrame(data, columns=['collection','token_id','feature_name','feature_value']).drop_duplicates()
|
||||
if False:
|
||||
df['token_id'] = df.token_id + 1
|
||||
@ -1021,7 +1069,22 @@ def metadata_from_solscan():
|
||||
old = old.drop_duplicates(keep='last')
|
||||
print(old[['collection','token_id']].drop_duplicates().groupby('collection').token_id.count())
|
||||
old.to_csv('./data/solscan_metadata.csv', index=False)
|
||||
|
||||
|
||||
def add_solscan_metadata():
|
||||
solscan_metadata = pd.read_csv('./data/solscan_metadata.csv')
|
||||
metadata = pd.read_csv('./data/metadata.csv')
|
||||
solscan_metadata['collection'] = solscan_metadata.collection.apply(lambda x: clean_name(x) )
|
||||
metadata['collection'] = metadata.collection.apply(lambda x: clean_name(x) )
|
||||
rem = solscan_metadata[['collection','feature_name']].drop_duplicates()
|
||||
rem['rem'] = 1
|
||||
metadata = merge(metadata, rem, how='left', ensure=True)
|
||||
metadata = metadata[metadata.rem.isnull()]
|
||||
del metadata['rem']
|
||||
metadata[metadata.collection == 'DeGods'][['collection','feature_name']].drop_duplicates()
|
||||
metadata = metadata.append(solscan_metadata)
|
||||
metadata[metadata.collection == 'DeGods'][['collection','feature_name']].drop_duplicates()
|
||||
metadata.to_csv('./data/metadata.csv', index=False)
|
||||
|
||||
def scrape_mints():
|
||||
|
||||
nft_mint_addresses = pd.read_csv('./data/nft_mint_addresses.csv')
|
||||
@ -1040,6 +1103,8 @@ def scrape_mints():
|
||||
nft_mint_addresses = nft_mint_addresses.merge( solana_nfts )
|
||||
nft_mint_addresses.collection.unique()
|
||||
mints = pd.read_csv('./data/solana_mints.csv')
|
||||
mints[mints.collection == 'DeGods'].to_csv('~/Downloads/tmp.csv', index=False)
|
||||
mints[mints.collection == 'DeGods'].drop_duplicates().to_csv('~/Downloads/tmp.csv', index=False)
|
||||
sorted(mints.collection.unique())
|
||||
mints[mints.collection == 'Kaiju Cards']
|
||||
# mints['tmp'] = mints.mint_address.apply(lambda x: x.lower() )
|
||||
@ -1088,6 +1153,10 @@ def scrape_mints():
|
||||
mints.to_csv('./data/solana_mints.csv', index=False)
|
||||
mints[mints.collection == 'Balloonsville'].to_csv('./data/solana_mints_2.csv', index=False)
|
||||
|
||||
url = 'https://solscan.io/token/GxERCTcBDmB6pfEoYYNWvioAhACifEGdn3dXNqVh5rXz'
|
||||
url = 'https://explorer.solana.com/address/6CCprsgJT4nxBMSitGathXcLshDTL3BE4LcJXvSFwoe2'
|
||||
r = requests.get(url)
|
||||
|
||||
# scrape_listings(['smb'])
|
||||
# alerted = []
|
||||
# for i in range(1):
|
||||
|
||||
1
scripts/solana-rpc-app/output.txt
Normal file
1
scripts/solana-rpc-app/output.txt
Normal file
File diff suppressed because one or more lines are too long
1
scripts/solana-rpc-app/output2.txt
Normal file
1
scripts/solana-rpc-app/output2.txt
Normal file
File diff suppressed because one or more lines are too long
59
scripts/solana-rpc-app/src/test.js
Normal file
59
scripts/solana-rpc-app/src/test.js
Normal file
@ -0,0 +1,59 @@
|
||||
"use strict";
|
||||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
||||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
||||
return new (P || (P = Promise))(function (resolve, reject) {
|
||||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
||||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
||||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
||||
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
||||
});
|
||||
};
|
||||
var __generator = (this && this.__generator) || function (thisArg, body) {
|
||||
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
||||
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
||||
function verb(n) { return function (v) { return step([n, v]); }; }
|
||||
function step(op) {
|
||||
if (f) throw new TypeError("Generator is already executing.");
|
||||
while (_) try {
|
||||
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
||||
if (y = 0, t) op = [op[0] & 2, t.value];
|
||||
switch (op[0]) {
|
||||
case 0: case 1: t = op; break;
|
||||
case 4: _.label++; return { value: op[1], done: false };
|
||||
case 5: _.label++; y = op[1]; op = [0]; continue;
|
||||
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
||||
default:
|
||||
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
||||
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
||||
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
||||
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
||||
if (t[2]) _.ops.pop();
|
||||
_.trys.pop(); continue;
|
||||
}
|
||||
op = body.call(thisArg, _);
|
||||
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
||||
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
||||
}
|
||||
};
|
||||
exports.__esModule = true;
|
||||
var js_1 = require("@metaplex/js");
|
||||
var mpl_token_metadata_1 = require("@metaplex-foundation/mpl-token-metadata");
|
||||
var web3_js_1 = require("@solana/web3.js");
|
||||
(function () { return __awaiter(void 0, void 0, void 0, function () {
|
||||
var connection, tokenMint, metadataPDA, tokenMetadata;
|
||||
return __generator(this, function (_a) {
|
||||
switch (_a.label) {
|
||||
case 0:
|
||||
connection = new js_1.Connection('mainnet-beta');
|
||||
tokenMint = '5XKoz4nuPFU78jcEVREMZoh9kKsYnCvrTAmpRzvVdJp1';
|
||||
return [4 /*yield*/, mpl_token_metadata_1.Metadata.getPDA(new web3_js_1.PublicKey(tokenMint))];
|
||||
case 1:
|
||||
metadataPDA = _a.sent();
|
||||
return [4 /*yield*/, mpl_token_metadata_1.Metadata.load(connection, metadataPDA)];
|
||||
case 2:
|
||||
tokenMetadata = _a.sent();
|
||||
console.log(tokenMetadata.data);
|
||||
return [2 /*return*/];
|
||||
}
|
||||
});
|
||||
}); })();
|
||||
94
scripts/solana-rpc-app/src/test2.js
Normal file
94
scripts/solana-rpc-app/src/test2.js
Normal file
@ -0,0 +1,94 @@
|
||||
"use strict";
|
||||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
||||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
||||
return new (P || (P = Promise))(function (resolve, reject) {
|
||||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
||||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
||||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
||||
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
||||
});
|
||||
};
|
||||
var __generator = (this && this.__generator) || function (thisArg, body) {
|
||||
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
||||
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
||||
function verb(n) { return function (v) { return step([n, v]); }; }
|
||||
function step(op) {
|
||||
if (f) throw new TypeError("Generator is already executing.");
|
||||
while (_) try {
|
||||
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
||||
if (y = 0, t) op = [op[0] & 2, t.value];
|
||||
switch (op[0]) {
|
||||
case 0: case 1: t = op; break;
|
||||
case 4: _.label++; return { value: op[1], done: false };
|
||||
case 5: _.label++; y = op[1]; op = [0]; continue;
|
||||
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
||||
default:
|
||||
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
||||
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
||||
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
||||
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
||||
if (t[2]) _.ops.pop();
|
||||
_.trys.pop(); continue;
|
||||
}
|
||||
op = body.call(thisArg, _);
|
||||
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
||||
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
||||
}
|
||||
};
|
||||
exports.__esModule = true;
|
||||
var web3_js_1 = require("@solana/web3.js");
|
||||
// import bs58 from 'bs58';
|
||||
var connection = new web3_js_1.Connection(web3_js_1.clusterApiUrl('mainnet-beta'));
|
||||
var MAX_NAME_LENGTH = 32;
|
||||
var MAX_URI_LENGTH = 200;
|
||||
var MAX_SYMBOL_LENGTH = 10;
|
||||
var MAX_CREATOR_LEN = 32 + 1 + 1;
|
||||
var MAX_CREATOR_LIMIT = 5;
|
||||
var MAX_DATA_SIZE = 4 + MAX_NAME_LENGTH + 4 + MAX_SYMBOL_LENGTH + 4 + MAX_URI_LENGTH + 2 + 1 + 4 + MAX_CREATOR_LIMIT * MAX_CREATOR_LEN;
|
||||
var MAX_METADATA_LEN = 1 + 32 + 32 + MAX_DATA_SIZE + 1 + 1 + 9 + 172;
|
||||
var CREATOR_ARRAY_START = 1 + 32 + 32 + 4 + MAX_NAME_LENGTH + 4 + MAX_URI_LENGTH + 4 + MAX_SYMBOL_LENGTH + 2 + 1 + 4;
|
||||
// const TOKEN_METADATA_PROGRAM = new PublicKey('metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s');
|
||||
var TOKEN_METADATA_PROGRAM = new web3_js_1.PublicKey('TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA');
|
||||
var candyMachineId = new web3_js_1.PublicKey('8mNmf15xNrMFQLNSNrHxxswy7a1NfaSFwXHkVUPeMWwU');
|
||||
var getMintAddresses = function (firstCreatorAddress) { return __awaiter(void 0, void 0, void 0, function () {
|
||||
var metadataAccounts;
|
||||
return __generator(this, function (_a) {
|
||||
switch (_a.label) {
|
||||
case 0: return [4 /*yield*/, connection.getProgramAccounts(TOKEN_METADATA_PROGRAM, {
|
||||
// The mint address is located at byte 33 and lasts for 32 bytes.
|
||||
// dataSlice: { offset: 33, length: 32 },
|
||||
filters: [
|
||||
// Only get Metadata accounts.
|
||||
// { dataSize: MAX_METADATA_LEN },
|
||||
{ dataSize: 165 },
|
||||
// Filter using the first creator.
|
||||
{
|
||||
memcmp: {
|
||||
// offset: CREATOR_ARRAY_START,
|
||||
// bytes: firstCreatorAddress.toBase58(),
|
||||
offset: 1,
|
||||
bytes: new web3_js_1.PublicKey('4FYjfa71puV4PD12cyqXotu6z2FhLiqFSHjEfYiFLnbj').toBase58()
|
||||
}
|
||||
},
|
||||
]
|
||||
})];
|
||||
case 1:
|
||||
metadataAccounts = _a.sent();
|
||||
return [2 /*return*/, metadataAccounts];
|
||||
}
|
||||
});
|
||||
}); };
|
||||
(function () { return __awaiter(void 0, void 0, void 0, function () {
|
||||
var a;
|
||||
return __generator(this, function (_a) {
|
||||
switch (_a.label) {
|
||||
case 0: return [4 /*yield*/, getMintAddresses(candyMachineId)];
|
||||
case 1:
|
||||
a = _a.sent();
|
||||
console.log("a");
|
||||
console.log(a);
|
||||
console.log(a.length);
|
||||
return [2 /*return*/];
|
||||
}
|
||||
});
|
||||
}); })();
|
||||
73
scripts/solana-rpc-app/src/test3.js
Normal file
73
scripts/solana-rpc-app/src/test3.js
Normal file
File diff suppressed because one or more lines are too long
37
scripts/solana-rpc-app/src/test3.ts
Normal file
37
scripts/solana-rpc-app/src/test3.ts
Normal file
File diff suppressed because one or more lines are too long
@ -19,8 +19,9 @@ from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
|
||||
from sklearn.linear_model import LinearRegression, RidgeCV, Lasso, Ridge
|
||||
from sklearn.model_selection import train_test_split, KFold, GridSearchCV, RandomizedSearchCV
|
||||
|
||||
from utils import merge, clean_name
|
||||
|
||||
os.chdir('/Users/kellenblumberg/git/nft-deal-score')
|
||||
from scrape_sol_nfts import clean_name
|
||||
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
@ -43,22 +44,6 @@ def standardize_df(df, cols, usedf=None, verbose=False):
|
||||
df['std_{}'.format(c)] = (df[c] - mu) / sd
|
||||
return(df)
|
||||
|
||||
def merge(left, right, on=None, how='inner', ensure=True, verbose=True, message = ''):
|
||||
df = left.merge(right, on=on, how=how)
|
||||
if len(df) != len(left) and (ensure or verbose):
|
||||
if message:
|
||||
print(message)
|
||||
print('{} -> {}'.format(len(left), len(df)))
|
||||
cur = left.merge(right, on=on, how='left')
|
||||
cols = set(right.columns).difference(set(left.columns))
|
||||
print(cols)
|
||||
if ensure:
|
||||
col = list(cols)[0]
|
||||
missing = cur[cur[col].isnull()]
|
||||
print(missing.head())
|
||||
assert(False)
|
||||
return(df)
|
||||
|
||||
def just_float(x):
|
||||
x = re.sub('[^\d\.]', '', str(x))
|
||||
return(float(x))
|
||||
@ -170,6 +155,8 @@ def train_model(check_exclude, supplement_with_listings):
|
||||
# Load Metadata #
|
||||
#########################
|
||||
m_df = pd.read_csv('./data/metadata.csv')
|
||||
m_df[m_df.collection == 'DeGods'][['feature_name']].drop_duplicates()
|
||||
sorted(m_df.collection.unique())
|
||||
# m_df[m_df.collection == 'Aurory'][['collection','feature_name']].drop_duplicates().to_csv('~/Downloads/tmp.csv', index=False)
|
||||
sorted([x for x in m_df.feature_name.unique() if 'nft_' in x])
|
||||
m_df['token_id'] = m_df.token_id.astype(str)
|
||||
@ -187,11 +174,11 @@ def train_model(check_exclude, supplement_with_listings):
|
||||
tokens = pd.read_csv('./data/tokens.csv')
|
||||
tokens['collection'] = tokens.collection.apply(lambda x: clean_name(x))
|
||||
tokens['token_id'] = tokens.token_id.astype(str)
|
||||
m_df = merge(m_df, tokens[['collection','token_id','clean_token_id']], how='left', ensure=True, on=['collection','token_id'], message='m_df x tokens')
|
||||
m_df['token_id'] = m_df.clean_token_id.fillna(m_df.token_id).astype(int).astype(str)
|
||||
m_df = merge(m_df, tokens[['collection','token_id','clean_token_id']].dropna().drop_duplicates() , how='left', ensure=True, on=['collection','token_id'], message='m_df x tokens')
|
||||
m_df['token_id'] = m_df.clean_token_id.fillna(m_df.token_id).astype(float).astype(int).astype(str)
|
||||
s_df = merge(s_df, tokens[['collection','token_id','clean_token_id']], how='left', ensure=True, on=['collection','token_id'], message='s_df x tokens')
|
||||
s_df['token_id'] = s_df.clean_token_id.fillna(s_df.token_id).astype(int).astype(str)
|
||||
tokens.token_id.unique()
|
||||
s_df[s_df.token_id.isnull()]
|
||||
s_df['token_id'] = (s_df.clean_token_id.replace('nan', None).fillna(s_df.token_id.replace('nan', None))).astype(float).astype(int).astype(str)
|
||||
lunabullsrem = tokens[tokens.clean_token_id>=10000].token_id.unique()
|
||||
m_df = m_df[ -((m_df.collection == 'LunaBulls') & (m_df.token_id.isin(lunabullsrem))) ]
|
||||
s_df = s_df[ -((s_df.collection == 'LunaBulls') & (s_df.token_id.isin(lunabullsrem))) ]
|
||||
@ -292,7 +279,7 @@ def train_model(check_exclude, supplement_with_listings):
|
||||
|
||||
collection = 'Aurory'
|
||||
collection = 'Solana Monkey Business'
|
||||
collection = 'Levana Dragon Eggs'
|
||||
collection = 'DeGods'
|
||||
# for collection in [ 'Solana Monkey Business' ]:
|
||||
# for collection in [ 'Aurory' ]:
|
||||
# for collection in [ 'Aurory','Solana Monkey Business' ]:
|
||||
@ -300,8 +287,8 @@ def train_model(check_exclude, supplement_with_listings):
|
||||
sorted(pred_price.collection.unique())
|
||||
sorted(s_df.collection.unique())
|
||||
print(sorted(m_df.collection.unique()))
|
||||
# for collection in [ 'Solana Monkey Business' ]:
|
||||
for collection in m_df.collection.unique():
|
||||
# for collection in m_df.collection.unique():
|
||||
for collection in [ 'DeGods' ]:
|
||||
coefsdf = coefsdf[coefsdf.collection != collection]
|
||||
salesdf = salesdf[salesdf.collection != collection]
|
||||
attributes = attributes[attributes.collection != collection]
|
||||
@ -310,8 +297,9 @@ def train_model(check_exclude, supplement_with_listings):
|
||||
print('Working on collection {}'.format(collection))
|
||||
sales = s_df[ s_df.collection == collection ]
|
||||
metadata = m_df[ m_df.collection == collection ]
|
||||
metadata.groupby(['feature_name','feature_value']).token_id.count().reset_index().to_csv('~/Downloads/tmp.csv', index=False)
|
||||
metadata[metadata.token_id == '1']
|
||||
sorted(metadata.feature_name.unique())
|
||||
# metadata.groupby(['feature_name','feature_value']).token_id.count().reset_index().to_csv('~/Downloads/tmp.csv', index=False)
|
||||
# metadata[metadata.token_id == '1']
|
||||
metadata['feature_name'] = metadata.feature_name.apply(lambda x: x.strip() )
|
||||
metadata[metadata.token_id == '1']
|
||||
metadata[metadata.feature_name == 'rank']
|
||||
@ -688,24 +676,28 @@ def train_model(check_exclude, supplement_with_listings):
|
||||
|
||||
coefsdf.to_csv('./data/coefsdf.csv', index=False)
|
||||
salesdf.to_csv('./data/model_sales.csv', index=False)
|
||||
old = pd.read_csv('./data/pred_price copy.csv')
|
||||
old['token_id'] = old.token_id.astype(str)
|
||||
old = pred_price.merge(old, on=['collection','token_id'])
|
||||
old['ratio'] = old.pred_price_x / old.pred_price_y
|
||||
old = old.sort_values('ratio')
|
||||
old.columns = [ 'collection', 'token_id', 'nft_rank', 'rk_new', 'pred_price_new', 'pred_sd_x', 'rank', 'rk_old', 'pred_price_old', 'pred_sd_y', 'clean_token_id', 'ratio' ]
|
||||
m = m_df[(m_df.collection.isin(pred_price.collection.unique())) & (-(m_df.feature_name.isin(['nft_rank','adj_nft_rank_0','adj_nft_rank_1','adj_nft_rank_2'])))]
|
||||
m_p = m.pivot(['collection','token_id'], ['feature_name'], ['feature_value']).reset_index()
|
||||
m_p.columns = [ 'collection','token_id' ] + sorted(m.feature_name.unique())
|
||||
m_p.head()
|
||||
old = old.merge(m_p, on=['collection','token_id'])
|
||||
if len(old) and 'rank' in old.columns:
|
||||
old = old[[ 'token_id', 'nft_rank', 'rk_old', 'rk_new', 'pred_price_old', 'pred_price_new', 'ratio' ] + [c for c in m_p.columns if not c in ['token_id','collection']]]
|
||||
old.to_csv('~/Downloads/tmp1.csv', index=False)
|
||||
pred_price.head()
|
||||
old[old.token_id == '4857']
|
||||
old.head()
|
||||
old.tail()
|
||||
pred_price[pred_price.collection == 'DeGods'].to_csv('~/Downloads/tmp1.csv', index=False)
|
||||
# old = pd.read_csv('./data/pred_price.csv')
|
||||
# old = old[old.collection == 'DeGods']
|
||||
# old['token_id'] = old.token_id.astype(str)
|
||||
# old = pred_price.merge(old, on=['collection','token_id'])
|
||||
# old['ratio'] = old.pred_price_x / old.pred_price_y
|
||||
# old = old.sort_values('ratio')
|
||||
# old.columns = [ 'collection', 'token_id', 'nft_rank', 'rk_new', 'pred_price_new', 'pred_sd_x', 'rank', 'rk_old', 'pred_price_old', 'pred_sd_y', 'ratio' ]
|
||||
# # old.columns = [ 'collection', 'token_id', 'nft_rank', 'rk_new', 'pred_price_new', 'pred_sd_x', 'rank', 'rk_old', 'pred_price_old', 'pred_sd_y', 'clean_token_id', 'ratio' ]
|
||||
# m = m_df[(m_df.collection.isin(pred_price.collection.unique())) & (-(m_df.feature_name.isin(['nft_rank','adj_nft_rank_0','adj_nft_rank_1','adj_nft_rank_2'])))]
|
||||
# m_p = m.pivot(['collection','token_id'], ['feature_name'], ['feature_value']).reset_index()
|
||||
# m_p.columns = [ 'collection','token_id' ] + sorted(m.feature_name.unique())
|
||||
# m_p.head()
|
||||
# old = old.merge(m_p, on=['collection','token_id'])
|
||||
# if len(old) and 'rank' in old.columns:
|
||||
# # old = old[[ 'token_id', 'nft_rank', 'rk_old', 'rk_new', 'pred_price_old', 'pred_price_new', 'ratio' ] + [c for c in m_p.columns if not c in ['token_id','collection']]]
|
||||
# old = old[[ 'token_id', 'nft_rank', 'rk_old', 'rk_new', 'pred_price_old', 'pred_price_new', 'ratio' ] + [c for c in m_p.columns if not c in ['token_id','collection','rank']]]
|
||||
# old.to_csv('~/Downloads/tmp1.csv', index=False)
|
||||
# pred_price.head()
|
||||
# old[old.token_id == '4857']
|
||||
# old.head()
|
||||
# old.tail()
|
||||
|
||||
# nft_rank = m_df[m_df.feature_name=='nft_rank'][['collection','token_id','feature_value']].rename(columns={'feature_value': 'nft_rank'})
|
||||
# nft_rank['token_id'] = nft_rank.token_id.astype(str)
|
||||
|
||||
174
update.py
174
update.py
@ -17,15 +17,15 @@ browser = webdriver.Chrome()
|
||||
|
||||
|
||||
if False:
|
||||
alerted = []
|
||||
for i in range(10):
|
||||
ssn.scrape_randomearth(browser)
|
||||
update_token_ids()
|
||||
listings = pd.read_csv('./data/listings.csv')
|
||||
listings = listings[listings.chain == 'Terra']
|
||||
listings.collection.unique()
|
||||
alerted = ssn.calculate_deal_scores(listings, alerted)
|
||||
sleep(10 * 60)
|
||||
alerted = []
|
||||
for i in range(10):
|
||||
ssn.scrape_randomearth(browser)
|
||||
update_token_ids()
|
||||
listings = pd.read_csv('./data/listings.csv')
|
||||
listings = listings[listings.chain == 'Terra']
|
||||
listings.collection.unique()
|
||||
alerted = ssn.calculate_deal_scores(listings, alerted)
|
||||
sleep(10 * 60)
|
||||
|
||||
|
||||
|
||||
@ -67,84 +67,92 @@ ssn.scrape_listings(browser)
|
||||
# sales.price.max()
|
||||
|
||||
def add_model_sales():
|
||||
sales = pd.read_csv('./data/sales.csv').rename(columns={'sale_date':'block_timestamp'})
|
||||
print(sales.groupby('collection').token_id.count())
|
||||
sales.token_id.unique()
|
||||
sales.groupby('collection').token_id.count()
|
||||
sales[sales.collection == 'Galactic Punks']
|
||||
del sales['tx_id']
|
||||
old = pd.read_csv('./data/pred_price.csv').rename(columns={'rank':'nft_rank'})
|
||||
old = pd.read_csv('./data/pred_price copy.csv').rename(columns={'rank':'nft_rank'})
|
||||
old.groupby('collection').token_id.count()
|
||||
sales['token_id'] = sales.token_id.astype(int).astype(str)
|
||||
old['token_id'] = old.token_id.astype(str)
|
||||
sales = sales.merge( old[['collection','token_id','nft_rank']] )
|
||||
sales.head()
|
||||
sales['block_timestamp'] = sales.block_timestamp.apply(lambda x: str(x)[:19] )
|
||||
sales['price'] = sales.price.apply(lambda x: round(x, 2))
|
||||
print(sales.groupby('collection').token_id.count())
|
||||
sales.to_csv('./data/model_sales.csv', index=False)
|
||||
sales = pd.read_csv('./data/sales.csv').rename(columns={'sale_date':'block_timestamp'})
|
||||
print(sales.groupby('collection').token_id.count())
|
||||
sales.token_id.unique()
|
||||
sales.groupby('collection').token_id.count()
|
||||
sales[sales.collection == 'Galactic Punks']
|
||||
del sales['tx_id']
|
||||
old = pd.read_csv('./data/pred_price.csv').rename(columns={'rank':'nft_rank'})
|
||||
old = pd.read_csv('./data/pred_price copy.csv').rename(columns={'rank':'nft_rank'})
|
||||
old.groupby('collection').token_id.count()
|
||||
sales['token_id'] = sales.token_id.astype(int).astype(str)
|
||||
old['token_id'] = old.token_id.astype(str)
|
||||
sales = sales.merge( old[['collection','token_id','nft_rank']] )
|
||||
sales.head()
|
||||
sales['block_timestamp'] = sales.block_timestamp.apply(lambda x: str(x)[:19] )
|
||||
sales['price'] = sales.price.apply(lambda x: round(x, 2))
|
||||
print(sales.groupby('collection').token_id.count())
|
||||
sales.to_csv('./data/model_sales.csv', index=False)
|
||||
sales = pd.read_csv('./data/model_sales.csv')
|
||||
print(len(sales))
|
||||
sales = sales.drop_duplicates(subset=['collection','token_id','price'])
|
||||
print(len(sales))
|
||||
sales.to_csv('./data/model_sales.csv', index=False)
|
||||
|
||||
|
||||
def update_token_ids():
|
||||
tokens = pd.read_csv('./data/tokens.csv')
|
||||
tokens.groupby('collection').token_id.count()
|
||||
tokens['tmp'] = tokens.token_id.apply(lambda x: (int(float(x))) )
|
||||
tokens[tokens.token_id == 223838831896070003935953339589523931136]
|
||||
tokens[tokens.collection=='Galactic Punks']
|
||||
tokens['token_id'] = tokens.token_id.apply(lambda x: str(int(float(x))) )
|
||||
tokens['tmp'] = tokens.token_id.apply(lambda x: len(x) )
|
||||
tokens.tmp.max()
|
||||
# df[ (df.collection == 'Pesky Penguins') & (df.token_id == '3362') ]
|
||||
tokens[ (tokens.collection == 'Pesky Penguins') & (tokens.token_id == '3362') ]
|
||||
tokens[ (tokens.collection == 'Pesky Penguins') & (tokens.token_id == 3362) ]
|
||||
# df.token_id.unique()
|
||||
c = 'listings'
|
||||
for c in [ 'attributes','sales','listings' ]:
|
||||
print(c)
|
||||
df = pd.read_csv('./data/{}.csv'.format(c))
|
||||
df['token_id'] = df.token_id.apply(lambda x: str(int(float(x))) )
|
||||
df['tmp'] = df.token_id.apply(lambda x: (str(x)[:5]))
|
||||
df['tmp'] = df.token_id.apply(lambda x: x[:10] )
|
||||
tokens['tmp'] = tokens.token_id.apply(lambda x: x[:10] )
|
||||
len(tokens)
|
||||
len(tokens[['collection','token_id']].drop_duplicates())
|
||||
len(tokens[['collection','tmp']].drop_duplicates())
|
||||
df.to_csv('~/Downloads/tmp2.csv', index=False)
|
||||
if 'clean_token_id' in df.columns:
|
||||
del df['clean_token_id']
|
||||
|
||||
tokens[tokens.collection=='Galactic Punks']
|
||||
len(tokens[tokens.collection=='Galactic Punks'])
|
||||
tokens[(tokens.collection=='Galactic Punks') & (tokens.token_id=='25984997114855597728010029317878710272')]
|
||||
# 25984997114855639851202718743284654443
|
||||
# 25984997114855597728010029317878710272
|
||||
tokens = pd.read_csv('./data/tokens.csv')
|
||||
tokens.groupby('collection').token_id.count()
|
||||
tokens['tmp'] = tokens.token_id.apply(lambda x: (int(float(x))) )
|
||||
tokens[tokens.token_id == 223838831896070003935953339589523931136]
|
||||
tokens[tokens.collection=='Galactic Punks']
|
||||
tokens['token_id'] = tokens.token_id.apply(lambda x: str(int(float(x))) )
|
||||
tokens['tmp'] = tokens.token_id.apply(lambda x: len(x) )
|
||||
tokens.tmp.max()
|
||||
# df[ (df.collection == 'Pesky Penguins') & (df.token_id == '3362') ]
|
||||
tokens[ (tokens.collection == 'Pesky Penguins') & (tokens.token_id == '3362') ]
|
||||
tokens[ (tokens.collection == 'Pesky Penguins') & (tokens.token_id == 3362) ]
|
||||
# df.token_id.unique()
|
||||
c = 'listings'
|
||||
for c in [ 'attributes','sales','listings' ]:
|
||||
print(c)
|
||||
df = pd.read_csv('./data/{}.csv'.format(c))
|
||||
# df.token_id.unique()
|
||||
df = df[df.token_id.notnull()]
|
||||
df['token_id'] = df.token_id.apply(lambda x: None if x == 'nan' else str(int(float(x))) )
|
||||
df['token_id'] = df.token_id.apply(lambda x: None if x == 'nan' else str(int(float(x))) )
|
||||
# df['tmp'] = df.token_id.apply(lambda x: (str(x)[:5]))
|
||||
df['tmp'] = df.token_id.apply(lambda x: x[:10] )
|
||||
# tokens['tmp'] = tokens.token_id.apply(lambda x: x[:10] )
|
||||
# len(tokens)
|
||||
# len(tokens[['collection','token_id']].drop_duplicates())
|
||||
# len(tokens[['collection','tmp']].drop_duplicates())
|
||||
# df.to_csv('~/Downloads/tmp2.csv', index=False)
|
||||
if 'clean_token_id' in df.columns:
|
||||
del df['clean_token_id']
|
||||
|
||||
a = set(tokens[tokens.collection=='Galactic Punks'].token_id.unique())
|
||||
b = set(df[df.collection=='Galactic Punks'].token_id.unique())
|
||||
len(a.intersection(b))
|
||||
[ x for x in a if x in b ]
|
||||
len([ x for x in a if x in b ])
|
||||
# df[(df.collection=='Galactic Punks')].token_id.values[0]
|
||||
df = df.merge(tokens[['collection','tmp','clean_token_id']], how='left', on=['collection','tmp'])
|
||||
df[df.collection == 'Galactic Punks'].sort_values('clean_token_id')
|
||||
# print(df[ (df.clean_token_id.isnull()) & ( df.collection == 'Galactic Punks')])
|
||||
# print(len(df[ (df.clean_token_id.isnull()) & ( df.chain == 'Terra')]))
|
||||
# print(len(df[ (df.clean_token_id.isnull())]))
|
||||
# print(df[ (df.clean_token_id.isnull())].groupby('collection').token_id.count() )
|
||||
print(df[ (df.clean_token_id.notnull())].groupby('collection').token_id.count() )
|
||||
# print(len(df[ (df.clean_token_id.notnull()) & ( df.collection == 'Galactic Punks')]))
|
||||
# min(df[df.collection == 'Galactic Punks'].token_id.values)
|
||||
# min(tokens[tokens.collection == 'Galactic Punks'].token_id.values)
|
||||
df['clean_token_id'] = df.clean_token_id.fillna(df.token_id).astype(float).astype(int).astype(str)
|
||||
# print(df[ (df.token_id.isnull()) & ( df.collection == 'Galactic Punks')])
|
||||
df[df.clean_token_id.isnull()].groupby('collection').token_id.count()
|
||||
df[df.clean_token_id.notnull()].groupby('collection').token_id.count()
|
||||
df['token_id'] = df.clean_token_id
|
||||
del df['clean_token_id']
|
||||
df[df.collection == 'Galactic Punks']
|
||||
print(df.groupby('collection').token_id.count() )
|
||||
df.to_csv('./data/{}.csv'.format(c), index=False)
|
||||
# tokens[tokens.collection=='Galactic Punks']
|
||||
# len(tokens[tokens.collection=='Galactic Punks'])
|
||||
# tokens[(tokens.collection=='Galactic Punks') & (tokens.token_id=='25984997114855597728010029317878710272')]
|
||||
# 25984997114855639851202718743284654443
|
||||
# 25984997114855597728010029317878710272
|
||||
|
||||
# a = set(tokens[tokens.collection=='Galactic Punks'].token_id.unique())
|
||||
# b = set(df[df.collection=='Galactic Punks'].token_id.unique())
|
||||
# len(a.intersection(b))
|
||||
# [ x for x in a if x in b ]
|
||||
# len([ x for x in a if x in b ])
|
||||
# df[(df.collection=='Galactic Punks')].token_id.values[0]
|
||||
df = df.merge(tokens[['collection','tmp','clean_token_id']], how='left', on=['collection','tmp'])
|
||||
# df[df.collection == 'Galactic Punks'].sort_values('clean_token_id')
|
||||
# print(df[ (df.clean_token_id.isnull()) & ( df.collection == 'Galactic Punks')])
|
||||
# print(len(df[ (df.clean_token_id.isnull()) & ( df.chain == 'Terra')]))
|
||||
# print(len(df[ (df.clean_token_id.isnull())]))
|
||||
# print(df[ (df.clean_token_id.isnull())].groupby('collection').token_id.count() )
|
||||
# print(df[ (df.clean_token_id.notnull())].groupby('collection').token_id.count() )
|
||||
# print(len(df[ (df.clean_token_id.notnull()) & ( df.collection == 'Galactic Punks')]))
|
||||
# min(df[df.collection == 'Galactic Punks'].token_id.values)
|
||||
# min(tokens[tokens.collection == 'Galactic Punks'].token_id.values)
|
||||
df['clean_token_id'] = df.clean_token_id.fillna(df.token_id).astype(float).astype(int).astype(str)
|
||||
# print(df[ (df.token_id.isnull()) & ( df.collection == 'Galactic Punks')])
|
||||
df[df.clean_token_id.isnull()].groupby('collection').token_id.count()
|
||||
df[df.clean_token_id.notnull()].groupby('collection').token_id.count()
|
||||
df['token_id'] = df.clean_token_id
|
||||
del df['clean_token_id']
|
||||
df[df.collection == 'Galactic Punks']
|
||||
print(df.groupby('collection').token_id.count() )
|
||||
df.to_csv('./data/{}.csv'.format(c), index=False)
|
||||
|
||||
update_token_ids()
|
||||
# add_model_sales()
|
||||
|
||||
37
utils.py
Normal file
37
utils.py
Normal file
@ -0,0 +1,37 @@
|
||||
import pandas as pd
|
||||
|
||||
|
||||
clean_names = {
|
||||
'aurory': 'Aurory'
|
||||
,'thugbirdz': 'Thugbirdz'
|
||||
,'smb': 'Solana Monkey Business'
|
||||
,'degenapes': 'Degen Apes'
|
||||
,'peskypenguinclub': 'Pesky Penguins'
|
||||
,'meerkatmillionaires': 'Meerkat Millionaires'
|
||||
,'boryokudragonz': 'Boryoku Dragonz'
|
||||
,'degods': 'DeGods'
|
||||
}
|
||||
|
||||
def clean_name(name):
|
||||
if name.lower() in clean_names.keys():
|
||||
return(clean_names[name.lower()])
|
||||
name = re.sub('-', '', name.title())
|
||||
return(name)
|
||||
|
||||
|
||||
def merge(left, right, on=None, how='inner', ensure=True, verbose=True, message = ''):
|
||||
df = left.merge(right, on=on, how=how)
|
||||
if len(df) != len(left) and (ensure or verbose):
|
||||
if message:
|
||||
print(message)
|
||||
print('{} -> {}'.format(len(left), len(df)))
|
||||
cur = left.merge(right, on=on, how='left')
|
||||
cols = set(right.columns).difference(set(left.columns))
|
||||
print(cols)
|
||||
if ensure:
|
||||
col = list(cols)[0]
|
||||
missing = cur[cur[col].isnull()]
|
||||
print(missing.head())
|
||||
assert(False)
|
||||
return(df)
|
||||
|
||||
@ -408,7 +408,7 @@ server <- function(input, output, session) {
|
||||
selectInput(
|
||||
inputId = 'collectionname'
|
||||
, label = NULL
|
||||
, selected = 'Solana Monkey Business'
|
||||
, selected = 'DeGods'
|
||||
, choices = choices
|
||||
, width = "100%"
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user