nft-deal-score/format_data.py

from curses import meta
import re
import os
import math
import json
from typing import Collection
from nbformat import write
import pandas as pd
import snowflake.connector

os.chdir('/Users/kellenblumberg/git/nft-deal-score')
from utils import merge
from scrape_sol_nfts import clean_name


#########################
#     Connect to DB     #
#########################
with open('snowflake.pwd', 'r') as f:
    pwd = f.readlines()[0].strip()
with open('snowflake.usr', 'r') as f:
    usr = f.readlines()[0].strip()

ctx = snowflake.connector.connect(
    user=usr,
    password=pwd,
    account='vna27887.us-east-1'
)

###################################
#     Define Helper Functions     #
###################################
def clean_colnames(df):
	names = [ x.lower() for x in df.columns ]
	df.columns = names
	return(df)

def solana_2():
	mints = pd.read_csv('./data/solana_mints_3.csv')

	collection = mints.collection.values[0]
	collection_dir = re.sub(' ', '', collection)
	mints['image_url'] = 'https://wblrifk3oz3qpmqbxsunfvtmzi3c6vafn3un4f57ysesotlale.arweave.net/sFcUFVt2dweyAbyo0tZsyj-YvVAVu6N4Xv8SJJ01gWQ?ext=gif'
	mints['token_metadata_uri'] = 'https://arweave.net/8pJZCyLAY9TTAyuOB__2P_OwsmLzQEeKhLo5Ojiwflk'
	results = []
	token_id = 0
	for row in mints.iterrows():
		row = row[1]
		token_metadata = {}
		mint_address = row['mint_address']

		d = {
			'commission_rate': None
			, 'mint_address': mint_address
			, 'token_id': token_id
			, 'contract_address': mint_address
			, 'contract_name': row['collection']
			, 'created_at_block_id': 0
			, 'created_at_timestamp': str('2021-01-01')
			, 'created_at_tx_id': ''
			, 'creator_address': mint_address
			, 'creator_name': row['collection']
			, 'image_url': row['image_url']
			, 'project_name': row['collection']
			, 'token_id': int(token_id)
			, 'token_metadata': token_metadata
			, 'token_metadata_uri': row['token_metadata_uri']
			, 'token_name': row['collection']
		}
		results.append(d)
		token_id += 1
	print('Uploading {} results'.format(len(results)))

	dir = './data/metadata/{}/'.format(collection)
	if not os.path.exists(dir):
		os.makedirs(dir)

	n = 50
	r = math.ceil(len(results) / n)
	for i in range(r):
		newd = {
			"model": {
				"blockchain": "solana",
				"sinks": [
					{
						"destination": "{database_name}.silver.nft_metadata",
						"type": "snowflake",
						"unique_key": "blockchain || contract_address || token_id"
					}
				],
			},
			"results": results[(i * n):((i * n)+r)]
		}
		with open('./data/metadata/{}/{}.txt'.format(collection, i), 'w') as outfile:
			outfile.write(json.dumps(newd))

def levana():
	query = '''
		WITH legendary_traits AS (
			SELECT block_timestamp,
			block_id,
			tx_id,
			msg_value:execute_msg:mint:extension:name::string as name,
			msg_value:execute_msg:mint:extension:image::string as image,
			msg_value:execute_msg:mint:token_id::float as tokenid,
			msg_value:execute_msg:mint:extension:attributes[0]:value::string as rarity,
			msg_value:execute_msg:mint:extension:attributes[1]:value::string as rank,
			msg_value:execute_msg:mint:extension:attributes[2]:value::string as origin,
			msg_value:execute_msg:mint:extension:attributes[3]:value::string as cracking_date,
			msg_value:execute_msg:mint:extension:attributes[4]:value::string as essence,
			msg_value:execute_msg:mint:extension:attributes[5]:value::string as rune,
			msg_value:execute_msg:mint:extension:attributes[6]:value::string as infusion,
			msg_value:execute_msg:mint:extension:attributes[7]:value::string as affecting_moon,
			msg_value:execute_msg:mint:extension:attributes[8]:value::string as lucky_number,
			msg_value:execute_msg:mint:extension:attributes[9]:value::string as constellation,
			msg_value:execute_msg:mint:extension:attributes[10]:value::string as temperature,
			msg_value:execute_msg:mint:extension:attributes[11]:value::string as weight,
			msg_value:execute_msg:mint:extension:attributes[12]:value::string as family,
			msg_value:execute_msg:mint:extension:attributes[13]:value::string as genus,
			msg_value:execute_msg:mint:extension:attributes[18]:value::string as shower,
			msg_value:execute_msg:mint:extension:attributes[19]:value::string as meteor_id,
			msg_value:execute_msg:mint:extension:attributes[14]:value::string as legendary_composition,
			msg_value:execute_msg:mint:extension:attributes[15]:value::string as ancient_composition,
			msg_value:execute_msg:mint:extension:attributes[16]:value::string as rare_composition,
			msg_value:execute_msg:mint:extension:attributes[17]:value::string as common_composition
			from terra.msgs 
			where msg_value:contract::string = 'terra1k0y373yxqne22pc9g7jvnr4qclpsxtafevtrpg' 
			and	msg_value:sender::string = 'terra1awy9ychm2z2hd696kz6yeq67l30l7nxs7n762t'
			and	msg_value:execute_msg:mint is not null
			and	tx_status = 'SUCCEEDED'
			and	rarity = 'Legendary'
		),
		ancient_traits as (
			SELECT block_timestamp,
			block_id,
			tx_id,
			msg_value:execute_msg:mint:extension:name::string as name,
			msg_value:execute_msg:mint:extension:image::string as image,
			msg_value:execute_msg:mint:token_id::float as tokenid,
			msg_value:execute_msg:mint:extension:attributes[0]:value::string as rarity,
			msg_value:execute_msg:mint:extension:attributes[1]:value::string as rank,
			msg_value:execute_msg:mint:extension:attributes[2]:value::string as origin,
			msg_value:execute_msg:mint:extension:attributes[3]:value::string as cracking_date,
			msg_value:execute_msg:mint:extension:attributes[4]:value::string as essence,
			msg_value:execute_msg:mint:extension:attributes[5]:value::string as rune,
			msg_value:execute_msg:mint:extension:attributes[6]:value::string as infusion,
			msg_value:execute_msg:mint:extension:attributes[7]:value::string as affecting_moon,
			msg_value:execute_msg:mint:extension:attributes[8]:value::string as lucky_number,
			msg_value:execute_msg:mint:extension:attributes[9]:value::string as constellation,
			msg_value:execute_msg:mint:extension:attributes[10]:value::string as temperature,
			msg_value:execute_msg:mint:extension:attributes[11]:value::string as weight,
			msg_value:execute_msg:mint:extension:attributes[12]:value::string as family,
			msg_value:execute_msg:mint:extension:attributes[13]:value::string as genus,
			msg_value:execute_msg:mint:extension:attributes[17]:value::string as shower,
			msg_value:execute_msg:mint:extension:attributes[18]:value::string as meteor_id,
			null																 as legendary_composition,
			msg_value:execute_msg:mint:extension:attributes[14]:value::string as ancient_composition,
			msg_value:execute_msg:mint:extension:attributes[15]:value::string as rare_composition,
			msg_value:execute_msg:mint:extension:attributes[16]:value::string as common_composition
			from terra.msgs 
			where msg_value:contract::string = 'terra1k0y373yxqne22pc9g7jvnr4qclpsxtafevtrpg' 
			and	msg_value:sender::string = 'terra1awy9ychm2z2hd696kz6yeq67l30l7nxs7n762t'
			and	msg_value:execute_msg:mint is not null
			and	tx_status = 'SUCCEEDED'
			and	rarity = 'Ancient'
		),
		rare_traits as (
			SELECT block_timestamp,
			block_id,
			tx_id,
			msg_value:execute_msg:mint:extension:name::string as name,
			msg_value:execute_msg:mint:extension:image::string as image,
			msg_value:execute_msg:mint:token_id::float as tokenid,
			msg_value:execute_msg:mint:extension:attributes[0]:value::string as rarity,
			msg_value:execute_msg:mint:extension:attributes[1]:value::string as rank,
			msg_value:execute_msg:mint:extension:attributes[2]:value::string as origin,
			msg_value:execute_msg:mint:extension:attributes[3]:value::string as cracking_date,
			msg_value:execute_msg:mint:extension:attributes[4]:value::string as essence,
			msg_value:execute_msg:mint:extension:attributes[5]:value::string as rune,
			msg_value:execute_msg:mint:extension:attributes[6]:value::string as infusion,
			msg_value:execute_msg:mint:extension:attributes[7]:value::string as affecting_moon,
			msg_value:execute_msg:mint:extension:attributes[8]:value::string as lucky_number,
			msg_value:execute_msg:mint:extension:attributes[9]:value::string as constellation,
			msg_value:execute_msg:mint:extension:attributes[10]:value::string as temperature,
			msg_value:execute_msg:mint:extension:attributes[11]:value::string as weight,
			msg_value:execute_msg:mint:extension:attributes[12]:value::string as family,
			msg_value:execute_msg:mint:extension:attributes[13]:value::string as genus,
			msg_value:execute_msg:mint:extension:attributes[16]:value::string as shower,
			msg_value:execute_msg:mint:extension:attributes[17]:value::string as meteor_id,
			null																 as legendary_composition,
			null																 as ancient_composition,
			msg_value:execute_msg:mint:extension:attributes[14]:value::string as rare_composition,
			msg_value:execute_msg:mint:extension:attributes[15]:value::string as common_composition
			from terra.msgs 
			where msg_value:contract::string = 'terra1k0y373yxqne22pc9g7jvnr4qclpsxtafevtrpg' 
			and	msg_value:sender::string = 'terra1awy9ychm2z2hd696kz6yeq67l30l7nxs7n762t'
			and	msg_value:execute_msg:mint is not null
			and	tx_status = 'SUCCEEDED'
			and	rarity = 'Rare'
		),

		common_traits as (
			SELECT block_timestamp,
			block_id,
			tx_id,
			msg_value:execute_msg:mint:extension:name::string as name,
			msg_value:execute_msg:mint:extension:image::string as image,
			msg_value:execute_msg:mint:token_id::float as tokenid,
			msg_value:execute_msg:mint:extension:attributes[0]:value::string as rarity,
			msg_value:execute_msg:mint:extension:attributes[1]:value::string as rank,
			msg_value:execute_msg:mint:extension:attributes[2]:value::string as origin,
			msg_value:execute_msg:mint:extension:attributes[3]:value::string as cracking_date,
			msg_value:execute_msg:mint:extension:attributes[4]:value::string as essence,
			msg_value:execute_msg:mint:extension:attributes[5]:value::string as rune,
			msg_value:execute_msg:mint:extension:attributes[6]:value::string as infusion,
			msg_value:execute_msg:mint:extension:attributes[7]:value::string as affecting_moon,
			msg_value:execute_msg:mint:extension:attributes[8]:value::string as lucky_number,
			msg_value:execute_msg:mint:extension:attributes[9]:value::string as constellation,
			msg_value:execute_msg:mint:extension:attributes[10]:value::string as temperature,
			msg_value:execute_msg:mint:extension:attributes[11]:value::string as weight,
			msg_value:execute_msg:mint:extension:attributes[12]:value::string as family,
			msg_value:execute_msg:mint:extension:attributes[13]:value::string as genus,
			msg_value:execute_msg:mint:extension:attributes[15]:value::string as shower,
			msg_value:execute_msg:mint:extension:attributes[16]:value::string as meteor_id,
			null																 as legendary_composition,
			null																 as ancient_composition,
			null																 as rare_composition,
			msg_value:execute_msg:mint:extension:attributes[14]:value::string as common_composition
			from terra.msgs 
			where msg_value:contract::string = 'terra1k0y373yxqne22pc9g7jvnr4qclpsxtafevtrpg' 
			and	msg_value:sender::string = 'terra1awy9ychm2z2hd696kz6yeq67l30l7nxs7n762t'
			and	msg_value:execute_msg:mint is not null
			and	tx_status = 'SUCCEEDED'
			and	rarity = 'Common'
		),

		combine AS
		(
			SELECT * from legendary_traits
			UNION ALL
			SELECT * from ancient_traits
			UNION ALL
			SELECT * from rare_traits
			UNION ALL
			SELECT * from common_traits
		)

		SELECT * from combine order by tokenid
	'''
	metadata = ctx.cursor().execute(query)
	metadata = pd.DataFrame.from_records(iter(metadata), columns=[x[0] for x in metadata.description])
	metadata = clean_colnames(metadata)
	metadata.head()
	metadata[ metadata.tokenid == '0027' ]
	metadata[ metadata.tokenid == 27 ]
	metadata.columns
	results = []
	for row in metadata.iterrows():
		row = row[1]
		token_metadata = {}
		for a in list(metadata.columns):
			if not a in [ 'block_timestamp','block_id','tx_id','name','tokenid','image' ]:
				token_metadata[a] = row[a]
		d = {
			'commission_rate': None
			, 'contract_address': 'terra1k0y373yxqne22pc9g7jvnr4qclpsxtafevtrpg'
			, 'contract_name': 'levana_dragon_eggs'
			, 'created_at_block_id': 0
			, 'created_at_timestamp': str(row['block_timestamp'])
			, 'created_at_tx_id': row['tx_id']
			, 'creator_address': 'terra1k0y373yxqne22pc9g7jvnr4qclpsxtafevtrpg'
			, 'creator_name': 'levana_dragon_eggs'
			, 'image_url': row['image']
			, 'project_name': 'levana_dragon_eggs'
			, 'token_id': row['tokenid']
			, 'token_metadata': token_metadata
			, 'token_metadata_uri': row['image']
			, 'token_name': row['name']
		}
		results.append(d)
	print('Uploading {} results'.format(len(results)))

	n = 50
	r = math.ceil(len(results) / n)
	for i in range(r):
		newd = {
			"model": {
				"blockchain": "terra",
				"sinks": [
					{
						"destination": "{database_name}.silver.nft_metadata",
						"type": "snowflake",
						"unique_key": "blockchain || contract_address || token_id"
					}
				],
			},
			"results": results[(i * n):((i * n)+r)]
		}
		with open('./data/metadata/Levana Dragon Eggs/{}.txt'.format(i), 'w') as outfile:
			outfile.write(json.dumps(newd))


def mint_to_token_id():
	data = []
	collection = 'DeGods'
	dir = './data/mints/DeGods/'
	for fname in os.listdir(dir):
		mint_address = fname[:-5]
		path = dir+fname
		with open(path) as f:
			s = f.read()
			d = json.loads(s)
			try:
				token_id = int(re.split('#', d['name'])[1])
				data += [[ collection, token_id, mint_address ]]
			except:
				pass
	data += [[ collection, 2508, 'CcWedWffikLoj3aYtgSK46LqF8LkLCgpu4RCo8RMxF3e' ]]
	df = pd.DataFrame(data, columns=['collection','token_id','mint_address']).drop_duplicates(subset=['token_id'], keep='last')
	[ x for x in range(1, 10001) if not x in df.token_id.unique() ]
	g = df.groupby('token_id').collection.count().reset_index()
	g[g.collection > 1]
	df.to_csv('./data/mint_to_token_id_map.csv', index=False)

def solana():
	mints = pd.read_csv('./data/solana_rarities.csv')
	sorted(mints.collection.unique())
	mints[mints.collection == 'smb'][['mint_address']].to_csv('~/Downloads/tmp.csv', index=False)
	collection_info = pd.read_csv('./data/collection_info.csv')
	metadata = pd.read_csv('./data/metadata.csv')
	metadata.collection.unique()
	tokens = pd.read_csv('./data/tokens.csv')
	tokens['token_id'] = tokens.token_id.astype(str)
	metadata['token_id'] = metadata.token_id.astype(str)
	metadata = metadata.merge(tokens)
	# metadata = metadata.merge(collection_info)
	metadata['token_id'] = metadata.clean_token_id.fillna(metadata.token_id)
	metadata = metadata[-metadata.feature_name.isin(['nft_rank','adj_nft_rank_0','adj_nft_rank_1','adj_nft_rank_2'])]

	metadata['token_id'] = metadata.token_id.astype(int)
	mints['token_id'] = mints.token_id.astype(int)
	mints['collection'] = mints.collection.apply(lambda x: clean_name(x) )

	metadata = pd.read_csv('./data/sf_metadata.csv')
	print(sorted(metadata.collection.unique()))
	# metadata = metadata[-metadata.collection.isin([''])]
	metadata['image_url'] = 'None'
	metadata = metadata.drop_duplicates()

	a = metadata.groupby('collection').token_id.count().reset_index()
	b = metadata[metadata.feature_value.isnull()].groupby('collection').token_id.count().reset_index()
	c = a.merge(b, on=['collection'])
	c['pct'] = c.token_id_y / c.token_id_x
	c.sort_values('pct')
	metadata[ (metadata.feature_value.isnull()) & (metadata.collection == 'mindfolk')]

	metadata = pd.read_csv('./data/solscan_metadata.csv')
	assert(len(metadata[metadata.feature_value.isnull()]) == 0)
	metadata['collection'] = metadata.collection.apply(lambda x: re.sub('-', ' ', x).title() )
	metadata['collection'] = metadata.collection.apply(lambda x: re.sub(' Cc', ' CC', x) )
	metadata['collection'] = metadata.collection.apply(lambda x: re.sub('Og ', 'OG ', x) )

	collection = 'Cets On Creck'
	collection = 'Astrals'



	query = '''
		SELECT DISTINCT project_name
		FROM solana.dim_nft_metadata
	'''
	seen = ctx.cursor().execute(query)
	seen = pd.DataFrame.from_records(iter(seen), columns=[x[0] for x in seen.description])
	seen = clean_colnames(seen)
	seen = list(seen.project_name.values)
	seen = [ x.lower() for x in seen ]

	metadata = pd.read_csv('./data/metadata.csv')
	len(metadata)
	# print(sorted(metadata.collection.unique()))
	# metadata = metadata[metadata.collection == collection]
	# print(sorted(metadata.collection.unique()))
	metadata = metadata[-(metadata.feature_name.isin(['adj_nft_rank_0','adj_nft_rank_1','adj_nft_rank_2','nft_rank']))]
	metadata[['collection']].drop_duplicates().to_csv('~/Downloads/tmp.csv', index=False)
	len(metadata.token_id.unique())
	# id_map = pd.read_csv('./data/mint_to_token_id_map.csv')
	id_map = pd.read_csv('./data/tokens.csv')
	tokens = pd.read_csv('./data/tokens.csv')
	tokens.collection.unique()
	len(tokens.collection.unique())
	cs = [ x for x in id_map.collection.unique() if not x.lower() in seen ]
	len(id_map.collection.unique())
	len(cs)
	id_map = id_map[id_map.collection.isin(cs)]
	metadata = metadata[metadata.collection.isin(cs)]

	# cs = metadata[metadata.chain.fillna('Solana') == 'Solana'].collection.unique()
	cs = metadata.collection.unique()
	id_map = id_map[id_map.collection.isin(cs)]
	metadata = metadata[metadata.collection.isin(cs)]
	sorted(id_map.collection.unique())
	sorted(metadata.collection.unique())

	# id_map['token_id'] = id_map.token_id.astype(int)
	# metadata['token_id'] = metadata.token_id.astype(int)
	id_map['token_id'] = id_map.token_id.astype(str)
	metadata['token_id'] = metadata.token_id.astype(str)

	metadata = merge(metadata, id_map[['collection','token_id','mint_address','image_url']], ensure = False)
	metadata = metadata[metadata.collection.isin(cs)]

	metadata['feature_name'] = metadata.feature_name.apply(lambda x: x.title() )
	# metadata['image_url'] = metadata.token_id.apply(lambda x: 'https://metadata.degods.com/g/{}.png'.format(x - 1) )
	metadata.head()
	metadata = metadata[-(metadata.feature_name.isin(['Nft_Rank','Adj_Nft_Rank_0','Adj_Nft_Rank_1','Adj_Nft_Rank_2']))]
	# print(metadata.groupby('feature_name').token_id.count().reset_index().sort_values('token_id', ascending=0).head(10))

	metadata = metadata[metadata.feature_name != 'L3G3Nd4Ry']

	# print(sorted(metadata.collection.unique()))
	# sorted(metadata[metadata.collection == collection].feature_name.unique())
	# sorted(metadata.feature_name.unique())

	# metadata[['collection']].drop_duplicates().to_csv('~/Downloads/tmp.csv', index=False)
	# Python code to convert into dictionary
	def Convert(tup, di):
		di = dict(tup)
		return di

	metadata = metadata[-metadata.collection.isin(['LunaBulls', 'Levana Dragon Eggs'])]
	metadata['token_id'] = metadata.token_id.astype(float)
	metadata['token_id'] = metadata.token_id.astype(int)
	metadata.groupby(['collection','feature_name']).token_id.count()
	metadata.head()
	metadata[metadata.mint_address.isnull()].collection.unique()
	assert(len(metadata[metadata.mint_address.isnull()]) == 0)
	dirs = sorted(list(set(os.listdir('./data/metadata/')).intersection(set(metadata.collection.unique()))))
	sorted(list(metadata.collection.unique()))
	# collection = 'Bubblegoose Ballers'
	it = 0
	tot = len(metadata.collection.unique())
	data = []
	for collection in metadata.collection.unique()[:1]:
		print('#{} / {}: {}'.format(it, tot, collection))
		mdf = metadata[metadata.collection == collection]
		df.groupby('Column1')[['Column2', 'Column3']].apply(lambda g: g.values.tolist()).to_dict()
		mdf.head(20).groupby(['collection','image_url','token_id'])[[ 'feature_name','feature_value' ]].apply(lambda g: g.values.tolist()).to_dict()

		mdf.head(20).groupby(['collection','image_url','token_id'])[[ 'feature_name','feature_value' ]].apply(lambda g: list(map(tuple, g.values.tolist())) ).to_dict()

		mdf.head(20).groupby(['collection','image_url','token_id'])[[ 'feature_name','feature_value' ]].apply(lambda g: Convert(list(map(tuple, g.values.tolist())), {}) ).to_dict()
		a = mdf.head(20).groupby(['collection','mint_address','token_id','image_url'])[[ 'feature_name','feature_value' ]].apply(lambda g: Convert(list(map(tuple, g.values.tolist())), {}) ).reset_index()


		a = metadata.groupby(['collection','mint_address','token_id','image_url'])[[ 'feature_name','feature_value' ]].apply(lambda g: Convert(list(map(tuple, g.values.tolist())), {}) ).reset_index()
		a.columns = ['collection','mint_address','token_id','image_url', 'token_metadata']
		a['commission_rate'] = None
		a['contract_address'] = a.mint_address
		a['contract_name'] = a.collection
		a['created_at_block_id'] = 0
		a['created_at_timestamp'] = '2021-01-01'
		a['created_at_tx_id'] = ''
		a['creator_address'] = a.mint_address
		a['creator_name'] = a.collection
		a['project_name'] = a.collection
		a['token_metadata_uri'] = a.image_url
		a['token_name'] = a.collection
		a.to_csv('./data/metadata/results.csv', index=False)
		a['n'] = range(len(a))
		a['n'] = a.n.apply(lambda x: int(x/50) )
		a['token_id'] = a.token_id.astype(int)
		cols = ['collection', 'mint_address', 'token_id', 'image_url', 'token_metadata',
       'commission_rate', 'contract_address', 'contract_name',
       'created_at_block_id', 'created_at_timestamp', 'created_at_tx_id',
       'creator_address', 'creator_name', 'project_name', 'token_metadata_uri',
       'token_name']

		n = 100000
		tot = int(len(a) / n) + 1
		for i in range(0, len(a), n):
			ind = int(i/n)
			print('#{} / {}'.format(ind, tot))
			g = a.head(i+n).tail(n).to_dict('records')
			txt  = [
				{
					"model": {
						"blockchain": "solana",
						"sinks": [
							{
								"destination": "{database_name}.silver.nft_metadata",
								"type": "snowflake",
								"unique_key": "blockchain || contract_address || token_id"
							}
						],
					},
					"results": g[x:x+50]
				}
				for x in range(0, len(g), 50)
			]
			w = pd.DataFrame({'ind': range(len(txt)), 'results':[json.dumps(x) for x in txt] })
			# w['results'] = w.results.apply(lambda x: x[1:-1] )
			w.to_csv('./data/metadata/results/{}.csv'.format(ind), index=False)
			# with open('./data/metadata/results/{}.json'.format(i), 'w') as outfile:
			# 	json.dump(results[i:i+100000], outfile)

		g = a.head(200).groupby('n')[cols].apply(lambda g: Convert(list(map(tuple, g.values.tolist())), {}) ).to_dict()
		g = a.head(200).groupby('n')[cols].apply(lambda g: (list(map(tuple, g.values.tolist())), {}) )
		g = a.head(200).groupby('n')[cols].apply(lambda g: g.values.tolist()).reset_index()
		g = a.head(200).to_dict('records')
		sorted(a.collection.unique())
		g = a[a.collection == 'Jungle Cats'].head(20000).to_dict('records')
		txt  = [
			{
				"model": {
					"blockchain": "solana",
					"sinks": [
						{
							"destination": "{database_name}.silver.nft_metadata",
							"type": "snowflake",
							"unique_key": "blockchain || contract_address || token_id"
						}
					],
				},
				"results": g[i:i+50]
			}
			for i in range(0, len(g), 50)
		]
		w = pd.DataFrame({'ind': range(len(txt)), 'results':[json.dumps(x) for x in txt] })
		# w['results'] = w.results.apply(lambda x: x[1:-1] )
		w.to_csv('./data/metadata/results.csv', index=False)
		with open('./data/metadata/results.txt', 'w') as outfile:
			outfile.write(json.dumps(txt))
		g = list(a.head(200).values)
		results = a.to_dict('records')
		for i in range(0, len(results), 100000):
			print(i)
			with open('./data/metadata/results/{}.json'.format(i), 'w') as outfile:
				json.dump(results[i:i+100000], outfile)

		n = 50
		r = math.ceil(len(results) / n)
		for i in range(r):
			print('#{} / {}'.format(i, r))
			newd = {
				"model": {
					"blockchain": "solana",
					"sinks": [
						{
							"destination": "{database_name}.silver.nft_metadata",
							"type": "snowflake",
							"unique_key": "blockchain || contract_address || token_id"
						}
					],
				},
				"results": results[(i * n):((i * n)+r)]
			}
			data += [ json.dumps(newd) ]
			with open('./data/metadata/results/{}.txt'.format(collection, i), 'w') as outfile:
				outfile.write(json.dumps(newd))


		# results = []
		# for token_id in sorted(mdf.token_id.unique()):
		# 	if token_id % 1000 == 1:
		# 		print(token_id, len(results))
		# 	cur = mdf[mdf.token_id == token_id]
		# 	token_metadata = {}
		# 	# m = mints[(mints.collection == collection) & (mints.token_id == token_id) ]
		# 	m = metadata[(metadata.collection == collection) & (metadata.token_id == token_id) ]
		# 	m = m.fillna('None')
		# 	if not len(m):
		# 		print(token_id)
		# 		continue
		# 	# mint_address = m.mint_address.values[0] if 'mint_address' in m.columns else ''
		# 	mint_address = m.mint_address.values[0]
		# 	for row in cur.iterrows():
		# 		row = row[1]
		# 		token_metadata[row['feature_name']] = row['feature_value']

		# 	d = {
		# 		'commission_rate': None
		# 		, 'mint_address': mint_address
		# 		, 'token_id': token_id
		# 		, 'contract_address': mint_address
		# 		, 'contract_name': row['collection']
		# 		, 'created_at_block_id': 0
		# 		, 'created_at_timestamp': str('2021-01-01')
		# 		, 'created_at_tx_id': ''
		# 		, 'creator_address': mint_address
		# 		, 'creator_name': row['collection']
		# 		, 'image_url': row['image_url']
		# 		, 'project_name': row['collection']
		# 		, 'token_id': int(token_id)
		# 		, 'token_metadata': token_metadata
		# 		, 'token_metadata_uri': row['image_url']
		# 		, 'token_name': row['collection']
		# 	}
		# 	results.append(d)
		print('Uploading {} results'.format(len(results)))

		dir = './data/metadata/{}/'.format(collection)
		if not os.path.exists(dir):
			os.makedirs(dir)

		n = 50
		r = math.ceil(len(results) / n)
		for i in range(r):
			newd = {
				"model": {
					"blockchain": "solana",
					"sinks": [
						{
							"destination": "{database_name}.silver.nft_metadata",
							"type": "snowflake",
							"unique_key": "blockchain || contract_address || token_id"
						}
					],
				},
				"results": results[(i * n):((i * n)+r)]
			}
			data += [ json.dumps(newd) ]
			with open('./data/metadata/{}/{}.txt'.format(collection, i), 'w') as outfile:
				outfile.write(json.dumps(newd))

def bayc():
	with open('./data/bayc.json') as f:
		j = json.load(f)
	results = []
	for row in j:
		token_metadata = {}
		for a in row['metadata']['attributes']:
			token_metadata[a['trait_type']] = a['value']
		d = {
			'commission_rate': None
			, 'contract_address': '0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d'
			, 'contract_name': 'bayc'
			, 'created_at_block_id': row['blockNumber']
			, 'created_at_timestamp': '2021-04-30 14:21:08.000'
			, 'created_at_tx_id': row['transactionHash']
			, 'creator_address': '0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d'
			, 'creator_name': 'bayc'
			, 'image_url': row['metadata']['image']
			, 'project_name': 'bored_ape_yacht_club'
			, 'token_id': row['id']
			, 'token_metadata': token_metadata
			, 'token_metadata_uri': row['uri']
			, 'token_name': 'BAYC #{}'.format(row['id'])
		}
		results.append(d)
	df = pd.DataFrame(results)
	g = df.groupby('token_id').contract_address.count().reset_index()
	g = g.sort_values('contract_address', ascending=0)
	# df = pd.DataFrame([[str(newd['model']), str(results[:2])]], columns=['model','results'])
	# df.to_csv('./data/bayc.csv', index=False)

	# s = str(json.dumps(newd))
	# str(newd)

	# s = str(json.loads(json.dumps(newd)))
	# s = re.sub('\\', '', s)


	# json.dumps(json.JSONDecoder().decode(newd))

	n = 50
	r = math.ceil(len(results) / n)
	for i in range(r):
		newd = {
			"model": {
				"blockchain": "ethereum",
				"sinks": [
					{
						"destination": "{database_name}.silver.nft_metadata",
						"type": "snowflake",
						"unique_key": "blockchain || contract_address || token_id"
					}
				],
			},
			"results": results[(i * n):((i * n)+r)]
		}
		with open('./data/metadata/bayc/{}.txt'.format(i), 'w') as outfile:
			outfile.write(json.dumps(newd))