artin/LazyLibrarian
1
0
Fork 0
mirror of https://gitlab.com/LazyLibrarian/LazyLibrarian.git synced 2026-02-06 10:47:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

genre table, goodreads and googlebooks genres

Browse Source
This commit is contained in:
Phil Borman 2018-11-23 18:30:18 +01:00
parent 492f7e7eef
commit 01fbfb8cbe
10 changed files with 338 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
example.gr_genres.json Normal file
View File

@ -0,0 +1,39 @@
{
"genreLimit": 4,
"genreUsers": 10,
"genreExclude": [
"owned", "books-i-own", "owned-books", "my-books", "my-library",
"books", "audio", "ebook", "ebooks", "e-books", "e-book",
"kindle", "kindle-unlimited", "kindle-books", "nook-books", "nook", "kobo", "calibre",
"library", "default", "favorites", "favourites", "to-buy", "english", "british",
"general", "series", "audible", "nope", "hell-no", "not-interested",
"giveaways", "dnf", "ya", "wish-list", "s2e", "non", "1", "sub"
],
"genreExcludeParts": ["read-", "-read"],
"genreReplace": {"humor": "humour",
"science-fiction": "sci-fi",
"scifi": "sci-fi",
"sf": "sci-fi",
"math": "mathematics",
"nonfiction": "non-fiction",
"children": "childrens",
"children-s": "childrens",
"children-s-books": "childrens",
"graphic-novel": "graphic-novels",
"40k": "warhammer-40k",
"audiobook": "audiobooks",
"audio-book": "audiobooks",
"cookbook": "cookbooks",
"cook-book": "cookbooks",
"cook-books": "cookbooks",
"hist": "history",
"dc": "dc-comics",
"ww2": "world-war-2",
"world-war-ii": "world-war-2",
"alex-cross-series": "alex-cross",
"michael-bennett-series": "michael-bennett",
"rory-gilmore-reading-list": "rory-gilmore-challenge",
"gilmore-girls-reading-list": "rory-gilmore-challenge",
"rory-gilmore-reading-challenge": "rory-gilmore-challenge"
}
}

37
lazylibrarian/__init__.py
View File

@ -40,7 +40,7 @@ from lib.six.moves import configparser
# Transient globals NOT stored in config
# These are used/modified by LazyLibrarian.py before config.ini is read
FULL_PATH = None
PROG_DIR = None
PROG_DIR = ''
ARGS = None
DAEMON = False
SIGNAL = None
@ -102,6 +102,7 @@ GROUP_CONCAT = 0
FOREIGN_KEY = 0
HIST_REFRESH = 1000
GITLAB_TOKEN = 'gitlab+deploy-token-26212:Hbo3d8rfZmSx4hL1Fdms@gitlab.com'
GRGENRES = None
# extended loglevels
log_magdates = 1 << 2 # 4 magazine date matching
@ -617,7 +618,7 @@ def initialize():
UPDATE_MSG, CURRENT_TAB, CACHE_HIT, CACHE_MISS, LAST_LIBRARYTHING, LAST_GOODREADS, SHOW_SERIES, SHOW_MAGS, \
SHOW_AUDIO, CACHEDIR, BOOKSTRAP_THEMELIST, MONTHNAMES, CONFIG_DEFINITIONS, isbn_979_dict, isbn_978_dict, \
CONFIG_NONWEB, CONFIG_NONDEFAULT, CONFIG_GIT, MAG_UPDATE, AUDIO_UPDATE, EBOOK_UPDATE, \
GROUP_CONCAT, GR_SLEEP, LT_SLEEP, GB_CALLS, FOREIGN_KEY
GROUP_CONCAT, GR_SLEEP, LT_SLEEP, GB_CALLS, FOREIGN_KEY, GRGENRES
with INIT_LOCK:
@ -677,8 +678,8 @@ def initialize():
except OSError as e:
logger.error('Could not create cachedir; %s' % e)
for cache in ['book', 'author', 'SeriesCache', 'JSONCache', 'XMLCache', 'WorkCache', 'magazine']:
cachelocation = os.path.join(CACHEDIR, cache)
for item in ['book', 'author', 'SeriesCache', 'JSONCache', 'XMLCache', 'WorkCache', 'magazine']:
cachelocation = os.path.join(CACHEDIR, item)
try:
os.makedirs(cachelocation)
except OSError as e:
@ -687,9 +688,8 @@ def initialize():
# nest these caches 2 levels to make smaller directory lists
caches = ["XMLCache", "JSONCache", "WorkCache"]
for cache in caches:
pth = os.path.join(CACHEDIR, cache)
start = time.time()
for item in caches:
pth = os.path.join(CACHEDIR, item)
for i in '0123456789abcdef':
for j in '0123456789abcdef':
cachelocation = os.path.join(pth, i, j)
@ -698,9 +698,9 @@ def initialize():
except OSError as e:
if not os.path.isdir(cachelocation):
logger.error('Could not create %s: %s' % (cachelocation, e))
for item in os.listdir(pth):
if len(item) > 2:
os.rename(os.path.join(pth, item), os.path.join(pth, item[0], item[1], item))
for itm in os.listdir(pth):
if len(itm) > 2:
os.rename(os.path.join(pth, itm), os.path.join(pth, itm[0], itm[1], itm))
# keep track of last api calls so we don't call more than once per second
# to respect api terms, but don't wait un-necessarily either
@ -749,6 +749,9 @@ def initialize():
if 'missing' in item:
logger.warn(item)
GRGENRES = build_genres()
MONTHNAMES = build_monthtable()
try: # optional module, check database health, could also be upgraded to modify/repair db or run other code
# noinspection PyUnresolvedReferences
from .dbcheck import dbcheck
@ -756,7 +759,6 @@ def initialize():
except ImportError:
pass
MONTHNAMES = build_monthtable()
BOOKSTRAP_THEMELIST = build_bookstrap_themes(PROG_DIR)
__INITIALIZED__ = True
@ -1404,6 +1406,19 @@ def build_bookstrap_themes(prog_dir):
return themelist
def build_genres():
json_file = os.path.join(DATADIR, 'gr_genres.json')
if not os.path.isfile(json_file):
json_file = os.path.join(PROG_DIR, 'example.gr_genres.json')
if os.path.isfile(json_file):
try:
with open(json_file) as json_data:
return json.load(json_data)
except Exception as e:
logger.error('Failed to load gr_genres.json, %s %s' % (type(e).__name__, str(e)))
return None
def build_monthtable():
table = []
json_file = os.path.join(DATADIR, 'monthnames.json')

35
lazylibrarian/api.py
View File

@ -144,8 +144,8 @@ cmd_dict = {'help': 'list available commands. ' +
'getBookAuthors': '&id= Get list of authors associated with this book',
'cleanCache': '[&wait] Clean unused and expired files from the LazyLibrarian caches',
'deleteEmptySeries': 'Delete any book series that have no members',
'setNoDesc': '[&refresh] Set book descriptions for all books without one',
'setNoGenre': '[&refresh] Set book genre for all books without one',
'setNoDesc': '[&refresh] Set descriptions for all books, include "No Description" entries on refresh',
'setNoGenre': '[&refresh] Set book genre for all books without one, include "Unknown" entries on refresh',
'setWorkPages': '[&wait] Set the WorkPages links in the database',
'setAllBookSeries': '[&wait] Set the series details from goodreads or librarything workpages',
'setAllBookAuthors': '[&wait] Set all authors for all books from book workpages',
@ -331,7 +331,10 @@ class Api(object):
self.data = get_capabilities(prov, True)
def _help(self):
self.data = dict(cmd_dict)
res = ''
for key in sorted(cmd_dict):
res += "%s: %s<p>" % (key, cmd_dict[key])
self.data = res
def _getHistory(self):
self.data = self._dic_from_query(
@ -513,12 +516,12 @@ class Api(object):
self.data = self._dic_from_query(q)
def _listNoGenre(self):
q = 'SELECT BookID,BookName,AuthorName from books,authors where '
q = 'SELECT BookID,BookName,AuthorName from books,authors where books.Status != "Ignored" and '
q += '(BookGenre="Unknown" or BookGenre="" or BookGenre is NULL) and books.AuthorID = authors.AuthorID'
self.data = self._dic_from_query(q)
def _listNoDesc(self):
q = 'SELECT BookID,BookName,AuthorName from books,authors where '
q = 'SELECT BookID,BookName,AuthorName from books,authors where books.Status != "Ignored" and '
q += '(BookDesc="" or BookDesc is NULL) and books.AuthorID = authors.AuthorID'
self.data = self._dic_from_query(q)
@ -529,13 +532,16 @@ class Api(object):
else:
expire = False
extra = ''
q = 'SELECT BookID,BookName,AuthorName,BookISBN from books,authors where '
q = 'SELECT BookID,BookName,AuthorName,BookISBN from books,authors where books.Status != "Ignored" and '
q += '(BookDesc="" or BookDesc is NULL' + extra + ') and books.AuthorID = authors.AuthorID'
myDB = database.DBConnection()
res = myDB.select(q)
descs = 0
cnt = 0
logger.debug("Checking description for %s book%s" % (len(res), plural(len(res))))
data = ''
for item in res:
cnt += 1
isbn = item['BookISBN']
auth = item['AuthorName']
book = item['BookName']
@ -545,9 +551,12 @@ class Api(object):
logger.debug("Updated description for %s:%s" % (auth, book))
myDB.action('UPDATE books SET bookdesc=? WHERE bookid=?', (data['desc'], item['BookID']))
elif data is None:
self.data = "Error reading description, see debug log"
break
self.data = "Scanned %s book%s, found %s new description%s" % (len(res), plural(len(res)), descs, plural(descs))
msg = "Scanned %d book%s, found %d new description%s from %d" % \
(cnt, plural(cnt), descs, plural(descs), len(res))
if data is None:
msg += ': Access Blocked'
self.data = msg
logger.info(self.data)
def _setNoGenre(self, **kwargs):
@ -557,13 +566,16 @@ class Api(object):
else:
expire = False
extra = ''
q = 'SELECT BookID,BookName,AuthorName,BookISBN from books,authors where '
q = 'SELECT BookID,BookName,AuthorName,BookISBN from books,authors where books.Status != "Ignored" and '
q += '(BookGenre="" or BookGenre is NULL' + extra + ') and books.AuthorID = authors.AuthorID'
myDB = database.DBConnection()
res = myDB.select(q)
genre = 0
cnt = 0
logger.debug("Checking genre for %s book%s" % (len(res), plural(len(res))))
data = ''
for item in res:
cnt += 1
isbn = item['BookISBN']
auth = item['AuthorName']
book = item['BookName']
@ -575,7 +587,10 @@ class Api(object):
elif data is None:
self.data = "Error reading genre, see debug log"
break
self.data = "Scanned %s book%s, found %s new genre%s" % (len(res), plural(len(res)), genre, plural(genre))
msg = "Scanned %d book%s, found %d new genre%s from %d" % (cnt, plural(cnt), genre, plural(genre), len(res))
if data is None:
msg += ': Access Blocked'
self.data = msg
logger.info(self.data)
def _listNoISBN(self):

98
lazylibrarian/bookwork.py
View File

@ -1009,6 +1009,102 @@ def getWorkSeries(bookID=None):
return serieslist
def setGenres(genrelist=None, bookid=None):
""" set genre details in genres/genrebooks tables from the supplied list
and a displayable summary in book table """
myDB = database.DBConnection()
if bookid:
# delete any old genrebooks entries
myDB.action('DELETE from genrebooks WHERE BookID=?', (bookid,))
for item in genrelist:
match = myDB.match('SELECT GenreID from genres where GenreName=? COLLATE NOCASE', (item,))
if not match:
myDB.action('INSERT into genres (GenreName) VALUES (?)', (item,))
match = myDB.match('SELECT GenreID from genres where GenreName=?', (item,))
myDB.action('INSERT into genrebooks (GenreID, BookID) VALUES (?,?)',
(match['GenreID'], bookid), suppress='UNIQUE')
myDB.action('UPDATE books set BookGenre=? WHERE BookID=?', (', '.join(genrelist), bookid))
def get_gr_genres(bookid, refresh=False):
if lazylibrarian.GRGENRES:
genreLimit = lazylibrarian.GRGENRES['genreLimit']
genreUsers = lazylibrarian.GRGENRES['genreUsers']
genreExclude = lazylibrarian.GRGENRES['genreExclude']
genreExcludeParts = lazylibrarian.GRGENRES['genreExcludeParts']
genreReplace = lazylibrarian.GRGENRES['genreReplace']
else:
genreLimit = 3
genreUsers = 10
genreExclude = []
genreExcludeParts = []
genreReplace = {}
myDB = database.DBConnection()
URL = 'https://www.goodreads.com/book/show/' + bookid + '.xml?key=' + lazylibrarian.CONFIG['GR_API']
genres = []
try:
rootxml, in_cache = gr_xml_request(URL, useCache=not refresh)
except Exception as e:
logger.error("%s fetching book genres: %s" % (type(e).__name__, str(e)))
return genres, False
if rootxml is None:
logger.debug("Error requesting book genres")
return genres, in_cache
shelves = []
try:
shelves = rootxml.find('book/popular_shelves')
if shelves is None:
return genres, in_cache
except (KeyError, AttributeError):
logger.error("Error reading shelves for GoodReads bookid %s" % bookid)
for shelf in shelves:
# check shelf name is used by >= users
if check_int(shelf.attrib['count'], 0) >= genreUsers:
genres.append([int(shelf.attrib['count']), shelf.attrib['name']])
# return max (limit) genres sorted by most popular first
res = sorted(genres, key=lambda x: x[0], reverse=True)
res = [item[1] for item in res]
cnt = genreLimit
genres = []
for item in res:
if item not in genreExclude:
if item in genreReplace:
item = genreReplace[item]
reject = False
# reject to-read, not-read, read-2017 etc...
for entry in genreExcludeParts:
if entry in item:
reject = True
break
if not reject:
# try to reject author names, check both tom-holt and holt-tom
words = item.split('-')
if len(words) == 2:
res = myDB.match('SELECT authorid from authors WHERE authorname=? COLLATE NOCASE',
("%s %s" % (words[0], words[1]),))
if len(res):
reject = True
else:
res = myDB.match('SELECT authorid from authors WHERE authorname=? COLLATE NOCASE',
("%s %s" % (words[1], words[0]),))
if len(res):
reject = True
if not reject and item not in genres:
genres.append(item)
cnt -= 1
if not cnt:
break
logger.debug("GoodReads bookid %s %d from %d genre%s, cached=%s" %
(bookid, len(genres), len(res), plural(len(res)), in_cache))
return genres, in_cache
def get_book_pubdate(bookid, refresh=False):
URL = 'https://www.goodreads.com/book/show/' + bookid + '.xml?key=' + lazylibrarian.CONFIG['GR_API']
bookdate = "0000"
@ -1029,7 +1125,7 @@ def get_book_pubdate(bookid, refresh=False):
except (KeyError, AttributeError):
logger.error("Error reading pubdate for GoodReads bookid %s pubdate [%s]" % (bookid, bookdate))
logger.debug("GoodReads bookid %s pubdate [%s] %s" % (bookid, bookdate, in_cache))
logger.debug("GoodReads bookid %s pubdate [%s] cached=%s" % (bookid, bookdate, in_cache))
return bookdate, in_cache

23
lazylibrarian/common.py
View File

@ -1,3 +1,4 @@
# This file is part of Lazylibrarian.
#
# Lazylibrarian is free software':'you can redistribute it and/or modify
@ -727,11 +728,25 @@ def showStats():
book_stats.append([status, res['counter']])
res = myDB.match('SELECT count(*) as counter FROM books WHERE AudioStatus="%s"' % status)
audio_stats.append([status, res['counter']])
for column in ['BookGenre', 'BookDesc', 'BookISBN', 'BookLang']:
cmd = "SELECT count(*) as counter FROM books WHERE %s is null or %s = ''"
cmd += " or %s = 'Unknown' or %s = 'No Description'"
res = myDB.match(cmd % (column, column, column, column))
for column in ['BookGenre', 'BookDesc']:
cmd = "SELECT count(*) as counter FROM books WHERE Status != 'Ignored' and "
cmd += "(%s is null or %s = '')"
res = myDB.match(cmd % (column, column))
missing_stats.append([column.replace('Book', 'No'), res['counter']])
cmd = "SELECT count(*) as counter FROM books WHERE Status != 'Ignored' and BookGenre='Unknown'"
res = myDB.match(cmd)
missing_stats.append(['X_Genre', res['counter']])
cmd = "SELECT count(*) as counter FROM books WHERE Status != 'Ignored' and BookDesc='No Description'"
res = myDB.match(cmd)
missing_stats.append(['X_Desc', res['counter']])
for column in ['BookISBN', 'BookLang']:
cmd = "SELECT count(*) as counter FROM books WHERE "
cmd += "(%s is null or %s = '' or %s = 'Unknown')"
res = myDB.match(cmd % (column, column, column))
missing_stats.append([column.replace('Book', 'No'), res['counter']])
cmd = "SELECT count(*) as counter FROM genres"
res = myDB.match(cmd)
missing_stats.append(['Genres', res['counter']])
author_stats = []
res = myDB.match("SELECT count(*) as counter FROM authors")

4
lazylibrarian/database.py
View File

@ -19,6 +19,7 @@ import os
import sqlite3
import threading
import time
import traceback
import lazylibrarian
from lazylibrarian import logger
@ -124,7 +125,8 @@ class DBConnection:
with open(self.dblog, 'a') as f:
f.write("%s %d %.4f %s [%s]\n" % (time.asctime(), attempt, elapsed, query, args))
f.write("%s DatabaseError: %s\n" % (time.asctime(), e))
logger.error('Fatal error executing %s :: %s' % (query, e))
logger.error('Fatal error executing %s :%s: %s' % (query, args, e))
logger.error("%s" % traceback.format_exc())
raise
except Exception as e:

86
lazylibrarian/dbupgrade.py
View File

@ -21,7 +21,7 @@ import traceback
import lazylibrarian
from lazylibrarian import logger, database
from lazylibrarian.common import restartJobs, pwd_generator
from lazylibrarian.formatter import plural, makeUnicode, makeBytestr, md5_utf8
from lazylibrarian.formatter import plural, makeUnicode, makeBytestr, md5_utf8, getList
from lazylibrarian.importer import addAuthorToDB, update_totals
from lazylibrarian.versioncheck import runGit
@ -92,8 +92,9 @@ def upgrade_needed():
# 44 move hosting to gitlab
# 45 update local git repo to new origin
# 46 remove pastissues table and rebuild to ensure no foreign key
# 47 genres and genrebooks tables
db_current_version = 46
db_current_version = 47
if db_version < db_current_version:
return db_current_version
@ -170,6 +171,7 @@ def dbupgrade(db_current_version):
'Email TEXT, Name TEXT, Perms INTEGER DEFAULT 0, HaveRead TEXT, ToRead TEXT, ' +
'CalibreRead TEXT, CalibreToRead TEXT, BookType TEXT, SendTo TEXT)')
myDB.action('CREATE TABLE isbn (Words TEXT, ISBN TEXT)')
myDB.action('CREATE TABLE genres (GenreID INTEGER UNIQUE, GenreName TEXT)')
if lazylibrarian.FOREIGN_KEY:
myDB.action('CREATE TABLE books (AuthorID TEXT REFERENCES authors (AuthorID) ' +
@ -194,6 +196,9 @@ def dbupgrade(db_current_version):
myDB.action('CREATE TABLE failedsearch (BookID TEXT REFERENCES books (BookID) ' +
'ON DELETE CASCADE, Library TEXT, Time TEXT, Interval INTEGER DEFAULT 0, ' +
'Count INTEGER DEFAULT 0)')
myDB.action('CREATE TABLE genrebooks (GenreID INTEGER, ' +
'BookID TEXT REFERENCES books (BookID) ON DELETE CASCADE, ' +
'UNIQUE (GenreID,BookID))')
else:
# running a very old sqlite on a nas that can't be updated, no foreign key support
# so orphans get cleaned up at program startup
@ -214,6 +219,8 @@ def dbupgrade(db_current_version):
myDB.action('CREATE TABLE sync (UserID TEXT, Label TEXT, Date TEXT, SyncList TEXT)')
myDB.action('CREATE TABLE failedsearch (BookID TEXT, Library TEXT, Time TEXT, ' +
'Interval INTEGER DEFAULT 0, Count INTEGER DEFAULT 0)')
myDB.action('CREATE TABLE genrebooks (GenreID INTEGER, BookID TEXT, ' +
'UNIQUE (GenreID,BookID))')
# pastissues table has same layout as wanted table, code below is to save typos if columns change
res = myDB.match("SELECT sql FROM sqlite_master WHERE type='table' AND name='wanted'")
@ -268,7 +275,7 @@ def check_db(myDB):
data = list(res)
if data[2] == 'AuthorID':
unique = True
break;
break
if not unique:
res = myDB.match('select count(distinct authorid) as d,count(authorid) as c from authors')
if res['d'] == res['c']:
@ -345,6 +352,49 @@ def check_db(myDB):
seriesid = item["SeriesID"]
myDB.action('DELETE from series WHERE SeriesID=?', (seriesid,))
# check if genre exclusions/translations have altered
if lazylibrarian.GRGENRES:
for item in lazylibrarian.GRGENRES['genreExclude']:
match = myDB.match('SELECT GenreID from genres where GenreName=? COLLATE NOCASE', (item,))
if match:
cnt += 1
msg = 'Removing excluded genre [%s]' % item
logger.warn(msg)
myDB.action('DELETE from genre WHERE GenreID=?', (match['GenreID'],))
for item in lazylibrarian.GRGENRES['genreExcludeParts']:
cmd = 'SELECT GenreID,GenreName from genres where GenreName like "%' + item + '%" COLLATE NOCASE'
matches = myDB.select(cmd)
if matches:
cnt += len(matches)
for itm in matches:
msg = 'Removing excluded genre [%s]' % itm['GenreName']
logger.warn(msg)
myDB.action('DELETE from genre WHERE GenreID=?', (itm['GenreID'],))
for item in lazylibrarian.GRGENRES['genreReplace']:
match = myDB.match('SELECT GenreID from genres where GenreName=? COLLATE NOCASE', (item,))
if match:
newitem = lazylibrarian.GRGENRES['genreReplace'][item]
newmatch = myDB.match('SELECT GenreID from genres where GenreName=? COLLATE NOCASE', (newitem,))
cnt += 1
msg = 'Replacing genre [%s] with [%s]' % (item, newitem)
logger.warn(msg)
if not newmatch:
myDB.action('INSERT into genres (GenreName) VALUES (?)', (newitem,))
newmatch = myDB.match('SELECT GenreID from genres where GenreName=?', (newitem,))
myDB.action('UPDATE genrebooks SET GenreID=? WHERE GenreID=?', (newmatch, match))
# remove genres with no books
cmd = 'select GenreID, (select count(*) as counter from genrebooks where genres.genreid = genrebooks.genreid)'
cmd += ' as cnt from genres where cnt = 0'
genres = myDB.select(cmd)
if genres:
cnt += len(genres)
msg = 'Removing %s empty genre%s' % (len(genres), plural(len(genres)))
logger.warn(msg)
for item in genres:
genreid = item["GenreID"]
myDB.action('DELETE from genres WHERE GenreID=?', (genreid,))
# remove orphan entries (foreign key not available)
for entry in [
['authorid', 'books', 'authors'],
@ -352,6 +402,7 @@ def check_db(myDB):
['seriesid', 'seriesauthors', 'series'],
['authorid', 'seriesauthors', 'authors'],
['title', 'issues', 'magazines'],
['genreid', 'genrebooks', 'genres'],
]:
orphans = myDB.select('select %s from %s except select %s from %s' %
(entry[0], entry[1], entry[0], entry[2]))
@ -1296,3 +1347,32 @@ def db_v46(myDB, upgradelog):
myDB.action(res['sql'].replace('wanted', 'pastissues'))
upgradelog.write("%s v46: complete\n" % time.ctime())
def db_v47(myDB, upgradelog):
upgradelog.write("%s v47: %s\n" % (time.ctime(), "Creating genre tables"))
if not has_column(myDB, "genres", "GenreID"):
myDB.action('CREATE TABLE genres (GenreID INTEGER PRIMARY KEY AUTOINCREMENT, GenreName TEXT UNIQUE)')
if lazylibrarian.FOREIGN_KEY:
myDB.action('CREATE TABLE genrebooks (GenreID INTEGER REFERENCES genres (GenreID), ' +
'BookID TEXT REFERENCES books (BookID) ON DELETE CASCADE, ' +
'UNIQUE (GenreID,BookID))')
else:
myDB.action('CREATE TABLE genrebooks (GenreID INTEGER, BookID TEXT, ' +
'UNIQUE (GenreID,BookID))')
res = myDB.select('SELECT bookid,bookgenre FROM books WHERE (Status="Open" or AudioStatus="Open")')
tot = len(res)
if tot:
upgradelog.write("%s v47: Upgrading %s genres\n" % (time.ctime(), tot))
cnt = 0
for book in res:
cnt += 1
myDB.action('DELETE from genrebooks WHERE BookID=?', (book['bookid'],))
lazylibrarian.UPDATE_MSG = "Updating genres %s of %s" % (cnt, tot)
for item in getList(book['bookgenre'], ','):
match = myDB.match('SELECT GenreID from genres where GenreName=? COLLATE NOCASE', (item,))
if not match:
myDB.action('INSERT into genres (GenreName) VALUES (?)', (item,))
match = myDB.match('SELECT GenreID from genres where GenreName=?', (item,))
myDB.action('INSERT into genrebooks (GenreID, BookID) VALUES (?,?)',
(match['GenreID'], book['bookid']), suppress='UNIQUE')
upgradelog.write("%s v47: complete\n" % time.ctime())

43
lazylibrarian/gr.py
View File

@ -24,11 +24,12 @@ except ImportError:
import lazylibrarian
from lazylibrarian import logger, database
from lazylibrarian.bookwork import getWorkSeries, getWorkPage, deleteEmptySeries, \
setSeries, setStatus, isbn_from_words, thingLang, get_book_pubdate, get_gb_info
setSeries, setStatus, isbn_from_words, thingLang, get_book_pubdate, get_gb_info, \
get_gr_genres, setGenres
from lazylibrarian.images import getBookCover
from lazylibrarian.cache import gr_xml_request, cache_img
from lazylibrarian.formatter import plural, today, replace_all, bookSeries, unaccented, split_title, getList, \
cleanName, is_valid_isbn, formatAuthorName, check_int, makeUnicode, check_year
cleanName, is_valid_isbn, formatAuthorName, check_int, makeUnicode, check_year, check_float
from lib.fuzzywuzzy import fuzz
from lib.six import PY2
# noinspection PyUnresolvedReferences
@ -108,11 +109,11 @@ class GoodReads:
bookimg = 'images/nocover.png'
try:
bookrate = author.find('average_rating').text
bookrate = check_float(author.find('average_rating').text, 0)
except KeyError:
bookrate = 0
bookrate = 0.0
try:
bookrate_count = int(author.find('ratings_count').text)
bookrate_count = check_int(author.find('ratings_count').text, 0)
except KeyError:
bookrate_count = 0
@ -184,7 +185,7 @@ class GoodReads:
'bookdate': bookdate,
'booklang': booklang,
'booklink': booklink,
'bookrate': float(bookrate),
'bookrate': bookrate,
'bookrate_count': bookrate_count,
'bookimg': bookimg,
'bookpages': bookpages,
@ -195,7 +196,7 @@ class GoodReads:
'book_fuzz': book_fuzz,
'isbn_fuzz': isbn_fuzz,
'highest_fuzz': highest_fuzz,
'num_reviews': float(bookrate)
'num_reviews': bookrate
})
resultcount += 1
@ -347,7 +348,7 @@ class GoodReads:
if value is None:
value = default
if idx == 'rate':
value = float(value)
value = check_float(value, 0.0)
mydict[val] = value
return mydict
@ -785,9 +786,13 @@ class GoodReads:
if locked:
locked_count += 1
else:
if not bookgenre:
genres, _ = get_gr_genres(bookid)
if genres:
bookgenre = ', '.join(genres)
if not bookdesc: # or not bookgenre:
infodict = get_gb_info(isbn=bookisbn, author=authorNameResult, title=bookname,
expire=False)
infodict = get_gb_info(isbn=bookisbn, author=authorNameResult,
title=bookname, expire=False)
if infodict is not None: # None if api blocked
if not bookdesc:
if infodict and infodict['desc']:
@ -829,6 +834,8 @@ class GoodReads:
myDB.upsert("books", newValueDict, controlValueDict)
setGenres(getList(bookgenre), bookid)
updateValueDict = {}
# need to run getWorkSeries AFTER adding to book table (foreign key constraint)
serieslist = []
@ -1112,7 +1119,7 @@ class GoodReads:
bookisbn = rootxml.find('./book/isbn').text
bookpub = rootxml.find('./book/publisher').text
booklink = rootxml.find('./book/link').text
bookrate = float(rootxml.find('./book/average_rating').text)
bookrate = check_float(rootxml.find('./book/average_rating').text, 0)
bookpages = rootxml.find('.book/num_pages').text
workid = rootxml.find('.book/work/id').text
@ -1172,6 +1179,9 @@ class GoodReads:
bookisbn = res
bookgenre = ''
genres, _ = get_gr_genres(bookid)
if genres:
bookgenre = ', '.join(genres)
if not bookdesc:
infodict = get_gb_info(isbn=bookisbn, author=authorname, title=bookname, expire=False)
if infodict is not None: # None if api blocked
@ -1179,10 +1189,11 @@ class GoodReads:
bookdesc = infodict['desc']
else:
bookdesc = 'No Description'
if infodict and infodict['genre']:
bookgenre = infodict['genre']
else:
bookgenre = 'Unknown'
if not bookgenre:
if infodict and infodict['genre']:
bookgenre = infodict['genre']
else:
bookgenre = 'Unknown'
controlValueDict = {"BookID": bookid}
newValueDict = {
"AuthorID": AuthorID,
@ -1237,6 +1248,8 @@ class GoodReads:
logger.debug('Updated series: %s [%s]' % (bookid, serieslist))
setSeries(serieslist, bookid)
setGenres(getList(bookgenre), bookid)
worklink = getWorkPage(bookid)
if worklink:
controlValueDict = {"BookID": bookid}

29
lazylibrarian/opds.py
View File

@ -308,8 +308,10 @@ class OPDS(object):
'rel': 'subsection',
}
)
cmd = "select distinct BookGenre from books where Status='Open' and BookGenre != '' and BookGenre !='Unknown'"
cmd = 'select genrename,(select count(*) as counter from genrebooks,books where '
cmd += 'genrebooks.genreid = genres.genreid and books.status="Open" '
cmd += 'and books.bookid=genrebooks.bookid) as cnt from genres where cnt > 0'
# cmd = "select distinct BookGenre from books where Status='Open' and BookGenre != '' and BookGenre !='Unknown'"
res = myDB.select(cmd)
if res and len(res) > 0:
entries.append(
@ -365,10 +367,13 @@ class OPDS(object):
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchgenres.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Genre'))
cmd = "SELECT distinct BookGenre from books WHERE Status='Open' AND "
cmd = 'select genrename,(select count(*) as counter from genrebooks,books where '
cmd += 'genrebooks.genreid = genres.genreid and books.status="Open" '
cmd += 'and books.bookid=genrebooks.bookid) as cnt from genres where cnt > 0'
if 'query' in kwargs:
cmd += "BookGenre LIKE '%" + kwargs['query'] + "%' AND "
cmd += "BookGenre !='' AND BookGenre != 'Unknown' order by BookGenre"
cmd += " and genrename LIKE '%" + kwargs['query'] + "%'"
cmd += ' order by cnt DESC,genrename ASC'
results = myDB.select(cmd)
if limit:
page = results[index:(index + limit)]
@ -376,16 +381,14 @@ class OPDS(object):
page = results
limit = len(page)
for genre in page:
res = myDB.match('SELECT count(*) as counter from books where Status="Open" AND BookGenre=?',
(genre['BookGenre'],))
totalbooks = res['counter']
name = makeUnicode(genre['BookGenre'])
totalbooks = genre['cnt']
name = makeUnicode(genre['genrename'])
entry = {
'title': escape('%s (%s)' % (name, totalbooks)),
'id': escape('genre:%s' % genre['BookGenre']),
'id': escape('genre:%s' % genre['genrename']),
'updated': now(),
'content': escape('%s (%s)' % (name, totalbooks)),
'href': '%s?cmd=Genre&amp;genre=%s%s' % (self.opdsroot, quote_plus(genre['BookGenre']), userid),
'href': '%s?cmd=Genre&amp;genre=%s%s' % (self.opdsroot, quote_plus(genre['genrename']), userid),
'author': escape('%s' % name),
'kind': 'navigation',
'rel': 'subsection',
@ -427,8 +430,10 @@ class OPDS(object):
return
links = []
entries = []
cmd = "SELECT BookName,BookDate,BookAdded,BookDesc,BookImg,BookFile,AudioFile,books.BookID "
cmd += "from books where (Status='Open' or AudioStatus='Open') and BookGenre=? "
cmd += "from genrebooks,genres,books WHERE (books.Status='Open' or books.AudioStatus='Open') "
cmd += "AND books.Bookid=genrebooks.BookID AND genrebooks.genreid=genres.genreid AND genrename=?"
cmd += "order by BookName"
results = myDB.select(cmd, (kwargs['genre'],))
if not len(results):

1
lazylibrarian/webStart.py
View File

@ -144,6 +144,7 @@ def initialize(options=None):
})
conf['/api'].update({
'tools.auth_basic.on': False,
'response.timeout': 3600,
})
if options['opds_authentication']:
user_list = {}