diff options
Diffstat (limited to 'module/database/FileDatabase.py')
| -rw-r--r-- | module/database/FileDatabase.py | 157 |
1 files changed, 52 insertions, 105 deletions
diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 7e7efb028..f01d02769 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -68,12 +67,12 @@ class FileHandler: return func(*args) return new - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- def save(self): """saves all data to backend""" self.db.commit() - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- def syncSave(self): """saves all data to backend and waits until all data are written""" pyfiles = self.cache.values() @@ -131,7 +130,7 @@ class FileHandler: #@TODO change from reloadAll event to package update event self.core.pullManager.addEvent(ReloadAllEvent("collector")) - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- @lock @change def addPackage(self, name, folder, queue=0): @@ -142,7 +141,7 @@ class FileHandler: self.core.pullManager.addEvent(e) return lastID - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- @lock @change def deletePackage(self, id): @@ -178,7 +177,7 @@ class FileHandler: pack.order -= 1 pack.notifyChange() - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- @lock @change def deleteLink(self, id): @@ -190,7 +189,7 @@ class FileHandler: pid = f.packageid e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") - + oldorder = f.order if id in self.core.threadManager.processingIds(): @@ -206,26 +205,26 @@ class FileHandler: p = self.getPackage(pid) if not len(p.getChildren()): p.delete() - + pyfiles = self.cache.values() for pyfile in pyfiles: if pyfile.packageid == pid and pyfile.order > oldorder: pyfile.order -= 1 pyfile.notifyChange() - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- def releaseLink(self, id): """removes pyfile from cache""" if id in self.cache: del self.cache[id] - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- def releasePackage(self, id): """removes package from cache""" if id in self.packageCache: del self.packageCache[id] - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- def updateLink(self, pyfile): """updates link""" self.db.updateLink(pyfile) @@ -233,7 +232,7 @@ class FileHandler: e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") self.core.pullManager.addEvent(e) - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- def updatePackage(self, pypack): """updates a package""" self.db.updatePackage(pypack) @@ -241,7 +240,7 @@ class FileHandler: e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") self.core.pullManager.addEvent(e) - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- def getPackage(self, id): """return package instance""" @@ -250,7 +249,7 @@ class FileHandler: else: return self.db.getPackage(id) - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- def getPackageData(self, id): """returns dict with package information""" pack = self.getPackage(id) @@ -274,7 +273,7 @@ class FileHandler: return pack - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- def getFileData(self, id): """returns dict with file information""" if id in self.cache: @@ -282,7 +281,7 @@ class FileHandler: return self.db.getLinkData(id) - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- def getFile(self, id): """returns pyfile instance""" if id in self.cache: @@ -290,7 +289,7 @@ class FileHandler: else: return self.db.getFile(id) - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- @lock def getJob(self, occ): """get suitable job""" @@ -434,7 +433,7 @@ class FileHandler: e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") self.core.pullManager.addEvent(e) - + self.db.clearPackageOrder(p) p = self.db.getPackage(id) @@ -443,7 +442,7 @@ class FileHandler: self.db.updatePackage(p) self.db.reorderPackage(p, -1, True) - + packs = self.packageCache.values() for pack in packs: if pack.queue != queue and pack.order > oldorder: @@ -453,7 +452,7 @@ class FileHandler: self.db.commit() self.releasePackage(id) p = self.getPackage(id) - + e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue") self.core.pullManager.addEvent(e) @@ -540,7 +539,7 @@ class FileHandler: urls = [] for pyfile in data.itervalues(): - if pyfile["status"] not in (0, 12, 13): + if pyfile["status"] not in (0, 12, 13): urls.append((pyfile["url"], pyfile["plugin"])) self.core.threadManager.createInfoThread(urls, pid) @@ -573,23 +572,23 @@ class FileHandler: """ restart all failed links """ self.db.restartFailed() -class FileMethods(): +class FileMethods: @style.queue def filecount(self, queue): """returns number of files in queue""" - self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue, )) + self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue,)) return self.c.fetchone()[0] @style.queue def queuecount(self, queue): """ number of files in queue not finished yet""" - self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0,4)", (queue, )) + self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0, 4)", (queue,)) return self.c.fetchone()[0] @style.queue def processcount(self, queue, fid): """ number of files which have to be proccessed """ - self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2,3,5,7,12) AND l.id != ?", (queue, str(fid))) + self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2, 3, 5, 7, 12) AND l.id != ?", (queue, str(fid))) return self.c.fetchone()[0] @style.inner @@ -600,7 +599,7 @@ class FileMethods(): return max + 1 else: return 0 - + @style.inner def _nextFileOrder(self, package): self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) @@ -609,7 +608,7 @@ class FileMethods(): return max + 1 else: return 0 - + @style.queue def addLink(self, url, name, plugin, package): order = self._nextFileOrder(package) @@ -618,7 +617,7 @@ class FileMethods(): @style.queue def addLinks(self, links, package): - """ links is a list of tupels (url,plugin)""" + """ links is a list of tupels (url, plugin)""" order = self._nextFileOrder(package) orders = [order + x for x in range(len(links))] links = [(x[0], x[0], x[1], package, o) for x, o in zip(links, orders)] @@ -658,7 +657,7 @@ class FileMethods(): } """ - self.c.execute('SELECT l.id,l.url,l.name,l.size,l.status,l.error,l.plugin,l.package,l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', (q,)) + self.c.execute('SELECT l.id, l.url, l.name, l.size, l.status, l.error, l.plugin, l.package, l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', (q,)) data = {} for r in self.c: data[r[0]] = { @@ -688,7 +687,7 @@ class FileMethods(): format: { - id: {'name': name ... 'links': {} }, ... + id: {'name': name ... 'links': {}}, ... } """ self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ @@ -713,11 +712,11 @@ class FileMethods(): } return data - + @style.queue def getLinkData(self, id): """get link information as dict""" - self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE id=?', (str(id), )) + self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?', (str(id),)) data = {} r = self.c.fetchone() if not r: @@ -741,7 +740,7 @@ class FileMethods(): @style.queue def getPackageData(self, id): """get data about links for a package""" - self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id), )) + self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id),)) data = {} for r in self.c: @@ -764,22 +763,22 @@ class FileMethods(): @style.async def updateLink(self, f): - self.c.execute('UPDATE links SET url=?,name=?,size=?,status=?,error=?,package=? WHERE id=?', (f.url, f.name, f.size, f.status, f.error, str(f.packageid), str(f.id))) + self.c.execute('UPDATE links SET url=?, name=?, size=?, status=?, error=?, package=? WHERE id=?', (f.url, f.name, f.size, f.status, f.error, str(f.packageid), str(f.id))) @style.queue def updatePackage(self, p): - self.c.execute('UPDATE packages SET name=?,folder=?,site=?,password=?,queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) - + self.c.execute('UPDATE packages SET name=?, folder=?, site=?, password=?, queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) + @style.queue def updateLinkInfo(self, data): """ data is list of tupels (name, size, status, url) """ - self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1,2,3,14)', data) + self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1, 2, 3, 14)', data) ids = [] self.c.execute('SELECT id FROM links WHERE url IN (\'%s\')' % "','".join([x[3] for x in data])) for r in self.c: ids.append(int(r[0])) return ids - + @style.queue def reorderPackage(self, p, position, noMove=False): if position == -1: @@ -791,7 +790,7 @@ class FileMethods(): self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND queue=? AND packageorder >= 0', (position, p.order, p.queue)) self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) - + @style.queue def reorderLink(self, f, position): """ reorder link with f as dict for pyfile """ @@ -801,34 +800,33 @@ class FileMethods(): self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder <= ? AND linkorder > ? AND package=?', (position, f["order"], f["package"])) self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"])) - - + @style.queue def clearPackageOrder(self, p): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) - + @style.async def restartFile(self, id): - self.c.execute('UPDATE links SET status=3,error="" WHERE id=?', (str(id),)) + self.c.execute('UPDATE links SET status=3, error="" WHERE id=?', (str(id),)) @style.async def restartPackage(self, id): self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) - + @style.queue def getPackage(self, id): """return package instance from id""" - self.c.execute("SELECT name,folder,site,password,queue,packageorder FROM packages WHERE id=?", (str(id), )) + self.c.execute("SELECT name, folder, site, password, queue, packageorder FROM packages WHERE id=?", (str(id),)) r = self.c.fetchone() if not r: return None return PyPackage(self.manager, id, * r) - #---------------------------------------------------------------------- + #-------------------------------------------------------------------------- @style.queue def getFile(self, id): """return link instance from id""" - self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id), )) + self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id),)) r = self.c.fetchone() if not r: return None return PyFile(self.manager, id, * r) @@ -845,11 +843,11 @@ class FileMethods(): for i, item in enumerate(occ): if i: cmd += ", " cmd += "'%s'" % item - + cmd += ")" - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) - + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) + self.c.execute(cmd) # very bad! return [x[0] for x in self.c] @@ -857,7 +855,7 @@ class FileMethods(): @style.queue def getPluginJob(self, plugins): """returns pyfile ids with suited plugins""" - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins self.c.execute(cmd) # very bad! @@ -866,18 +864,18 @@ class FileMethods(): @style.queue def getUnfinished(self, pid): """return list of max length 3 ids with pyfiles in package not finished or processed""" - + self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) return [r[0] for r in self.c] @style.queue def deleteFinished(self): - self.c.execute("DELETE FROM links WHERE status IN (0,4)") + self.c.execute("DELETE FROM links WHERE status IN (0, 4)") self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") @style.queue def restartFailed(self): - self.c.execute("UPDATE links SET status=3,error='' WHERE status IN (6, 8, 9)") + self.c.execute("UPDATE links SET status=3, error='' WHERE status IN (6, 8, 9)") @style.queue def findDuplicates(self, id, folder, filename): @@ -891,54 +889,3 @@ class FileMethods(): self.c.execute("DELETE FROM packages;") DatabaseBackend.registerSub(FileMethods) - -if __name__ == "__main__": - - pypath = "." - _ = lambda x: x - - db = FileHandler(None) - - #p = PyFile(db, 5) - #sleep(0.1) - - a = time() - - #print db.addPackage("package", "folder" , 1) - - pack = db.db.addPackage("package", "folder", 1) - - updates = [] - - - for x in range(0, 200): - x = str(x) - db.db.addLink("http://somehost.com/hoster/file/download?file_id=" + x, x, "BasePlugin", pack) - updates.append(("new name" + x, 0, 3, "http://somehost.com/hoster/file/download?file_id=" + x)) - - - for x in range(0, 100): - updates.append(("unimportant%s" % x, 0, 3, "a really long non existent url%s" % x)) - - db.db.commit() - - b = time() - print "adding 200 links, single sql execs, no commit", b-a - - print db.getCompleteData(1) - - c = time() - - - db.db.updateLinkInfo(updates) - - d = time() - - print "updates", d-c - - print db.getCompleteData(1) - - - e = time() - - print "complete data", e-d |