diff options
Diffstat (limited to 'pyload/database')
| -rw-r--r-- | pyload/database/DatabaseBackend.py | 304 | ||||
| -rw-r--r-- | pyload/database/FileDatabase.py | 891 | ||||
| -rw-r--r-- | pyload/database/StorageDatabase.py | 49 | ||||
| -rw-r--r-- | pyload/database/UserDatabase.py | 108 | ||||
| -rw-r--r-- | pyload/database/__init__.py | 8 |
5 files changed, 1360 insertions, 0 deletions
diff --git a/pyload/database/DatabaseBackend.py b/pyload/database/DatabaseBackend.py new file mode 100644 index 000000000..b25a3c32e --- /dev/null +++ b/pyload/database/DatabaseBackend.py @@ -0,0 +1,304 @@ +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN + @author: mkaay +""" +from threading import Event, Thread +from os import remove +from os.path import exists +from shutil import move + +from Queue import Queue +from traceback import print_exc + +from pyload.utils import chmod + +try: + from pysqlite2 import dbapi2 as sqlite3 +except: + import sqlite3 + +DB_VERSION = 4 + +class style: + db = None + + @classmethod + def setDB(cls, db): + cls.db = db + + @classmethod + def inner(cls, f): + @staticmethod + def x(*args, **kwargs): + if cls.db: + return f(cls.db, *args, **kwargs) + return x + + @classmethod + def queue(cls, f): + @staticmethod + def x(*args, **kwargs): + if cls.db: + return cls.db.queue(f, *args, **kwargs) + return x + + @classmethod + def async(cls, f): + @staticmethod + def x(*args, **kwargs): + if cls.db: + return cls.db.async(f, *args, **kwargs) + return x + +class DatabaseJob: + def __init__(self, f, *args, **kwargs): + self.done = Event() + + self.f = f + self.args = args + self.kwargs = kwargs + + self.result = None + self.exception = False + +# import inspect +# self.frame = inspect.currentframe() + + def __repr__(self): + from os.path import basename + frame = self.frame.f_back + output = "" + for i in range(5): + output += "\t%s:%s, %s\n" % (basename(frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name) + frame = frame.f_back + del frame + del self.frame + + return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result) + + def processJob(self): + try: + self.result = self.f(*self.args, **self.kwargs) + except Exception, e: + print_exc() + try: + print "Database Error @", self.f.__name__, self.args[1:], self.kwargs, e + except: + pass + + self.exception = e + finally: + self.done.set() + + def wait(self): + self.done.wait() + +class DatabaseBackend(Thread): + subs = [] + def __init__(self, core): + Thread.__init__(self) + self.setDaemon(True) + self.core = core + + self.jobs = Queue() + + self.setuplock = Event() + + style.setDB(self) + + def setup(self): + self.start() + self.setuplock.wait() + + def run(self): + """main loop, which executes commands""" + convert = self._checkVersion() #returns None or current version + + self.conn = sqlite3.connect("files.db") + chmod("files.db", 0600) + + self.c = self.conn.cursor() #compatibility + + if convert is not None: + self._convertDB(convert) + + self._createTables() + self._migrateUser() + + self.conn.commit() + + self.setuplock.set() + + while True: + j = self.jobs.get() + if j == "quit": + self.c.close() + self.conn.close() + break + j.processJob() + + @style.queue + def shutdown(self): + self.conn.commit() + self.jobs.put("quit") + + def _checkVersion(self): + """ check db version and delete it if needed""" + if not exists("files.version"): + f = open("files.version", "wb") + f.write(str(DB_VERSION)) + f.close() + return + + f = open("files.version", "rb") + v = int(f.read().strip()) + f.close() + if v < DB_VERSION: + if v < 2: + try: + self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) + except: + print "Filedatabase was deleted due to incompatible version." + remove("files.version") + move("files.db", "files.backup.db") + f = open("files.version", "wb") + f.write(str(DB_VERSION)) + f.close() + return v + + def _convertDB(self, v): + try: + getattr(self, "_convertV%i" % v)() + except: + try: + self.core.log.error(_("Filedatabase could NOT be converted.")) + except: + print "Filedatabase could NOT be converted." + + #convert scripts start----------------------------------------------------- + + def _convertV2(self): + self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + try: + self.manager.core.log.info(_("Database was converted from v2 to v3.")) + except: + print "Database was converted from v2 to v3." + self._convertV3() + + def _convertV3(self): + self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + try: + self.manager.core.log.info(_("Database was converted from v3 to v4.")) + except: + print "Database was converted from v3 to v4." + + #convert scripts end------------------------------------------------------- + + def _createTables(self): + """create tables for database""" + + self.c.execute('CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') + self.c.execute('CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') + self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)') + self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + + self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \ + SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\ + FROM packages p JOIN links l ON p.id = l.package LEFT OUTER JOIN\ + (SELECT p.id AS id, COUNT(*) AS linksdone, SUM(l.size) AS sizedone \ + FROM packages p JOIN links l ON p.id = l.package AND l.status in (0, 4, 13) GROUP BY p.id) s ON s.id = p.id \ + GROUP BY p.id') + + #try to lower ids + self.c.execute('SELECT max(id) FROM LINKS') + fid = self.c.fetchone()[0] + if fid: + fid = int(fid) + else: + fid = 0 + self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links")) + + + self.c.execute('SELECT max(id) FROM packages') + pid = self.c.fetchone()[0] + if pid: + pid = int(pid) + else: + pid = 0 + self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (pid, "packages")) + + self.c.execute('VACUUM') + + + def _migrateUser(self): + if exists("pyload.db"): + try: + self.core.log.info(_("Converting old Django DB")) + except: + print "Converting old Django DB" + conn = sqlite3.connect('pyload.db') + c = conn.cursor() + c.execute("SELECT username, password, email from auth_user WHERE is_superuser") + users = [] + for r in c: + pw = r[1].split("$") + users.append((r[0], pw[1] + pw[2], r[2])) + c.close() + conn.close() + + self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users) + move("pyload.db", "pyload.old.db") + + def createCursor(self): + return self.conn.cursor() + + @style.async + def commit(self): + self.conn.commit() + + @style.queue + def syncSave(self): + self.conn.commit() + + @style.async + def rollback(self): + self.conn.rollback() + + def async(self, f, *args, **kwargs): + args = (self,) + args + job = DatabaseJob(f, *args, **kwargs) + self.jobs.put(job) + + def queue(self, f, *args, **kwargs): + args = (self,) + args + job = DatabaseJob(f, *args, **kwargs) + self.jobs.put(job) + job.wait() + return job.result + + @classmethod + def registerSub(cls, klass): + cls.subs.append(klass) + + @classmethod + def unregisterSub(cls, klass): + cls.subs.remove(klass) + + def __getattr__(self, attr): + for sub in DatabaseBackend.subs: + if hasattr(sub, attr): + return getattr(sub, attr) diff --git a/pyload/database/FileDatabase.py b/pyload/database/FileDatabase.py new file mode 100644 index 000000000..5bd1e5a5f --- /dev/null +++ b/pyload/database/FileDatabase.py @@ -0,0 +1,891 @@ +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN + @author: mkaay +""" + + +from threading import RLock +from time import time + +from pyload.utils import formatSize, lock +from pyload.manager.event.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent +from pyload.datatype.PyPackage import PyPackage +from pyload.datatype.PyFile import PyFile +from pyload.database import style, DatabaseBackend + +try: + from pysqlite2 import dbapi2 as sqlite3 +except: + import sqlite3 + + +class FileHandler: + """Handles all request made to obtain information, + modify status or other request for links or packages""" + + def __init__(self, core): + """Constructor""" + self.core = core + + # translations + self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), _("downloading"), _("processing"), _("unknown")] + + self.cache = {} #holds instances for files + self.packageCache = {} # same for packages + #@TODO: purge the cache + + self.jobCache = {} + + self.lock = RLock() #@TODO should be a Lock w/o R + #self.lock._Verbose__verbose = True + + self.filecount = -1 # if an invalid value is set get current value from db + self.queuecount = -1 #number of package to be loaded + self.unchanged = False #determines if any changes was made since last call + + self.db = self.core.db + + def change(func): + def new(*args): + args[0].unchanged = False + args[0].filecount = -1 + args[0].queuecount = -1 + args[0].jobCache = {} + return func(*args) + return new + + #-------------------------------------------------------------------------- + def save(self): + """saves all data to backend""" + self.db.commit() + + #-------------------------------------------------------------------------- + def syncSave(self): + """saves all data to backend and waits until all data are written""" + pyfiles = self.cache.values() + for pyfile in pyfiles: + pyfile.sync() + + pypacks = self.packageCache.values() + for pypack in pypacks: + pypack.sync() + + self.db.syncSave() + + @lock + def getCompleteData(self, queue=1): + """gets a complete data representation""" + + data = self.db.getAllLinks(queue) + packs = self.db.getAllPackages(queue) + + data.update([(x.id, x.toDbDict()[x.id]) for x in self.cache.values()]) + + for x in self.packageCache.itervalues(): + if x.queue != queue or x.id not in packs: continue + packs[x.id].update(x.toDict()[x.id]) + + for key, value in data.iteritems(): + if value["package"] in packs: + packs[value["package"]]["links"][key] = value + + return packs + + @lock + def getInfoData(self, queue=1): + """gets a data representation without links""" + + packs = self.db.getAllPackages(queue) + for x in self.packageCache.itervalues(): + if x.queue != queue or x.id not in packs: continue + packs[x.id].update(x.toDict()[x.id]) + + return packs + + @lock + @change + def addLinks(self, urls, package): + """adds links""" + + self.core.addonManager.dispatchEvent("linksAdded", urls, package) + + data = self.core.pluginManager.parseUrls(urls) + + self.db.addLinks(data, package) + self.core.threadManager.createInfoThread(data, package) + + #@TODO change from reloadAll event to package update event + self.core.pullManager.addEvent(ReloadAllEvent("collector")) + + #-------------------------------------------------------------------------- + @lock + @change + def addPackage(self, name, folder, queue=0): + """adds a package, default to link collector""" + lastID = self.db.addPackage(name, folder, queue) + p = self.db.getPackage(lastID) + e = InsertEvent("pack", lastID, p.order, "collector" if not queue else "queue") + self.core.pullManager.addEvent(e) + return lastID + + #-------------------------------------------------------------------------- + @lock + @change + def deletePackage(self, id): + """delete package and all contained links""" + + p = self.getPackage(id) + if not p: + if id in self.packageCache: del self.packageCache[id] + return + + oldorder = p.order + queue = p.queue + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + + pyfiles = self.cache.values() + + for pyfile in pyfiles: + if pyfile.packageid == id: + pyfile.abortDownload() + pyfile.release() + + self.db.deletePackage(p) + self.core.pullManager.addEvent(e) + self.core.addonManager.dispatchEvent("packageDeleted", id) + + if id in self.packageCache: + del self.packageCache[id] + + packs = self.packageCache.values() + for pack in packs: + if pack.queue == queue and pack.order > oldorder: + pack.order -= 1 + pack.notifyChange() + + #-------------------------------------------------------------------------- + @lock + @change + def deleteLink(self, id): + """deletes links""" + + f = self.getFile(id) + if not f: + return None + + pid = f.packageid + e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") + + oldorder = f.order + + if id in self.core.threadManager.processingIds(): + self.cache[id].abortDownload() + + if id in self.cache: + del self.cache[id] + + self.db.deleteLink(f) + + self.core.pullManager.addEvent(e) + + p = self.getPackage(pid) + if not len(p.getChildren()): + p.delete() + + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid == pid and pyfile.order > oldorder: + pyfile.order -= 1 + pyfile.notifyChange() + + #-------------------------------------------------------------------------- + def releaseLink(self, id): + """removes pyfile from cache""" + if id in self.cache: + del self.cache[id] + + #-------------------------------------------------------------------------- + def releasePackage(self, id): + """removes package from cache""" + if id in self.packageCache: + del self.packageCache[id] + + #-------------------------------------------------------------------------- + def updateLink(self, pyfile): + """updates link""" + self.db.updateLink(pyfile) + + e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") + self.core.pullManager.addEvent(e) + + #-------------------------------------------------------------------------- + def updatePackage(self, pypack): + """updates a package""" + self.db.updatePackage(pypack) + + e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") + self.core.pullManager.addEvent(e) + + #-------------------------------------------------------------------------- + def getPackage(self, id): + """return package instance""" + + if id in self.packageCache: + return self.packageCache[id] + else: + return self.db.getPackage(id) + + #-------------------------------------------------------------------------- + def getPackageData(self, id): + """returns dict with package information""" + pack = self.getPackage(id) + + if not pack: + return None + + pack = pack.toDict()[id] + + data = self.db.getPackageData(id) + + tmplist = [] + + cache = self.cache.values() + for x in cache: + if int(x.toDbDict()[x.id]["package"]) == int(id): + tmplist.append((x.id, x.toDbDict()[x.id])) + data.update(tmplist) + + pack["links"] = data + + return pack + + #-------------------------------------------------------------------------- + def getFileData(self, id): + """returns dict with file information""" + if id in self.cache: + return self.cache[id].toDbDict() + + return self.db.getLinkData(id) + + #-------------------------------------------------------------------------- + def getFile(self, id): + """returns pyfile instance""" + if id in self.cache: + return self.cache[id] + else: + return self.db.getFile(id) + + #-------------------------------------------------------------------------- + @lock + def getJob(self, occ): + """get suitable job""" + + #@TODO clean mess + #@TODO improve selection of valid jobs + + if occ in self.jobCache: + if self.jobCache[occ]: + id = self.jobCache[occ].pop() + if id == "empty": + pyfile = None + self.jobCache[occ].append("empty") + else: + pyfile = self.getFile(id) + else: + jobs = self.db.getJob(occ) + jobs.reverse() + if not jobs: + self.jobCache[occ].append("empty") + pyfile = None + else: + self.jobCache[occ].extend(jobs) + pyfile = self.getFile(self.jobCache[occ].pop()) + + else: + self.jobCache = {} #better not caching to much + jobs = self.db.getJob(occ) + jobs.reverse() + self.jobCache[occ] = jobs + + if not jobs: + self.jobCache[occ].append("empty") + pyfile = None + else: + pyfile = self.getFile(self.jobCache[occ].pop()) + + #@TODO: maybe the new job has to be approved... + + + #pyfile = self.getFile(self.jobCache[occ].pop()) + return pyfile + + @lock + def getDecryptJob(self): + """return job for decrypting""" + if "decrypt" in self.jobCache: + return None + + plugins = self.core.pluginManager.crypterPlugins.keys() + self.core.pluginManager.containerPlugins.keys() + plugins = str(tuple(plugins)) + + jobs = self.db.getPluginJob(plugins) + if jobs: + return self.getFile(jobs[0]) + else: + self.jobCache["decrypt"] = "empty" + return None + + def getFileCount(self): + """returns number of files""" + + if self.filecount == -1: + self.filecount = self.db.filecount(1) + + return self.filecount + + def getQueueCount(self, force=False): + """number of files that have to be processed""" + if self.queuecount == -1 or force: + self.queuecount = self.db.queuecount(1) + + return self.queuecount + + def checkAllLinksFinished(self): + """checks if all files are finished and dispatch event""" + + if not self.getQueueCount(True): + self.core.addonManager.dispatchEvent("allDownloadsFinished") + self.core.log.debug("All downloads finished") + return True + + return False + + def checkAllLinksProcessed(self, fid): + """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" + + # reset count so statistic will update (this is called when dl was processed) + self.resetCount() + + if not self.db.processcount(1, fid): + self.core.addonManager.dispatchEvent("allDownloadsProcessed") + self.core.log.debug("All downloads processed") + return True + + return False + + def resetCount(self): + self.queuecount = -1 + + @lock + @change + def restartPackage(self, id): + """restart package""" + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid == id: + self.restartFile(pyfile.id) + + self.db.restartPackage(id) + + if id in self.packageCache: + self.packageCache[id].setFinished = False + + e = UpdateEvent("pack", id, "collector" if not self.getPackage(id).queue else "queue") + self.core.pullManager.addEvent(e) + + @lock + @change + def restartFile(self, id): + """ restart file""" + if id in self.cache: + self.cache[id].status = 3 + self.cache[id].name = self.cache[id].url + self.cache[id].error = "" + self.cache[id].abortDownload() + + + self.db.restartFile(id) + + e = UpdateEvent("file", id, "collector" if not self.getFile(id).package().queue else "queue") + self.core.pullManager.addEvent(e) + + @lock + @change + def setPackageLocation(self, id, queue): + """push package to queue""" + + p = self.db.getPackage(id) + oldorder = p.order + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + self.db.clearPackageOrder(p) + + p = self.db.getPackage(id) + + p.queue = queue + self.db.updatePackage(p) + + self.db.reorderPackage(p, -1, True) + + packs = self.packageCache.values() + for pack in packs: + if pack.queue != queue and pack.order > oldorder: + pack.order -= 1 + pack.notifyChange() + + self.db.commit() + self.releasePackage(id) + p = self.getPackage(id) + + e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + @lock + @change + def reorderPackage(self, id, position): + p = self.getPackage(id) + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + self.db.reorderPackage(p, position) + + packs = self.packageCache.values() + for pack in packs: + if pack.queue != p.queue or pack.order < 0 or pack == p: continue + if p.order > position: + if pack.order >= position and pack.order < p.order: + pack.order += 1 + pack.notifyChange() + elif p.order < position: + if pack.order <= position and pack.order > p.order: + pack.order -= 1 + pack.notifyChange() + + p.order = position + self.db.commit() + + e = InsertEvent("pack", id, position, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + @lock + @change + def reorderFile(self, id, position): + f = self.getFileData(id) + f = f[id] + + e = RemoveEvent("file", id, "collector" if not self.getPackage(f["package"]).queue else "queue") + self.core.pullManager.addEvent(e) + + self.db.reorderLink(f, position) + + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid != f["package"] or pyfile.order < 0: continue + if f["order"] > position: + if pyfile.order >= position and pyfile.order < f["order"]: + pyfile.order += 1 + pyfile.notifyChange() + elif f["order"] < position: + if pyfile.order <= position and pyfile.order > f["order"]: + pyfile.order -= 1 + pyfile.notifyChange() + + if id in self.cache: + self.cache[id].order = position + + self.db.commit() + + e = InsertEvent("file", id, position, "collector" if not self.getPackage(f["package"]).queue else "queue") + self.core.pullManager.addEvent(e) + + @change + def updateFileInfo(self, data, pid): + """ updates file info (name, size, status, url)""" + ids = self.db.updateLinkInfo(data) + e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue") + self.core.pullManager.addEvent(e) + + def checkPackageFinished(self, pyfile): + """ checks if package is finished and calls AddonManager """ + + ids = self.db.getUnfinished(pyfile.packageid) + if not ids or (pyfile.id in ids and len(ids) == 1): + if not pyfile.package().setFinished: + self.core.log.info(_("Package finished: %s") % pyfile.package().name) + self.core.addonManager.packageFinished(pyfile.package()) + pyfile.package().setFinished = True + + + def reCheckPackage(self, pid): + """ recheck links in package """ + data = self.db.getPackageData(pid) + + urls = [] + + for pyfile in data.itervalues(): + if pyfile["status"] not in (0, 12, 13): + urls.append((pyfile["url"], pyfile["plugin"])) + + self.core.threadManager.createInfoThread(urls, pid) + + @lock + @change + def deleteFinishedLinks(self): + """ deletes finished links and packages, return deleted packages """ + + old_packs = self.getInfoData(0) + old_packs.update(self.getInfoData(1)) + + self.db.deleteFinished() + + new_packs = self.db.getAllPackages(0) + new_packs.update(self.db.getAllPackages(1)) + #get new packages only from db + + deleted = [] + for id in old_packs.iterkeys(): + if id not in new_packs: + deleted.append(id) + self.deletePackage(int(id)) + + return deleted + + @lock + @change + def restartFailed(self): + """ restart all failed links """ + self.db.restartFailed() + +class FileMethods: + @style.queue + def filecount(self, queue): + """returns number of files in queue""" + self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue,)) + return self.c.fetchone()[0] + + @style.queue + def queuecount(self, queue): + """ number of files in queue not finished yet""" + self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0, 4)", (queue,)) + return self.c.fetchone()[0] + + @style.queue + def processcount(self, queue, fid): + """ number of files which have to be proccessed """ + self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2, 3, 5, 7, 12) AND l.id != ?", (queue, str(fid))) + return self.c.fetchone()[0] + + @style.inner + def _nextPackageOrder(self, queue=0): + self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) + max = self.c.fetchone()[0] + if max is not None: + return max + 1 + else: + return 0 + + @style.inner + def _nextFileOrder(self, package): + self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) + max = self.c.fetchone()[0] + if max is not None: + return max + 1 + else: + return 0 + + @style.queue + def addLink(self, url, name, plugin, package): + order = self._nextFileOrder(package) + self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', (url, name, (plugintype, pluginname), package, order)) + return self.c.lastrowid + + @style.queue + def addLinks(self, links, package): + """ links is a list of tupels (url, plugin)""" + order = self._nextFileOrder(package) + orders = [order + x for x in range(len(links))] + links = [(x[0], x[0], (x[1], x[2]), package, o) for x, o in zip(links, orders)] + self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) + + @style.queue + def addPackage(self, name, folder, queue): + order = self._nextPackageOrder(queue) + self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', (name, folder, queue, order)) + return self.c.lastrowid + + @style.queue + def deletePackage(self, p): + + self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) + self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue)) + + @style.queue + def deleteLink(self, f): + + self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) + self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid))) + + + @style.queue + def getAllLinks(self, q): + """return information about all links in queue q + + q0 queue + q1 collector + + format: + + { + id: {'name': name, ... 'package': id }, ... + } + + """ + self.c.execute('SELECT l.id, l.url, l.name, l.size, l.status, l.error, l.plugin, l.package, l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', (q,)) + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': r[6], + 'package': r[7], + 'order': r[8], + } + + return data + + @style.queue + def getAllPackages(self, q): + """return information about packages in queue q + (only useful in get all data) + + q0 queue + q1 collector + + format: + + { + id: {'name': name ... 'links': {}}, ... + } + """ + self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ + FROM packages p JOIN pstats s ON p.id = s.id \ + WHERE p.queue=? ORDER BY p.packageorder', str(q)) + + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'name': r[1], + 'folder': r[2], + 'site': r[3], + 'password': r[4], + 'queue': r[5], + 'order': r[6], + 'sizetotal': int(r[7]), + 'sizedone': r[8] if r[8] else 0, #these can be None + 'linksdone': r[9] if r[9] else 0, + 'linkstotal': r[10], + 'links': {} + } + + return data + + @style.queue + def getLinkData(self, id): + """get link information as dict""" + self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?', (str(id),)) + data = {} + r = self.c.fetchone() + if not r: + return None + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': r[6], + 'package': r[7], + 'order': r[8], + } + + return data + + @style.queue + def getPackageData(self, id): + """get data about links for a package""" + self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id),)) + + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': r[6], + 'package': r[7], + 'order': r[8], + } + + return data + + + @style.async + def updateLink(self, f): + self.c.execute('UPDATE links SET url=?, name=?, size=?, status=?, error=?, package=? WHERE id=?', (f.url, f.name, f.size, f.status, f.error, str(f.packageid), str(f.id))) + + @style.queue + def updatePackage(self, p): + self.c.execute('UPDATE packages SET name=?, folder=?, site=?, password=?, queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) + + @style.queue + def updateLinkInfo(self, data): + """ data is list of tupels (name, size, status, url) """ + self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1, 2, 3, 14)', data) + ids = [] + self.c.execute('SELECT id FROM links WHERE url IN (\'%s\')' % "','".join([x[3] for x in data])) + for r in self.c: + ids.append(int(r[0])) + return ids + + @style.queue + def reorderPackage(self, p, position, noMove=False): + if position == -1: + position = self._nextPackageOrder(p.queue) + if not noMove: + if p.order > position: + self.c.execute('UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND queue=? AND packageorder >= 0', (position, p.order, p.queue)) + elif p.order < position: + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND queue=? AND packageorder >= 0', (position, p.order, p.queue)) + + self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) + + @style.queue + def reorderLink(self, f, position): + """ reorder link with f as dict for pyfile """ + if f["order"] > position: + self.c.execute('UPDATE links SET linkorder=linkorder+1 WHERE linkorder >= ? AND linkorder < ? AND package=?', (position, f["order"], f["package"])) + elif f["order"] < position: + self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder <= ? AND linkorder > ? AND package=?', (position, f["order"], f["package"])) + + self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"])) + + @style.queue + def clearPackageOrder(self, p): + self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) + + @style.async + def restartFile(self, id): + self.c.execute('UPDATE links SET status=3, error="" WHERE id=?', (str(id),)) + + @style.async + def restartPackage(self, id): + self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) + + @style.queue + def getPackage(self, id): + """return package instance from id""" + self.c.execute("SELECT name, folder, site, password, queue, packageorder FROM packages WHERE id=?", (str(id),)) + r = self.c.fetchone() + if not r: return None + return PyPackage(self.manager, id, * r) + + #-------------------------------------------------------------------------- + @style.queue + def getFile(self, id): + """return link instance from id""" + self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id),)) + r = self.c.fetchone() + if not r: return None + return PyFile(self.manager, id, * r) + + + @style.queue + def getJob(self, occ): + """return pyfile ids, which are suitable for download and dont use a occupied plugin""" + + #@TODO improve this hardcoded method + pre = "('DLC', 'LinkList', 'SerienjunkiesOrg', 'CCF', 'RSDF')" #plugins which are processed in collector + + cmd = "(" + for i, item in enumerate(occ): + if i: cmd += ", " + cmd += "'%s'" % item + + cmd += ")" + + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) + + self.c.execute(cmd) # very bad! + + return [x[0] for x in self.c] + + @style.queue + def getPluginJob(self, plugins): + """returns pyfile ids with suited plugins""" + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins + + self.c.execute(cmd) # very bad! + + return [x[0] for x in self.c] + + @style.queue + def getUnfinished(self, pid): + """return list of max length 3 ids with pyfiles in package not finished or processed""" + + self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) + return [r[0] for r in self.c] + + @style.queue + def deleteFinished(self): + self.c.execute("DELETE FROM links WHERE status IN (0, 4)") + self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") + + @style.queue + def restartFailed(self): + self.c.execute("UPDATE links SET status=3, error='' WHERE status IN (6, 8, 9)") + + @style.queue + def findDuplicates(self, id, folder, filename): + """ checks if filename exists with different id and same package """ + self.c.execute("SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", (folder, id, filename)) + return self.c.fetchone() + + @style.queue + def purgeLinks(self): + self.c.execute("DELETE FROM links;") + self.c.execute("DELETE FROM packages;") + +DatabaseBackend.registerSub(FileMethods) diff --git a/pyload/database/StorageDatabase.py b/pyload/database/StorageDatabase.py new file mode 100644 index 000000000..c2473e7b7 --- /dev/null +++ b/pyload/database/StorageDatabase.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: mkaay +""" + +from pyload.database import style +from pyload.database import DatabaseBackend + +class StorageMethods: + @style.queue + def setStorage(db, identifier, key, value): + db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) + if db.c.fetchone() is not None: + db.c.execute("UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key)) + else: + db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) + + @style.queue + def getStorage(db, identifier, key=None): + if key is not None: + db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) + row = db.c.fetchone() + if row is not None: + return row[0] + else: + db.c.execute("SELECT key, value FROM storage WHERE identifier=?", (identifier,)) + d = {} + for row in db.c: + d[row[0]] = row[1] + return d + + @style.queue + def delStorage(db, identifier, key): + db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) + +DatabaseBackend.registerSub(StorageMethods) diff --git a/pyload/database/UserDatabase.py b/pyload/database/UserDatabase.py new file mode 100644 index 000000000..59b0f6dbf --- /dev/null +++ b/pyload/database/UserDatabase.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: mkaay +""" + +from hashlib import sha1 +import random + +from DatabaseBackend import DatabaseBackend +from DatabaseBackend import style + +class UserMethods: + @style.queue + def checkAuth(db, user, password): + c = db.c + c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user,)) + r = c.fetchone() + if not r: + return {} + + salt = r[2][:5] + pw = r[2][5:] + h = sha1(salt + password) + if h.hexdigest() == pw: + return {"id": r[0], "name": r[1], "role": r[3], + "permission": r[4], "template": r[5], "email": r[6]} + else: + return {} + + @style.queue + def addUser(db, user, password): + salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) + h = sha1(salt + password) + password = salt + h.hexdigest() + + c = db.c + c.execute('SELECT name FROM users WHERE name=?', (user,)) + if c.fetchone() is not None: + c.execute('UPDATE users SET password=? WHERE name=?', (password, user)) + else: + c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) + + + @style.queue + def changePassword(db, user, oldpw, newpw): + db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user,)) + r = db.c.fetchone() + if not r: + return False + + salt = r[2][:5] + pw = r[2][5:] + h = sha1(salt + oldpw) + if h.hexdigest() == pw: + salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) + h = sha1(salt + newpw) + password = salt + h.hexdigest() + + db.c.execute("UPDATE users SET password=? WHERE name=?", (password, user)) + return True + + return False + + + @style.async + def setPermission(db, user, perms): + db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) + + @style.async + def setRole(db, user, role): + db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) + + + @style.queue + def listUsers(db): + db.c.execute('SELECT name FROM users') + users = [] + for row in db.c: + users.append(row[0]) + return users + + @style.queue + def getAllUserData(db): + db.c.execute("SELECT name, permission, role, template, email FROM users") + user = {} + for r in db.c: + user[r[0]] = {"permission": r[1], "role": r[2], "template": r[3], "email": r[4]} + + return user + + @style.queue + def removeUser(db, user): + db.c.execute('DELETE FROM users WHERE name=?', (user,)) + +DatabaseBackend.registerSub(UserMethods) diff --git a/pyload/database/__init__.py b/pyload/database/__init__.py new file mode 100644 index 000000000..5f287a47f --- /dev/null +++ b/pyload/database/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +from DatabaseBackend import DatabaseBackend +from DatabaseBackend import style + +from FileDatabase import FileHandler +from UserDatabase import UserMethods +from StorageDatabase import StorageMethods |