diff options
Diffstat (limited to 'module/database')
| -rw-r--r-- | module/database/AccountDatabase.py | 21 | ||||
| -rw-r--r-- | module/database/DatabaseBackend.py | 232 | ||||
| -rw-r--r-- | module/database/FileDatabase.py | 224 | ||||
| -rw-r--r-- | module/database/StorageDatabase.py | 9 | ||||
| -rw-r--r-- | module/database/UserDatabase.py | 23 | ||||
| -rw-r--r-- | module/database/__init__.py | 6 |
6 files changed, 225 insertions, 290 deletions
diff --git a/module/database/AccountDatabase.py b/module/database/AccountDatabase.py new file mode 100644 index 000000000..1602451fa --- /dev/null +++ b/module/database/AccountDatabase.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +from module.database import queue, async +from module.database import DatabaseBackend + +class AccountMethods: + + @queue + def loadAccounts(db): + db.c.execute('SELECT plugin, loginname, activated, password, options FROM accounts;') + return db.c.fetchall() + + @async + def saveAccounts(db, data): + db.c.executemany('INSERT INTO accounts(plugin, loginname, activated, password, options) VALUES(?,?,?,?,?)', data) + + @async + def removeAccount(db, plugin, loginname): + db.c.execute('DELETE FROM accounts WHERE plugin=? AND loginname=?', (plugin, loginname)) + +DatabaseBackend.registerSub(AccountMethods)
\ No newline at end of file diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 9530390c3..32f75328c 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -25,62 +25,65 @@ from shutil import move from Queue import Queue from traceback import print_exc -from module.utils import chmod +from module.utils.fs import chmod try: from pysqlite2 import dbapi2 as sqlite3 except: import sqlite3 +DB = None DB_VERSION = 4 -class style(): - db = None - - @classmethod - def setDB(cls, db): - cls.db = db - - @classmethod - def inner(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return f(cls.db, *args, **kwargs) - return x - - @classmethod - def queue(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.queue(f, *args, **kwargs) - return x - - @classmethod - def async(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.async(f, *args, **kwargs) - return x +def set_DB(db): + global DB + DB = db + + +def queue(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return DB.queue(f, *args, **kwargs) + + return x + + +def async(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return DB.async(f, *args, **kwargs) + + return x + + +def inner(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return f(DB, *args, **kwargs) + + return x + class DatabaseJob(): def __init__(self, f, *args, **kwargs): self.done = Event() - + self.f = f self.args = args self.kwargs = kwargs - + self.result = None self.exception = False -# import inspect -# self.frame = inspect.currentframe() + # import inspect + # self.frame = inspect.currentframe() def __repr__(self): from os.path import basename + frame = self.frame.f_back output = "" for i in range(5): @@ -104,46 +107,50 @@ class DatabaseJob(): self.exception = e finally: self.done.set() - + def wait(self): self.done.wait() + class DatabaseBackend(Thread): subs = [] + + DB_FILE = "pyload.db" + VERSION_FILE = "db.version" + def __init__(self, core): Thread.__init__(self) self.setDaemon(True) self.core = core self.jobs = Queue() - + self.setuplock = Event() - - style.setDB(self) - + + set_DB(self) + def setup(self): self.start() self.setuplock.wait() - + def run(self): """main loop, which executes commands""" convert = self._checkVersion() #returns None or current version - - self.conn = sqlite3.connect("files.db") - chmod("files.db", 0600) + + self.conn = sqlite3.connect(self.DB_FILE) + chmod(self.DB_FILE, 0600) self.c = self.conn.cursor() #compatibility - + if convert is not None: self._convertDB(convert) - + self._createTables() - self._migrateUser() self.conn.commit() - + self.setuplock.set() - + while True: j = self.jobs.get() if j == "quit": @@ -152,20 +159,23 @@ class DatabaseBackend(Thread): break j.processJob() - @style.queue + @queue def shutdown(self): self.conn.commit() self.jobs.put("quit") def _checkVersion(self): """ check db version and delete it if needed""" - if not exists("files.version"): - f = open("files.version", "wb") + if not exists(self.VERSION_FILE): + f = open(self.VERSION_FILE, "wb") f.write(str(DB_VERSION)) f.close() return - - f = open("files.version", "rb") + + if exists("files.db") and not exists(self.DB_FILE): + move("files.db", self.DB_FILE) + + f = open(self.VERSION_FILE, "rb") v = int(f.read().strip()) f.close() if v < DB_VERSION: @@ -174,13 +184,13 @@ class DatabaseBackend(Thread): self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) except: print "Filedatabase was deleted due to incompatible version." - remove("files.version") - move("files.db", "files.backup.db") - f = open("files.version", "wb") + remove(self.VERSION_FILE) + move(self.DB_FILE, self.DB_FILE + ".backup") + f = open(self.VERSION_FILE, "wb") f.write(str(DB_VERSION)) f.close() return v - + def _convertDB(self, v): try: getattr(self, "_convertV%i" % v)() @@ -189,34 +199,28 @@ class DatabaseBackend(Thread): self.core.log.error(_("Filedatabase could NOT be converted.")) except: print "Filedatabase could NOT be converted." - + #--convert scripts start - - def _convertV2(self): - self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') - try: - self.manager.core.log.info(_("Database was converted from v2 to v3.")) - except: - print "Database was converted from v2 to v3." - self._convertV3() - - def _convertV3(self): - self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') - try: - self.manager.core.log.info(_("Database was converted from v3 to v4.")) - except: - print "Database was converted from v3 to v4." - + + def _convertV4(self): + pass + #--convert scripts end - + def _createTables(self): """create tables for database""" - self.c.execute('CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') - self.c.execute('CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)') - self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') - self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "users" ("name" TEXT PRIMARY KEY NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "accounts" ("plugin" TEXT NOT NULL, "loginname" TEXT NOT NULL, "activated" INTEGER DEFAULT 1, "password" TEXT DEFAULT "", "options" TEXT DEFAULT "", PRIMARY KEY (plugin, loginname) ON CONFLICT REPLACE)') self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \ SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\ @@ -234,7 +238,6 @@ class DatabaseBackend(Thread): fid = 0 self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links")) - self.c.execute('SELECT max(id) FROM packages') pid = self.c.fetchone()[0] if pid: @@ -246,60 +249,41 @@ class DatabaseBackend(Thread): self.c.execute('VACUUM') - def _migrateUser(self): - if exists("pyload.db"): - try: - self.core.log.info(_("Converting old Django DB")) - except: - print "Converting old Django DB" - conn = sqlite3.connect('pyload.db') - c = conn.cursor() - c.execute("SELECT username, password, email from auth_user WHERE is_superuser") - users = [] - for r in c: - pw = r[1].split("$") - users.append((r[0], pw[1] + pw[2], r[2])) - c.close() - conn.close() - - self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users) - move("pyload.db", "pyload.old.db") - def createCursor(self): return self.conn.cursor() - - @style.async + + @async def commit(self): self.conn.commit() - @style.queue + @queue def syncSave(self): self.conn.commit() - - @style.async + + @async def rollback(self): self.conn.rollback() - + def async(self, f, *args, **kwargs): args = (self, ) + args job = DatabaseJob(f, *args, **kwargs) self.jobs.put(job) - + def queue(self, f, *args, **kwargs): args = (self, ) + args job = DatabaseJob(f, *args, **kwargs) self.jobs.put(job) job.wait() return job.result - + @classmethod def registerSub(cls, klass): cls.subs.append(klass) - + @classmethod def unregisterSub(cls, klass): cls.subs.remove(klass) - + def __getattr__(self, attr): for sub in DatabaseBackend.subs: if hasattr(sub, attr): @@ -308,45 +292,47 @@ class DatabaseBackend(Thread): if __name__ == "__main__": db = DatabaseBackend() db.setup() - + class Test(): - @style.queue + @queue def insert(db): c = db.createCursor() for i in range(1000): c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - @style.async + + @async def insert2(db): c = db.createCursor() - for i in range(1000*1000): + for i in range(1000 * 1000): c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - - @style.queue + + @queue def select(db): c = db.createCursor() for i in range(10): res = c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", ("foo", i)) print res.fetchone() - - @style.queue + + @queue def error(db): c = db.createCursor() print "a" c.execute("SELECT myerror FROM storage WHERE identifier=? AND key=?", ("foo", i)) print "e" - + db.registerSub(Test) from time import time + start = time() for i in range(100): db.insert() end = time() - print end-start - + print end - start + start = time() db.insert2() end = time() - print end-start - + print end - start + db.error() diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 357cd766d..eb76f468b 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -22,10 +22,9 @@ from threading import RLock from time import time from module.utils import formatSize, lock -from module.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent from module.PyPackage import PyPackage from module.PyFile import PyFile -from module.database import style, DatabaseBackend +from module.database import DatabaseBackend, queue, async, inner try: from pysqlite2 import dbapi2 as sqlite3 @@ -40,11 +39,12 @@ class FileHandler: def __init__(self, core): """Constructor""" self.core = core + self.evm = core.eventManager # translations self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), _("downloading"), _("processing"), _("unknown")] - self.cache = {} #holds instances for files + self.cache = {} # holds instances for files self.packageCache = {} # same for packages #@TODO: purge the cache @@ -54,14 +54,12 @@ class FileHandler: #self.lock._Verbose__verbose = True self.filecount = -1 # if an invalid value is set get current value from db - self.queuecount = -1 #number of package to be loaded - self.unchanged = False #determines if any changes was made since last call + self.queuecount = -1 # number of package to be loaded self.db = self.core.db def change(func): def new(*args): - args[0].unchanged = False args[0].filecount = -1 args[0].queuecount = -1 args[0].jobCache = {} @@ -118,31 +116,23 @@ class FileHandler: @lock @change - def addLinks(self, urls, package): - """adds links""" - - self.core.hookManager.dispatchEvent("linksAdded", urls, package) - - data = self.core.pluginManager.parseUrls(urls) - + def addLinks(self, data, package): + """Add links, data = (plugin, url) tuple. Internal method you should use API.""" self.db.addLinks(data, package) - self.core.threadManager.createInfoThread(data, package) + self.evm.dispatchEvent("packageUpdated", package) - #@TODO change from reloadAll event to package update event - self.core.pullManager.addEvent(ReloadAllEvent("collector")) - #---------------------------------------------------------------------- @lock @change - def addPackage(self, name, folder, queue=0): + def addPackage(self, name, folder, queue=0, password=""): """adds a package, default to link collector""" - lastID = self.db.addPackage(name, folder, queue) - p = self.db.getPackage(lastID) - e = InsertEvent("pack", lastID, p.order, "collector" if not queue else "queue") - self.core.pullManager.addEvent(e) - return lastID + pid = self.db.addPackage(name, folder, queue, password) + p = self.db.getPackage(pid) + + self.evm.dispatchEvent("packageInserted", pid, p.queue, p.order) + return pid + - #---------------------------------------------------------------------- @lock @change def deletePackage(self, id): @@ -156,7 +146,6 @@ class FileHandler: oldorder = p.order queue = p.queue - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") pyfiles = self.cache.values() @@ -166,8 +155,7 @@ class FileHandler: pyfile.release() self.db.deletePackage(p) - self.core.pullManager.addEvent(e) - self.core.hookManager.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageDeleted", id) if id in self.packageCache: del self.packageCache[id] @@ -178,7 +166,7 @@ class FileHandler: pack.order -= 1 pack.notifyChange() - #---------------------------------------------------------------------- + @lock @change def deleteLink(self, id): @@ -189,8 +177,6 @@ class FileHandler: return None pid = f.packageid - e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") - oldorder = f.order if id in self.core.threadManager.processingIds(): @@ -201,11 +187,10 @@ class FileHandler: self.db.deleteLink(f) - self.core.pullManager.addEvent(e) + self.evm.dispatchEvent("linkDeleted", id, pid) p = self.getPackage(pid) - if not len(p.getChildren()): - p.delete() + p.deleteIfEmpty() pyfiles = self.cache.values() for pyfile in pyfiles: @@ -213,35 +198,26 @@ class FileHandler: pyfile.order -= 1 pyfile.notifyChange() - #---------------------------------------------------------------------- def releaseLink(self, id): """removes pyfile from cache""" if id in self.cache: del self.cache[id] - #---------------------------------------------------------------------- def releasePackage(self, id): """removes package from cache""" if id in self.packageCache: del self.packageCache[id] - #---------------------------------------------------------------------- def updateLink(self, pyfile): """updates link""" self.db.updateLink(pyfile) + self.evm.dispatchEvent("linkUpdated", pyfile.id, pyfile.packageid) - e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") - self.core.pullManager.addEvent(e) - - #---------------------------------------------------------------------- def updatePackage(self, pypack): """updates a package""" self.db.updatePackage(pypack) + self.evm.dispatchEvent("packageUpdated", pypack.id) - e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") - self.core.pullManager.addEvent(e) - - #---------------------------------------------------------------------- def getPackage(self, id): """return package instance""" @@ -250,7 +226,6 @@ class FileHandler: else: return self.db.getPackage(id) - #---------------------------------------------------------------------- def getPackageData(self, id): """returns dict with package information""" pack = self.getPackage(id) @@ -274,7 +249,7 @@ class FileHandler: return pack - #---------------------------------------------------------------------- + def getFileData(self, id): """returns dict with file information""" if id in self.cache: @@ -282,7 +257,7 @@ class FileHandler: return self.db.getLinkData(id) - #---------------------------------------------------------------------- + def getFile(self, id): """returns pyfile instance""" if id in self.cache: @@ -290,7 +265,7 @@ class FileHandler: else: return self.db.getFile(id) - #---------------------------------------------------------------------- + @lock def getJob(self, occ): """get suitable job""" @@ -334,21 +309,6 @@ class FileHandler: #pyfile = self.getFile(self.jobCache[occ].pop()) return pyfile - @lock - def getDecryptJob(self): - """return job for decrypting""" - if "decrypt" in self.jobCache: - return None - - plugins = self.core.pluginManager.crypterPlugins.keys() + self.core.pluginManager.containerPlugins.keys() - plugins = str(tuple(plugins)) - - jobs = self.db.getPluginJob(plugins) - if jobs: - return self.getFile(jobs[0]) - else: - self.jobCache["decrypt"] = "empty" - return None def getFileCount(self): """returns number of files""" @@ -405,8 +365,7 @@ class FileHandler: if id in self.packageCache: self.packageCache[id].setFinished = False - e = UpdateEvent("pack", id, "collector" if not self.getPackage(id).queue else "queue") - self.core.pullManager.addEvent(e) + self.evm.dispatchEvent("packageUpdated", id) @lock @change @@ -420,9 +379,8 @@ class FileHandler: self.db.restartFile(id) + self.evm.dispatchEvent("linkUpdated", id) - e = UpdateEvent("file", id, "collector" if not self.getFile(id).package().queue else "queue") - self.core.pullManager.addEvent(e) @lock @change @@ -431,17 +389,10 @@ class FileHandler: p = self.db.getPackage(id) oldorder = p.order + p.queue = queue - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - self.db.clearPackageOrder(p) - - p = self.db.getPackage(id) - - p.queue = queue self.db.updatePackage(p) - self.db.reorderPackage(p, -1, True) packs = self.packageCache.values() @@ -452,37 +403,34 @@ class FileHandler: self.db.commit() self.releasePackage(id) - p = self.getPackage(id) - - e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) + + self.evm.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change def reorderPackage(self, id, position): p = self.getPackage(id) - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) self.db.reorderPackage(p, position) packs = self.packageCache.values() for pack in packs: if pack.queue != p.queue or pack.order < 0 or pack == p: continue if p.order > position: - if pack.order >= position and pack.order < p.order: + if position <= pack.order < p.order: pack.order += 1 pack.notifyChange() elif p.order < position: - if pack.order <= position and pack.order > p.order: + if position >= pack.order > p.order: pack.order -= 1 pack.notifyChange() p.order = position self.db.commit() - e = InsertEvent("pack", id, position, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) + self.evm.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change @@ -490,20 +438,17 @@ class FileHandler: f = self.getFileData(id) f = f[id] - e = RemoveEvent("file", id, "collector" if not self.getPackage(f["package"]).queue else "queue") - self.core.pullManager.addEvent(e) - self.db.reorderLink(f, position) pyfiles = self.cache.values() for pyfile in pyfiles: if pyfile.packageid != f["package"] or pyfile.order < 0: continue if f["order"] > position: - if pyfile.order >= position and pyfile.order < f["order"]: + if position <= pyfile.order < f["order"]: pyfile.order += 1 pyfile.notifyChange() elif f["order"] < position: - if pyfile.order <= position and pyfile.order > f["order"]: + if position >= pyfile.order > f["order"]: pyfile.order -= 1 pyfile.notifyChange() @@ -512,15 +457,14 @@ class FileHandler: self.db.commit() - e = InsertEvent("file", id, position, "collector" if not self.getPackage(f["package"]).queue else "queue") - self.core.pullManager.addEvent(e) + self.evm.dispatchEvent("packageUpdated", f["package"]) + @change def updateFileInfo(self, data, pid): """ updates file info (name, size, status, url)""" ids = self.db.updateLinkInfo(data) - e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue") - self.core.pullManager.addEvent(e) + self.evm.dispatchEvent("packageUpdated", pid) def checkPackageFinished(self, pyfile): """ checks if package is finished and calls hookmanager """ @@ -574,25 +518,25 @@ class FileHandler: self.db.restartFailed() class FileMethods(): - @style.queue + @queue def filecount(self, queue): """returns number of files in queue""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue, )) return self.c.fetchone()[0] - @style.queue + @queue def queuecount(self, queue): """ number of files in queue not finished yet""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0,4)", (queue, )) return self.c.fetchone()[0] - @style.queue + @queue def processcount(self, queue, fid): """ number of files which have to be proccessed """ self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2,3,5,7,12) AND l.id != ?", (queue, str(fid))) return self.c.fetchone()[0] - @style.inner + @inner def _nextPackageOrder(self, queue=0): self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) max = self.c.fetchone()[0] @@ -601,7 +545,7 @@ class FileMethods(): else: return 0 - @style.inner + @inner def _nextFileOrder(self, package): self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) max = self.c.fetchone()[0] @@ -610,13 +554,13 @@ class FileMethods(): else: return 0 - @style.queue + @queue def addLink(self, url, name, plugin, package): order = self._nextFileOrder(package) self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', (url, name, plugin, package, order)) return self.c.lastrowid - @style.queue + @queue def addLinks(self, links, package): """ links is a list of tupels (url,plugin)""" order = self._nextFileOrder(package) @@ -624,27 +568,27 @@ class FileMethods(): links = [(x[0], x[0], x[1], package, o) for x, o in zip(links, orders)] self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) - @style.queue - def addPackage(self, name, folder, queue): + @queue + def addPackage(self, name, folder, queue, password): order = self._nextPackageOrder(queue) - self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', (name, folder, queue, order)) + self.c.execute('INSERT INTO packages(name, folder, queue, packageorder, password) VALUES(?,?,?,?,?)', (name, folder, queue, order, password)) return self.c.lastrowid - @style.queue + @queue def deletePackage(self, p): self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue)) - @style.queue + @queue def deleteLink(self, f): self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid))) - @style.queue + @queue def getAllLinks(self, q): """return information about all links in queue q @@ -677,7 +621,7 @@ class FileMethods(): return data - @style.queue + @queue def getAllPackages(self, q): """return information about packages in queue q (only useful in get all data) @@ -692,7 +636,7 @@ class FileMethods(): } """ self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ - FROM packages p JOIN pstats s ON p.id = s.id \ + FROM packages p LEFT OUTER JOIN pstats s ON p.id = s.id \ WHERE p.queue=? ORDER BY p.packageorder', str(q)) data = {} @@ -705,16 +649,16 @@ class FileMethods(): 'password': r[4], 'queue': r[5], 'order': r[6], - 'sizetotal': int(r[7]), - 'sizedone': r[8] if r[8] else 0, #these can be None + 'sizetotal': int(r[7]) if r[7] else 0, + 'sizedone': int(r[8]) if r[8] else 0, #these can be None 'linksdone': r[9] if r[9] else 0, - 'linkstotal': r[10], + 'linkstotal': r[10] if r[10] else 0, 'links': {} } return data - @style.queue + @queue def getLinkData(self, id): """get link information as dict""" self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE id=?', (str(id), )) @@ -738,7 +682,7 @@ class FileMethods(): return data - @style.queue + @queue def getPackageData(self, id): """get data about links for a package""" self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id), )) @@ -762,15 +706,15 @@ class FileMethods(): return data - @style.async + @async def updateLink(self, f): self.c.execute('UPDATE links SET url=?,name=?,size=?,status=?,error=?,package=? WHERE id=?', (f.url, f.name, f.size, f.status, f.error, str(f.packageid), str(f.id))) - @style.queue + @queue def updatePackage(self, p): self.c.execute('UPDATE packages SET name=?,folder=?,site=?,password=?,queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) - @style.queue + @queue def updateLinkInfo(self, data): """ data is list of tupels (name, size, status, url) """ self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1,2,3,14)', data) @@ -780,7 +724,7 @@ class FileMethods(): ids.append(int(r[0])) return ids - @style.queue + @queue def reorderPackage(self, p, position, noMove=False): if position == -1: position = self._nextPackageOrder(p.queue) @@ -792,7 +736,7 @@ class FileMethods(): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) - @style.queue + @queue def reorderLink(self, f, position): """ reorder link with f as dict for pyfile """ if f["order"] > position: @@ -803,20 +747,20 @@ class FileMethods(): self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"])) - @style.queue + @queue def clearPackageOrder(self, p): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) - @style.async + @async def restartFile(self, id): self.c.execute('UPDATE links SET status=3,error="" WHERE id=?', (str(id),)) - @style.async + @async def restartPackage(self, id): self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) - @style.queue + @queue def getPackage(self, id): """return package instance from id""" self.c.execute("SELECT name,folder,site,password,queue,packageorder FROM packages WHERE id=?", (str(id), )) @@ -824,8 +768,8 @@ class FileMethods(): if not r: return None return PyPackage(self.manager, id, * r) - #---------------------------------------------------------------------- - @style.queue + + @queue def getFile(self, id): """return link instance from id""" self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id), )) @@ -834,58 +778,44 @@ class FileMethods(): return PyFile(self.manager, id, * r) - @style.queue + @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" - - #@TODO improve this hardcoded method - pre = "('DLC', 'LinkList', 'SerienjunkiesOrg', 'CCF', 'RSDF')" #plugins which are processed in collector - cmd = "(" for i, item in enumerate(occ): if i: cmd += ", " cmd += "'%s'" % item - - cmd += ")" - - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) - - self.c.execute(cmd) # very bad! - return [x[0] for x in self.c] - - @style.queue - def getPluginJob(self, plugins): - """returns pyfile ids with suited plugins""" - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins + cmd += ")" + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=1 AND l.plugin NOT IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % cmd self.c.execute(cmd) # very bad! return [x[0] for x in self.c] - @style.queue + @queue def getUnfinished(self, pid): """return list of max length 3 ids with pyfiles in package not finished or processed""" self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) return [r[0] for r in self.c] - @style.queue + @queue def deleteFinished(self): self.c.execute("DELETE FROM links WHERE status IN (0,4)") self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") - @style.queue + @queue def restartFailed(self): self.c.execute("UPDATE links SET status=3,error='' WHERE status IN (8, 9)") - @style.queue + @queue def findDuplicates(self, id, folder, filename): """ checks if filename exists with different id and same package """ self.c.execute("SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", (folder, id, filename)) return self.c.fetchone() - @style.queue + @queue def purgeLinks(self): self.c.execute("DELETE FROM links;") self.c.execute("DELETE FROM packages;") diff --git a/module/database/StorageDatabase.py b/module/database/StorageDatabase.py index 3ed29625f..ffaf51763 100644 --- a/module/database/StorageDatabase.py +++ b/module/database/StorageDatabase.py @@ -16,11 +16,10 @@ @author: mkaay """ -from module.database import style -from module.database import DatabaseBackend +from module.database import DatabaseBackend, queue class StorageMethods(): - @style.queue + @queue def setStorage(db, identifier, key, value): db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) if db.c.fetchone() is not None: @@ -28,7 +27,7 @@ class StorageMethods(): else: db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) - @style.queue + @queue def getStorage(db, identifier, key=None): if key is not None: db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) @@ -42,7 +41,7 @@ class StorageMethods(): d[row[0]] = row[1] return d - @style.queue + @queue def delStorage(db, identifier, key): db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) diff --git a/module/database/UserDatabase.py b/module/database/UserDatabase.py index 0c781057d..43fd93df3 100644 --- a/module/database/UserDatabase.py +++ b/module/database/UserDatabase.py @@ -19,14 +19,13 @@ from hashlib import sha1 import random -from DatabaseBackend import DatabaseBackend -from DatabaseBackend import style +from DatabaseBackend import DatabaseBackend, queue, async class UserMethods(): - @style.queue + @queue def checkAuth(db, user, password): c = db.c - c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user, )) + c.execute('SELECT rowid, name, password, role, permission, template, email FROM "users" WHERE name=?', (user, )) r = c.fetchone() if not r: return {} @@ -40,7 +39,7 @@ class UserMethods(): else: return {} - @style.queue + @queue def addUser(db, user, password): salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) h = sha1(salt + password) @@ -54,9 +53,9 @@ class UserMethods(): c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) - @style.queue + @queue def changePassword(db, user, oldpw, newpw): - db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user, )) + db.c.execute('SELECT rowid, name, password FROM users WHERE name=?', (user, )) r = db.c.fetchone() if not r: return False @@ -75,16 +74,16 @@ class UserMethods(): return False - @style.async + @async def setPermission(db, user, perms): db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) - @style.async + @async def setRole(db, user, role): db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) - @style.queue + @queue def listUsers(db): db.c.execute('SELECT name FROM users') users = [] @@ -92,7 +91,7 @@ class UserMethods(): users.append(row[0]) return users - @style.queue + @queue def getAllUserData(db): db.c.execute("SELECT name, permission, role, template, email FROM users") user = {} @@ -101,7 +100,7 @@ class UserMethods(): return user - @style.queue + @queue def removeUser(db, user): db.c.execute('DELETE FROM users WHERE name=?', (user, )) diff --git a/module/database/__init__.py b/module/database/__init__.py index 545789c0c..39848ac58 100644 --- a/module/database/__init__.py +++ b/module/database/__init__.py @@ -1,6 +1,6 @@ -from DatabaseBackend import DatabaseBackend -from DatabaseBackend import style +from DatabaseBackend import DatabaseBackend, queue, async, inner from FileDatabase import FileHandler from UserDatabase import UserMethods -from StorageDatabase import StorageMethods
\ No newline at end of file +from StorageDatabase import StorageMethods +from AccountDatabase import AccountMethods
\ No newline at end of file |