From 5499be89203a18ca61a21cfc7266cf0f4ebe6547 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 15 Dec 2011 23:18:21 +0100 Subject: refractoring --- module/database/FileDatabase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/database') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 357cd766d..895e0de65 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -22,7 +22,7 @@ from threading import RLock from time import time from module.utils import formatSize, lock -from module.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent +from module.interaction.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent from module.PyPackage import PyPackage from module.PyFile import PyFile from module.database import style, DatabaseBackend -- cgit v1.2.3 From 6eae782f13953dd0ba2bbe1b582cf33fd4d7d90a Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 19 Dec 2011 23:10:49 +0100 Subject: configparser v2, warning CONFIG will be DELETED. --- module/database/DatabaseBackend.py | 40 ++++---------------------------------- 1 file changed, 4 insertions(+), 36 deletions(-) (limited to 'module/database') diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 9530390c3..db8b1aa3c 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -138,7 +138,6 @@ class DatabaseBackend(Thread): self._convertDB(convert) self._createTables() - self._migrateUser() self.conn.commit() @@ -191,22 +190,10 @@ class DatabaseBackend(Thread): print "Filedatabase could NOT be converted." #--convert scripts start - - def _convertV2(self): - self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') - try: - self.manager.core.log.info(_("Database was converted from v2 to v3.")) - except: - print "Database was converted from v2 to v3." - self._convertV3() - - def _convertV3(self): - self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') - try: - self.manager.core.log.info(_("Database was converted from v3 to v4.")) - except: - print "Database was converted from v3 to v4." - + + def _convertV4(self): + pass + #--convert scripts end def _createTables(self): @@ -246,25 +233,6 @@ class DatabaseBackend(Thread): self.c.execute('VACUUM') - def _migrateUser(self): - if exists("pyload.db"): - try: - self.core.log.info(_("Converting old Django DB")) - except: - print "Converting old Django DB" - conn = sqlite3.connect('pyload.db') - c = conn.cursor() - c.execute("SELECT username, password, email from auth_user WHERE is_superuser") - users = [] - for r in c: - pw = r[1].split("$") - users.append((r[0], pw[1] + pw[2], r[2])) - c.close() - conn.close() - - self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users) - move("pyload.db", "pyload.old.db") - def createCursor(self): return self.conn.cursor() -- cgit v1.2.3 From c9a359b75905d4910e42127cef8cbe62d4a13657 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 20 Dec 2011 21:35:43 +0100 Subject: some bugfixes --- module/database/FileDatabase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/database') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 895e0de65..4084c46f7 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -340,7 +340,7 @@ class FileHandler: if "decrypt" in self.jobCache: return None - plugins = self.core.pluginManager.crypterPlugins.keys() + self.core.pluginManager.containerPlugins.keys() + plugins = self.core.pluginManager.getPlugins("crypter").keys() + self.core.pluginManager.getPlugins("container").keys() plugins = str(tuple(plugins)) jobs = self.db.getPluginJob(plugins) -- cgit v1.2.3 From 958bf611f5d9d117f19f824990ec6fd6b537e967 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 22 Dec 2011 23:45:38 +0100 Subject: accountmanager v2, delete your accounts.conf and re-enter them in pyload, new nice debug functions, try core.shell() and core.breakpoint() --- module/database/AccountDatabase.py | 21 ++++ module/database/DatabaseBackend.py | 190 ++++++++++++++++++++----------------- module/database/FileDatabase.py | 64 ++++++------- module/database/StorageDatabase.py | 9 +- module/database/UserDatabase.py | 19 ++-- module/database/__init__.py | 6 +- 6 files changed, 173 insertions(+), 136 deletions(-) create mode 100644 module/database/AccountDatabase.py (limited to 'module/database') diff --git a/module/database/AccountDatabase.py b/module/database/AccountDatabase.py new file mode 100644 index 000000000..1602451fa --- /dev/null +++ b/module/database/AccountDatabase.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +from module.database import queue, async +from module.database import DatabaseBackend + +class AccountMethods: + + @queue + def loadAccounts(db): + db.c.execute('SELECT plugin, loginname, activated, password, options FROM accounts;') + return db.c.fetchall() + + @async + def saveAccounts(db, data): + db.c.executemany('INSERT INTO accounts(plugin, loginname, activated, password, options) VALUES(?,?,?,?,?)', data) + + @async + def removeAccount(db, plugin, loginname): + db.c.execute('DELETE FROM accounts WHERE plugin=? AND loginname=?', (plugin, loginname)) + +DatabaseBackend.registerSub(AccountMethods) \ No newline at end of file diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index db8b1aa3c..e10bcbbaf 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -32,55 +32,58 @@ try: except: import sqlite3 +DB = None DB_VERSION = 4 -class style(): - db = None - - @classmethod - def setDB(cls, db): - cls.db = db - - @classmethod - def inner(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return f(cls.db, *args, **kwargs) - return x - - @classmethod - def queue(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.queue(f, *args, **kwargs) - return x - - @classmethod - def async(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.async(f, *args, **kwargs) - return x +def set_DB(db): + global DB + DB = db + + +def queue(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return DB.queue(f, *args, **kwargs) + + return x + + +def async(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return DB.async(f, *args, **kwargs) + + return x + + +def inner(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return f(DB, *args, **kwargs) + + return x + class DatabaseJob(): def __init__(self, f, *args, **kwargs): self.done = Event() - + self.f = f self.args = args self.kwargs = kwargs - + self.result = None self.exception = False -# import inspect -# self.frame = inspect.currentframe() + # import inspect + # self.frame = inspect.currentframe() def __repr__(self): from os.path import basename + frame = self.frame.f_back output = "" for i in range(5): @@ -104,45 +107,50 @@ class DatabaseJob(): self.exception = e finally: self.done.set() - + def wait(self): self.done.wait() + class DatabaseBackend(Thread): subs = [] + + DB_FILE = "pyload.db" + VERSION_FILE = "db.version" + def __init__(self, core): Thread.__init__(self) self.setDaemon(True) self.core = core self.jobs = Queue() - + self.setuplock = Event() - - style.setDB(self) - + + set_DB(self) + def setup(self): self.start() self.setuplock.wait() - + def run(self): """main loop, which executes commands""" convert = self._checkVersion() #returns None or current version - - self.conn = sqlite3.connect("files.db") - chmod("files.db", 0600) + + self.conn = sqlite3.connect(self.DB_FILE) + chmod(self.DB_FILE, 0600) self.c = self.conn.cursor() #compatibility - + if convert is not None: self._convertDB(convert) - + self._createTables() self.conn.commit() - + self.setuplock.set() - + while True: j = self.jobs.get() if j == "quit": @@ -151,20 +159,23 @@ class DatabaseBackend(Thread): break j.processJob() - @style.queue + @queue def shutdown(self): self.conn.commit() self.jobs.put("quit") def _checkVersion(self): """ check db version and delete it if needed""" - if not exists("files.version"): - f = open("files.version", "wb") + if not exists(self.VERSION_FILE): + f = open(self.VERSION_FILE, "wb") f.write(str(DB_VERSION)) f.close() return - - f = open("files.version", "rb") + + if exists("files.db") and not exists(self.DB_FILE): + move("files.db", self.DB_FILE) + + f = open(self.VERSION_FILE, "rb") v = int(f.read().strip()) f.close() if v < DB_VERSION: @@ -173,13 +184,13 @@ class DatabaseBackend(Thread): self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) except: print "Filedatabase was deleted due to incompatible version." - remove("files.version") - move("files.db", "files.backup.db") - f = open("files.version", "wb") + remove(self.VERSION_FILE) + move(self.DB_FILE, self.DB_FILE + ".backup") + f = open(self.VERSION_FILE, "wb") f.write(str(DB_VERSION)) f.close() return v - + def _convertDB(self, v): try: getattr(self, "_convertV%i" % v)() @@ -188,22 +199,28 @@ class DatabaseBackend(Thread): self.core.log.error(_("Filedatabase could NOT be converted.")) except: print "Filedatabase could NOT be converted." - + #--convert scripts start def _convertV4(self): pass #--convert scripts end - + def _createTables(self): """create tables for database""" - self.c.execute('CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') - self.c.execute('CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)') - self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') - self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "users" ("name" TEXT PRIMARY KEY NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "accounts" ("plugin" TEXT NOT NULL, "loginname" TEXT NOT NULL, "activated" INTEGER DEFAULT 1, "password" TEXT DEFAULT "", "options" TEXT DEFAULT "", PRIMARY KEY (plugin, loginname) ON CONFLICT REPLACE)') self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \ SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\ @@ -221,7 +238,6 @@ class DatabaseBackend(Thread): fid = 0 self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links")) - self.c.execute('SELECT max(id) FROM packages') pid = self.c.fetchone()[0] if pid: @@ -235,39 +251,39 @@ class DatabaseBackend(Thread): def createCursor(self): return self.conn.cursor() - - @style.async + + @async def commit(self): self.conn.commit() - @style.queue + @queue def syncSave(self): self.conn.commit() - - @style.async + + @async def rollback(self): self.conn.rollback() - + def async(self, f, *args, **kwargs): args = (self, ) + args job = DatabaseJob(f, *args, **kwargs) self.jobs.put(job) - + def queue(self, f, *args, **kwargs): args = (self, ) + args job = DatabaseJob(f, *args, **kwargs) self.jobs.put(job) job.wait() return job.result - + @classmethod def registerSub(cls, klass): cls.subs.append(klass) - + @classmethod def unregisterSub(cls, klass): cls.subs.remove(klass) - + def __getattr__(self, attr): for sub in DatabaseBackend.subs: if hasattr(sub, attr): @@ -276,45 +292,47 @@ class DatabaseBackend(Thread): if __name__ == "__main__": db = DatabaseBackend() db.setup() - + class Test(): - @style.queue + @queue def insert(db): c = db.createCursor() for i in range(1000): c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - @style.async + + @async def insert2(db): c = db.createCursor() - for i in range(1000*1000): + for i in range(1000 * 1000): c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - - @style.queue + + @queue def select(db): c = db.createCursor() for i in range(10): res = c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", ("foo", i)) print res.fetchone() - - @style.queue + + @queue def error(db): c = db.createCursor() print "a" c.execute("SELECT myerror FROM storage WHERE identifier=? AND key=?", ("foo", i)) print "e" - + db.registerSub(Test) from time import time + start = time() for i in range(100): db.insert() end = time() - print end-start - + print end - start + start = time() db.insert2() end = time() - print end-start - + print end - start + db.error() diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 4084c46f7..b5c386802 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -25,7 +25,7 @@ from module.utils import formatSize, lock from module.interaction.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent from module.PyPackage import PyPackage from module.PyFile import PyFile -from module.database import style, DatabaseBackend +from module.database import DatabaseBackend, queue, async, inner try: from pysqlite2 import dbapi2 as sqlite3 @@ -574,25 +574,25 @@ class FileHandler: self.db.restartFailed() class FileMethods(): - @style.queue + @queue def filecount(self, queue): """returns number of files in queue""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue, )) return self.c.fetchone()[0] - @style.queue + @queue def queuecount(self, queue): """ number of files in queue not finished yet""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0,4)", (queue, )) return self.c.fetchone()[0] - @style.queue + @queue def processcount(self, queue, fid): """ number of files which have to be proccessed """ self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2,3,5,7,12) AND l.id != ?", (queue, str(fid))) return self.c.fetchone()[0] - @style.inner + @inner def _nextPackageOrder(self, queue=0): self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) max = self.c.fetchone()[0] @@ -601,7 +601,7 @@ class FileMethods(): else: return 0 - @style.inner + @inner def _nextFileOrder(self, package): self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) max = self.c.fetchone()[0] @@ -610,13 +610,13 @@ class FileMethods(): else: return 0 - @style.queue + @queue def addLink(self, url, name, plugin, package): order = self._nextFileOrder(package) self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', (url, name, plugin, package, order)) return self.c.lastrowid - @style.queue + @queue def addLinks(self, links, package): """ links is a list of tupels (url,plugin)""" order = self._nextFileOrder(package) @@ -624,27 +624,27 @@ class FileMethods(): links = [(x[0], x[0], x[1], package, o) for x, o in zip(links, orders)] self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) - @style.queue + @queue def addPackage(self, name, folder, queue): order = self._nextPackageOrder(queue) self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', (name, folder, queue, order)) return self.c.lastrowid - @style.queue + @queue def deletePackage(self, p): self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue)) - @style.queue + @queue def deleteLink(self, f): self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid))) - @style.queue + @queue def getAllLinks(self, q): """return information about all links in queue q @@ -677,7 +677,7 @@ class FileMethods(): return data - @style.queue + @queue def getAllPackages(self, q): """return information about packages in queue q (only useful in get all data) @@ -714,7 +714,7 @@ class FileMethods(): return data - @style.queue + @queue def getLinkData(self, id): """get link information as dict""" self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE id=?', (str(id), )) @@ -738,7 +738,7 @@ class FileMethods(): return data - @style.queue + @queue def getPackageData(self, id): """get data about links for a package""" self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id), )) @@ -762,15 +762,15 @@ class FileMethods(): return data - @style.async + @async def updateLink(self, f): self.c.execute('UPDATE links SET url=?,name=?,size=?,status=?,error=?,package=? WHERE id=?', (f.url, f.name, f.size, f.status, f.error, str(f.packageid), str(f.id))) - @style.queue + @queue def updatePackage(self, p): self.c.execute('UPDATE packages SET name=?,folder=?,site=?,password=?,queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) - @style.queue + @queue def updateLinkInfo(self, data): """ data is list of tupels (name, size, status, url) """ self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1,2,3,14)', data) @@ -780,7 +780,7 @@ class FileMethods(): ids.append(int(r[0])) return ids - @style.queue + @queue def reorderPackage(self, p, position, noMove=False): if position == -1: position = self._nextPackageOrder(p.queue) @@ -792,7 +792,7 @@ class FileMethods(): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) - @style.queue + @queue def reorderLink(self, f, position): """ reorder link with f as dict for pyfile """ if f["order"] > position: @@ -803,20 +803,20 @@ class FileMethods(): self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"])) - @style.queue + @queue def clearPackageOrder(self, p): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) - @style.async + @async def restartFile(self, id): self.c.execute('UPDATE links SET status=3,error="" WHERE id=?', (str(id),)) - @style.async + @async def restartPackage(self, id): self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) - @style.queue + @queue def getPackage(self, id): """return package instance from id""" self.c.execute("SELECT name,folder,site,password,queue,packageorder FROM packages WHERE id=?", (str(id), )) @@ -825,7 +825,7 @@ class FileMethods(): return PyPackage(self.manager, id, * r) #---------------------------------------------------------------------- - @style.queue + @queue def getFile(self, id): """return link instance from id""" self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id), )) @@ -834,7 +834,7 @@ class FileMethods(): return PyFile(self.manager, id, * r) - @style.queue + @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" @@ -854,7 +854,7 @@ class FileMethods(): return [x[0] for x in self.c] - @style.queue + @queue def getPluginJob(self, plugins): """returns pyfile ids with suited plugins""" cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins @@ -863,29 +863,29 @@ class FileMethods(): return [x[0] for x in self.c] - @style.queue + @queue def getUnfinished(self, pid): """return list of max length 3 ids with pyfiles in package not finished or processed""" self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) return [r[0] for r in self.c] - @style.queue + @queue def deleteFinished(self): self.c.execute("DELETE FROM links WHERE status IN (0,4)") self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") - @style.queue + @queue def restartFailed(self): self.c.execute("UPDATE links SET status=3,error='' WHERE status IN (8, 9)") - @style.queue + @queue def findDuplicates(self, id, folder, filename): """ checks if filename exists with different id and same package """ self.c.execute("SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", (folder, id, filename)) return self.c.fetchone() - @style.queue + @queue def purgeLinks(self): self.c.execute("DELETE FROM links;") self.c.execute("DELETE FROM packages;") diff --git a/module/database/StorageDatabase.py b/module/database/StorageDatabase.py index 3ed29625f..ffaf51763 100644 --- a/module/database/StorageDatabase.py +++ b/module/database/StorageDatabase.py @@ -16,11 +16,10 @@ @author: mkaay """ -from module.database import style -from module.database import DatabaseBackend +from module.database import DatabaseBackend, queue class StorageMethods(): - @style.queue + @queue def setStorage(db, identifier, key, value): db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) if db.c.fetchone() is not None: @@ -28,7 +27,7 @@ class StorageMethods(): else: db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) - @style.queue + @queue def getStorage(db, identifier, key=None): if key is not None: db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) @@ -42,7 +41,7 @@ class StorageMethods(): d[row[0]] = row[1] return d - @style.queue + @queue def delStorage(db, identifier, key): db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) diff --git a/module/database/UserDatabase.py b/module/database/UserDatabase.py index 0c781057d..a5077711d 100644 --- a/module/database/UserDatabase.py +++ b/module/database/UserDatabase.py @@ -19,11 +19,10 @@ from hashlib import sha1 import random -from DatabaseBackend import DatabaseBackend -from DatabaseBackend import style +from DatabaseBackend import DatabaseBackend, queue, async class UserMethods(): - @style.queue + @queue def checkAuth(db, user, password): c = db.c c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user, )) @@ -40,7 +39,7 @@ class UserMethods(): else: return {} - @style.queue + @queue def addUser(db, user, password): salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) h = sha1(salt + password) @@ -54,7 +53,7 @@ class UserMethods(): c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) - @style.queue + @queue def changePassword(db, user, oldpw, newpw): db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user, )) r = db.c.fetchone() @@ -75,16 +74,16 @@ class UserMethods(): return False - @style.async + @async def setPermission(db, user, perms): db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) - @style.async + @async def setRole(db, user, role): db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) - @style.queue + @queue def listUsers(db): db.c.execute('SELECT name FROM users') users = [] @@ -92,7 +91,7 @@ class UserMethods(): users.append(row[0]) return users - @style.queue + @queue def getAllUserData(db): db.c.execute("SELECT name, permission, role, template, email FROM users") user = {} @@ -101,7 +100,7 @@ class UserMethods(): return user - @style.queue + @queue def removeUser(db, user): db.c.execute('DELETE FROM users WHERE name=?', (user, )) diff --git a/module/database/__init__.py b/module/database/__init__.py index 545789c0c..39848ac58 100644 --- a/module/database/__init__.py +++ b/module/database/__init__.py @@ -1,6 +1,6 @@ -from DatabaseBackend import DatabaseBackend -from DatabaseBackend import style +from DatabaseBackend import DatabaseBackend, queue, async, inner from FileDatabase import FileHandler from UserDatabase import UserMethods -from StorageDatabase import StorageMethods \ No newline at end of file +from StorageDatabase import StorageMethods +from AccountDatabase import AccountMethods \ No newline at end of file -- cgit v1.2.3 From cb8b049b6c878065aebdd31c37cb67321de835ff Mon Sep 17 00:00:00 2001 From: Wieland Hoffmann Date: Sun, 25 Dec 2011 18:59:19 +0100 Subject: Use rowid in the user database --- module/database/UserDatabase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/database') diff --git a/module/database/UserDatabase.py b/module/database/UserDatabase.py index a5077711d..43fd93df3 100644 --- a/module/database/UserDatabase.py +++ b/module/database/UserDatabase.py @@ -25,7 +25,7 @@ class UserMethods(): @queue def checkAuth(db, user, password): c = db.c - c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user, )) + c.execute('SELECT rowid, name, password, role, permission, template, email FROM "users" WHERE name=?', (user, )) r = c.fetchone() if not r: return {} @@ -55,7 +55,7 @@ class UserMethods(): @queue def changePassword(db, user, oldpw, newpw): - db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user, )) + db.c.execute('SELECT rowid, name, password FROM users WHERE name=?', (user, )) r = db.c.fetchone() if not r: return False -- cgit v1.2.3 From d35c003cc53d4723d1dfe0d81eeb9bea78cee594 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 31 Dec 2011 16:01:24 +0100 Subject: new crypter plugin API, now decrypting possible for now. --- module/database/FileDatabase.py | 151 +++++++++++----------------------------- 1 file changed, 41 insertions(+), 110 deletions(-) (limited to 'module/database') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index b5c386802..abe7c8fc9 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -22,7 +22,6 @@ from threading import RLock from time import time from module.utils import formatSize, lock -from module.interaction.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent from module.PyPackage import PyPackage from module.PyFile import PyFile from module.database import DatabaseBackend, queue, async, inner @@ -40,11 +39,12 @@ class FileHandler: def __init__(self, core): """Constructor""" self.core = core + self.ev = None #event manager, set later # translations self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), _("downloading"), _("processing"), _("unknown")] - self.cache = {} #holds instances for files + self.cache = {} # holds instances for files self.packageCache = {} # same for packages #@TODO: purge the cache @@ -54,14 +54,12 @@ class FileHandler: #self.lock._Verbose__verbose = True self.filecount = -1 # if an invalid value is set get current value from db - self.queuecount = -1 #number of package to be loaded - self.unchanged = False #determines if any changes was made since last call + self.queuecount = -1 # number of package to be loaded self.db = self.core.db def change(func): def new(*args): - args[0].unchanged = False args[0].filecount = -1 args[0].queuecount = -1 args[0].jobCache = {} @@ -118,31 +116,23 @@ class FileHandler: @lock @change - def addLinks(self, urls, package): - """adds links""" - - self.core.hookManager.dispatchEvent("linksAdded", urls, package) - - data = self.core.pluginManager.parseUrls(urls) - + def addLinks(self, data, package): + """Add links, data = (plugin, url) tuple. Internal method you should use API.""" self.db.addLinks(data, package) - self.core.threadManager.createInfoThread(data, package) + self.ev.dispatchEvent("packageUpdated", package) - #@TODO change from reloadAll event to package update event - self.core.pullManager.addEvent(ReloadAllEvent("collector")) - #---------------------------------------------------------------------- @lock @change - def addPackage(self, name, folder, queue=0): + def addPackage(self, name, folder, queue=0, password=""): """adds a package, default to link collector""" - lastID = self.db.addPackage(name, folder, queue) - p = self.db.getPackage(lastID) - e = InsertEvent("pack", lastID, p.order, "collector" if not queue else "queue") - self.core.pullManager.addEvent(e) - return lastID + pid = self.db.addPackage(name, folder, queue, password) + p = self.db.getPackage(pid) + + self.ev.dispatchEvent("packageInserted", pid, p.queue, p.order) + return pid + - #---------------------------------------------------------------------- @lock @change def deletePackage(self, id): @@ -156,7 +146,6 @@ class FileHandler: oldorder = p.order queue = p.queue - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") pyfiles = self.cache.values() @@ -166,8 +155,7 @@ class FileHandler: pyfile.release() self.db.deletePackage(p) - self.core.pullManager.addEvent(e) - self.core.hookManager.dispatchEvent("packageDeleted", id) + self.ev.dispatchEvent("packageDeleted", id) if id in self.packageCache: del self.packageCache[id] @@ -178,7 +166,7 @@ class FileHandler: pack.order -= 1 pack.notifyChange() - #---------------------------------------------------------------------- + @lock @change def deleteLink(self, id): @@ -189,8 +177,6 @@ class FileHandler: return None pid = f.packageid - e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") - oldorder = f.order if id in self.core.threadManager.processingIds(): @@ -201,7 +187,7 @@ class FileHandler: self.db.deleteLink(f) - self.core.pullManager.addEvent(e) + self.ev.dispatchEvent("linkDeleted", id, pid) p = self.getPackage(pid) if not len(p.getChildren()): @@ -213,35 +199,26 @@ class FileHandler: pyfile.order -= 1 pyfile.notifyChange() - #---------------------------------------------------------------------- def releaseLink(self, id): """removes pyfile from cache""" if id in self.cache: del self.cache[id] - #---------------------------------------------------------------------- def releasePackage(self, id): """removes package from cache""" if id in self.packageCache: del self.packageCache[id] - #---------------------------------------------------------------------- def updateLink(self, pyfile): """updates link""" self.db.updateLink(pyfile) + self.ev.dispatchEvent("linkUpdated", pyfile.id, pyfile.packageid) - e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") - self.core.pullManager.addEvent(e) - - #---------------------------------------------------------------------- def updatePackage(self, pypack): """updates a package""" self.db.updatePackage(pypack) + self.ev.dispatchEvent("packageUpdated", pypack.id) - e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") - self.core.pullManager.addEvent(e) - - #---------------------------------------------------------------------- def getPackage(self, id): """return package instance""" @@ -250,7 +227,6 @@ class FileHandler: else: return self.db.getPackage(id) - #---------------------------------------------------------------------- def getPackageData(self, id): """returns dict with package information""" pack = self.getPackage(id) @@ -274,7 +250,7 @@ class FileHandler: return pack - #---------------------------------------------------------------------- + def getFileData(self, id): """returns dict with file information""" if id in self.cache: @@ -282,7 +258,7 @@ class FileHandler: return self.db.getLinkData(id) - #---------------------------------------------------------------------- + def getFile(self, id): """returns pyfile instance""" if id in self.cache: @@ -290,7 +266,7 @@ class FileHandler: else: return self.db.getFile(id) - #---------------------------------------------------------------------- + @lock def getJob(self, occ): """get suitable job""" @@ -334,21 +310,6 @@ class FileHandler: #pyfile = self.getFile(self.jobCache[occ].pop()) return pyfile - @lock - def getDecryptJob(self): - """return job for decrypting""" - if "decrypt" in self.jobCache: - return None - - plugins = self.core.pluginManager.getPlugins("crypter").keys() + self.core.pluginManager.getPlugins("container").keys() - plugins = str(tuple(plugins)) - - jobs = self.db.getPluginJob(plugins) - if jobs: - return self.getFile(jobs[0]) - else: - self.jobCache["decrypt"] = "empty" - return None def getFileCount(self): """returns number of files""" @@ -405,8 +366,7 @@ class FileHandler: if id in self.packageCache: self.packageCache[id].setFinished = False - e = UpdateEvent("pack", id, "collector" if not self.getPackage(id).queue else "queue") - self.core.pullManager.addEvent(e) + self.ev.dispatchEvent("packageUpdated", id) @lock @change @@ -420,9 +380,8 @@ class FileHandler: self.db.restartFile(id) + self.ev.dispatchEvent("linkUpdated", id) - e = UpdateEvent("file", id, "collector" if not self.getFile(id).package().queue else "queue") - self.core.pullManager.addEvent(e) @lock @change @@ -431,17 +390,10 @@ class FileHandler: p = self.db.getPackage(id) oldorder = p.order + p.queue = queue - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - self.db.clearPackageOrder(p) - - p = self.db.getPackage(id) - - p.queue = queue self.db.updatePackage(p) - self.db.reorderPackage(p, -1, True) packs = self.packageCache.values() @@ -452,37 +404,34 @@ class FileHandler: self.db.commit() self.releasePackage(id) - p = self.getPackage(id) - - e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) + + self.ev.dispatchEvent("packageDeleted", id) + self.ev.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change def reorderPackage(self, id, position): p = self.getPackage(id) - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) self.db.reorderPackage(p, position) packs = self.packageCache.values() for pack in packs: if pack.queue != p.queue or pack.order < 0 or pack == p: continue if p.order > position: - if pack.order >= position and pack.order < p.order: + if position <= pack.order < p.order: pack.order += 1 pack.notifyChange() elif p.order < position: - if pack.order <= position and pack.order > p.order: + if position >= pack.order > p.order: pack.order -= 1 pack.notifyChange() p.order = position self.db.commit() - e = InsertEvent("pack", id, position, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) + self.ev.dispatchEvent("packageDeleted", id) + self.ev.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change @@ -490,20 +439,17 @@ class FileHandler: f = self.getFileData(id) f = f[id] - e = RemoveEvent("file", id, "collector" if not self.getPackage(f["package"]).queue else "queue") - self.core.pullManager.addEvent(e) - self.db.reorderLink(f, position) pyfiles = self.cache.values() for pyfile in pyfiles: if pyfile.packageid != f["package"] or pyfile.order < 0: continue if f["order"] > position: - if pyfile.order >= position and pyfile.order < f["order"]: + if position <= pyfile.order < f["order"]: pyfile.order += 1 pyfile.notifyChange() elif f["order"] < position: - if pyfile.order <= position and pyfile.order > f["order"]: + if position >= pyfile.order > f["order"]: pyfile.order -= 1 pyfile.notifyChange() @@ -512,15 +458,14 @@ class FileHandler: self.db.commit() - e = InsertEvent("file", id, position, "collector" if not self.getPackage(f["package"]).queue else "queue") - self.core.pullManager.addEvent(e) + self.ev.dispatchEvent("packageUpdated", f["package"]) + @change def updateFileInfo(self, data, pid): """ updates file info (name, size, status, url)""" ids = self.db.updateLinkInfo(data) - e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue") - self.core.pullManager.addEvent(e) + self.ev.dispatchEvent("packageUpdated", pid) def checkPackageFinished(self, pyfile): """ checks if package is finished and calls hookmanager """ @@ -625,9 +570,9 @@ class FileMethods(): self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) @queue - def addPackage(self, name, folder, queue): + def addPackage(self, name, folder, queue, password): order = self._nextPackageOrder(queue) - self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', (name, folder, queue, order)) + self.c.execute('INSERT INTO packages(name, folder, queue, packageorder, password) VALUES(?,?,?,?,?)', (name, folder, queue, order, password)) return self.c.lastrowid @queue @@ -824,7 +769,7 @@ class FileMethods(): if not r: return None return PyPackage(self.manager, id, * r) - #---------------------------------------------------------------------- + @queue def getFile(self, id): """return link instance from id""" @@ -837,28 +782,14 @@ class FileMethods(): @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" - - #@TODO improve this hardcoded method - pre = "('DLC', 'LinkList', 'SerienjunkiesOrg', 'CCF', 'RSDF')" #plugins which are processed in collector - cmd = "(" for i, item in enumerate(occ): if i: cmd += ", " cmd += "'%s'" % item - - cmd += ")" - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) - - self.c.execute(cmd) # very bad! - - return [x[0] for x in self.c] - - @queue - def getPluginJob(self, plugins): - """returns pyfile ids with suited plugins""" - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins + cmd += ")" + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=1 AND l.plugin NOT IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % cmd self.c.execute(cmd) # very bad! return [x[0] for x in self.c] -- cgit v1.2.3 From 35742c2cb023ac49ab3056752d2040cdb030cc2b Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 1 Jan 2012 13:36:59 +0100 Subject: Happy new Year ! --- module/database/FileDatabase.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'module/database') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index abe7c8fc9..116f2b02b 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -190,8 +190,7 @@ class FileHandler: self.ev.dispatchEvent("linkDeleted", id, pid) p = self.getPackage(pid) - if not len(p.getChildren()): - p.delete() + p.deleteIfEmpty() pyfiles = self.cache.values() for pyfile in pyfiles: -- cgit v1.2.3 From 18466eb7f8f3cd4ca9a0824074d2ff454939fce6 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Wed, 4 Jan 2012 17:23:13 +0100 Subject: some fixes --- module/database/FileDatabase.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'module/database') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 116f2b02b..23e657ee4 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -39,7 +39,7 @@ class FileHandler: def __init__(self, core): """Constructor""" self.core = core - self.ev = None #event manager, set later + self.evm = core.eventManager # translations self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), _("downloading"), _("processing"), _("unknown")] @@ -119,7 +119,7 @@ class FileHandler: def addLinks(self, data, package): """Add links, data = (plugin, url) tuple. Internal method you should use API.""" self.db.addLinks(data, package) - self.ev.dispatchEvent("packageUpdated", package) + self.evm.dispatchEvent("packageUpdated", package) @lock @@ -129,7 +129,7 @@ class FileHandler: pid = self.db.addPackage(name, folder, queue, password) p = self.db.getPackage(pid) - self.ev.dispatchEvent("packageInserted", pid, p.queue, p.order) + self.evm.dispatchEvent("packageInserted", pid, p.queue, p.order) return pid @@ -155,7 +155,7 @@ class FileHandler: pyfile.release() self.db.deletePackage(p) - self.ev.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageDeleted", id) if id in self.packageCache: del self.packageCache[id] @@ -187,7 +187,7 @@ class FileHandler: self.db.deleteLink(f) - self.ev.dispatchEvent("linkDeleted", id, pid) + self.evm.dispatchEvent("linkDeleted", id, pid) p = self.getPackage(pid) p.deleteIfEmpty() @@ -211,12 +211,12 @@ class FileHandler: def updateLink(self, pyfile): """updates link""" self.db.updateLink(pyfile) - self.ev.dispatchEvent("linkUpdated", pyfile.id, pyfile.packageid) + self.evm.dispatchEvent("linkUpdated", pyfile.id, pyfile.packageid) def updatePackage(self, pypack): """updates a package""" self.db.updatePackage(pypack) - self.ev.dispatchEvent("packageUpdated", pypack.id) + self.evm.dispatchEvent("packageUpdated", pypack.id) def getPackage(self, id): """return package instance""" @@ -365,7 +365,7 @@ class FileHandler: if id in self.packageCache: self.packageCache[id].setFinished = False - self.ev.dispatchEvent("packageUpdated", id) + self.evm.dispatchEvent("packageUpdated", id) @lock @change @@ -379,7 +379,7 @@ class FileHandler: self.db.restartFile(id) - self.ev.dispatchEvent("linkUpdated", id) + self.evm.dispatchEvent("linkUpdated", id) @lock @@ -404,8 +404,8 @@ class FileHandler: self.db.commit() self.releasePackage(id) - self.ev.dispatchEvent("packageDeleted", id) - self.ev.dispatchEvent("packageInserted", id, p.queue, p.order) + self.evm.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change @@ -429,8 +429,8 @@ class FileHandler: p.order = position self.db.commit() - self.ev.dispatchEvent("packageDeleted", id) - self.ev.dispatchEvent("packageInserted", id, p.queue, p.order) + self.evm.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change @@ -457,14 +457,14 @@ class FileHandler: self.db.commit() - self.ev.dispatchEvent("packageUpdated", f["package"]) + self.evm.dispatchEvent("packageUpdated", f["package"]) @change def updateFileInfo(self, data, pid): """ updates file info (name, size, status, url)""" ids = self.db.updateLinkInfo(data) - self.ev.dispatchEvent("packageUpdated", pid) + self.evm.dispatchEvent("packageUpdated", pid) def checkPackageFinished(self, pyfile): """ checks if package is finished and calls hookmanager """ -- cgit v1.2.3 From d347b354b169044fabd0684f304f1777ae5b75d8 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 21:59:28 +0100 Subject: show empty packages --- module/database/FileDatabase.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/database') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 23e657ee4..eb76f468b 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -636,7 +636,7 @@ class FileMethods(): } """ self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ - FROM packages p JOIN pstats s ON p.id = s.id \ + FROM packages p LEFT OUTER JOIN pstats s ON p.id = s.id \ WHERE p.queue=? ORDER BY p.packageorder', str(q)) data = {} @@ -649,10 +649,10 @@ class FileMethods(): 'password': r[4], 'queue': r[5], 'order': r[6], - 'sizetotal': int(r[7]), - 'sizedone': r[8] if r[8] else 0, #these can be None + 'sizetotal': int(r[7]) if r[7] else 0, + 'sizedone': int(r[8]) if r[8] else 0, #these can be None 'linksdone': r[9] if r[9] else 0, - 'linkstotal': r[10], + 'linkstotal': r[10] if r[10] else 0, 'links': {} } -- cgit v1.2.3 From 1ecdd9f6b53fec45e1d48592e3ff56aa7a576bec Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 8 Jan 2012 16:47:52 +0100 Subject: some cleanups, closed #490 --- module/database/DatabaseBackend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/database') diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index e10bcbbaf..32f75328c 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -25,7 +25,7 @@ from shutil import move from Queue import Queue from traceback import print_exc -from module.utils import chmod +from module.utils.fs import chmod try: from pysqlite2 import dbapi2 as sqlite3 -- cgit v1.2.3 From 4df2b77fdf42046fe19bd371be7c7255986b5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 6 Mar 2012 13:36:39 +0100 Subject: renamed hooks to addons, new filemanager and database, many new api methods you will loose ALL your LINKS, webinterface will NOT work --- module/database/DatabaseBackend.py | 237 ++++++--- module/database/FileDatabase.py | 1033 ++++++++++-------------------------- module/database/UserDatabase.py | 56 +- module/database/__init__.py | 4 +- 4 files changed, 472 insertions(+), 858 deletions(-) (limited to 'module/database') diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 32f75328c..8159446bd 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -16,16 +16,13 @@ @author: RaNaN @author: mkaay """ -from threading import Thread -from threading import Event -from os import remove -from os.path import exists +from threading import Thread, Event from shutil import move from Queue import Queue from traceback import print_exc -from module.utils.fs import chmod +from module.utils.fs import chmod, exists, remove try: from pysqlite2 import dbapi2 as sqlite3 @@ -33,7 +30,7 @@ except: import sqlite3 DB = None -DB_VERSION = 4 +DB_VERSION = 5 def set_DB(db): global DB @@ -67,6 +64,18 @@ def inner(f): return x +class DatabaseMethods: + # stubs for autocompletion + core = None + manager = None + conn = None + c = None + + @classmethod + def register(cls): + DatabaseBackend.registerSub(cls) + + class DatabaseJob(): def __init__(self, f, *args, **kwargs): self.done = Event() @@ -122,34 +131,60 @@ class DatabaseBackend(Thread): Thread.__init__(self) self.setDaemon(True) self.core = core + self.manager = None # setted later + self.running = Event() self.jobs = Queue() - self.setuplock = Event() - set_DB(self) def setup(self): + self.start() - self.setuplock.wait() + self.running.wait() - def run(self): + def init(self): """main loop, which executes commands""" - convert = self._checkVersion() #returns None or current version + + version = self._checkVersion() self.conn = sqlite3.connect(self.DB_FILE) chmod(self.DB_FILE, 0600) - self.c = self.conn.cursor() #compatibility + self.c = self.conn.cursor() - if convert is not None: - self._convertDB(convert) + if version is not None and version < DB_VERSION: + success = self._convertDB(version) - self._createTables() + # delete database + if not success: + self.c.close() + self.conn.close() + try: + self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) + except: + print "Filedatabase was deleted due to incompatible version." + + remove(self.VERSION_FILE) + move(self.DB_FILE, self.DB_FILE + ".backup") + f = open(self.VERSION_FILE, "wb") + f.write(str(DB_VERSION)) + f.close() + + self.conn = sqlite3.connect(self.DB_FILE) + chmod(self.DB_FILE, 0600) + self.c = self.conn.cursor() + + self._createTables() self.conn.commit() - self.setuplock.set() + + def run(self): + try: + self.init() + finally: + self.running.set() while True: j = self.jobs.get() @@ -159,51 +194,40 @@ class DatabaseBackend(Thread): break j.processJob() - @queue + def shutdown(self): + self.running.clear() + self._shutdown() + + @queue + def _shutdown(self): self.conn.commit() self.jobs.put("quit") def _checkVersion(self): - """ check db version and delete it if needed""" + """ get db version""" if not exists(self.VERSION_FILE): f = open(self.VERSION_FILE, "wb") f.write(str(DB_VERSION)) f.close() return - if exists("files.db") and not exists(self.DB_FILE): - move("files.db", self.DB_FILE) - f = open(self.VERSION_FILE, "rb") v = int(f.read().strip()) f.close() - if v < DB_VERSION: - if v < 2: - try: - self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) - except: - print "Filedatabase was deleted due to incompatible version." - remove(self.VERSION_FILE) - move(self.DB_FILE, self.DB_FILE + ".backup") - f = open(self.VERSION_FILE, "wb") - f.write(str(DB_VERSION)) - f.close() - return v + + return v def _convertDB(self, v): try: - getattr(self, "_convertV%i" % v)() + return getattr(self, "_convertV%i" % v)() except: - try: - self.core.log.error(_("Filedatabase could NOT be converted.")) - except: - print "Filedatabase could NOT be converted." + return False #--convert scripts start - def _convertV4(self): - pass + def _convertV5(self): + return False #--convert scripts end @@ -211,39 +235,122 @@ class DatabaseBackend(Thread): """create tables for database""" self.c.execute( - 'CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') + 'CREATE TABLE IF NOT EXISTS "packages" (' + '"pid" INTEGER PRIMARY KEY AUTOINCREMENT, ' + '"name" TEXT NOT NULL, ' + '"folder" TEXT DEFAULT "" NOT NULL, ' + '"site" TEXT DEFAULT "" NOT NULL, ' + '"comment" TEXT DEFAULT "" NOT NULL, ' + '"password" TEXT DEFAULT "" NOT NULL, ' + '"added" INTEGER DEFAULT 0 NOT NULL,' # set by trigger + '"status" INTEGER DEFAULT 0 NOT NULL,' + '"packageorder" INTEGER DEFAULT -1 NOT NULL,' #incremented by trigger + '"root" INTEGER DEFAULT -1 NOT NULL,' + 'CHECK (root != pid) ' + ')' + ) + self.c.execute( - 'CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') - self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)') + 'CREATE TRIGGER IF NOT EXISTS "insert_package" AFTER INSERT ON "packages"' + 'BEGIN ' + 'UPDATE packages SET added = strftime("%s", "now"), ' + 'packageorder = (SELECT max(p.packageorder) + 1 FROM packages p WHERE p.root=new.root) ' + 'WHERE rowid = new.rowid;' + 'END' + ) + self.c.execute( - 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + 'CREATE TRIGGER IF NOT EXISTS "delete_package" AFTER DELETE ON "packages"' + 'BEGIN ' + 'DELETE FROM files WHERE package = old.pid;' + 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > old.packageorder AND root=old.pid;' + 'END' + ) + + self.c.execute('CREATE INDEX IF NOT EXISTS "root_index" ON packages(root)') + self.c.execute( - 'CREATE TABLE IF NOT EXISTS "users" ("name" TEXT PRIMARY KEY NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + 'CREATE TABLE IF NOT EXISTS "files" (' + '"fid" INTEGER PRIMARY KEY AUTOINCREMENT, ' + '"name" TEXT NOT NULL, ' + '"size" INTEGER DEFAULT 0 NOT NULL, ' + '"status" INTEGER DEFAULT 0 NOT NULL, ' + '"media" INTEGER DEFAULT 1 NOT NULL,' + '"added" INTEGER DEFAULT 0 NOT NULL,' + '"fileorder" INTEGER DEFAULT -1 NOT NULL, ' + '"url" TEXT DEFAULT "" NOT NULL, ' + '"plugin" TEXT DEFAULT "" NOT NULL, ' + '"hash" TEXT DEFAULT "" NOT NULL, ' + '"dlstatus" INTEGER DEFAULT 0 NOT NULL, ' + '"error" TEXT DEFAULT "" NOT NULL, ' + '"package" INTEGER NOT NULL, ' + 'FOREIGN KEY(package) REFERENCES packages(id)' + ')' + ) + + self.c.execute('CREATE INDEX IF NOT EXISTS "package_index" ON files(package)') + self.c.execute( - 'CREATE TABLE IF NOT EXISTS "accounts" ("plugin" TEXT NOT NULL, "loginname" TEXT NOT NULL, "activated" INTEGER DEFAULT 1, "password" TEXT DEFAULT "", "options" TEXT DEFAULT "", PRIMARY KEY (plugin, loginname) ON CONFLICT REPLACE)') + 'CREATE TRIGGER IF NOT EXISTS "insert_file" AFTER INSERT ON "files"' + 'BEGIN ' + 'UPDATE files SET added = strftime("%s", "now"), ' + 'fileorder = (SELECT max(f.fileorder) + 1 FROM files f WHERE f.package=new.package) ' + 'WHERE rowid = new.rowid;' + 'END' + ) - self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \ - SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\ - FROM packages p JOIN links l ON p.id = l.package LEFT OUTER JOIN\ - (SELECT p.id AS id, COUNT(*) AS linksdone, SUM(l.size) AS sizedone \ - FROM packages p JOIN links l ON p.id = l.package AND l.status in (0,4,13) GROUP BY p.id) s ON s.id = p.id \ - GROUP BY p.id') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "collector" (' + '"url" TEXT NOT NULL, ' + '"name" TEXT NOT NULL, ' + '"plugin" TEXT DEFAULT "BasePlugin" NOT NULL, ' + '"size" INTEGER DEFAULT 0 NOT NULL, ' + '"status" INTEGER DEFAULT 3 NOT NULL, ' + '"packagename" TEXT DEFAULT "" NOT NULL, ' + 'PRIMARY KEY (url, packagename) ON CONFLICT REPLACE' + ') ' + ) + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "storage" (' + '"identifier" TEXT NOT NULL, ' + '"key" TEXT NOT NULL, ' + '"value" TEXT DEFAULT "", ' + 'PRIMARY KEY (identifier, key) ON CONFLICT REPLACE' + ')' + ) + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "users" (' + '"name" TEXT PRIMARY KEY NOT NULL, ' + '"email" TEXT DEFAULT "" NOT NULL, ' + '"password" TEXT NOT NULL, ' + '"role" INTEGER DEFAULT 0 NOT NULL, ' + '"permission" INTEGER DEFAULT 0 NOT NULL, ' + '"template" TEXT DEFAULT "default" NOT NULL' + ')' + ) + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "accounts" (' + '"plugin" TEXT NOT NULL, ' + '"loginname" TEXT NOT NULL, ' + '"activated" INTEGER DEFAULT 1, ' + '"password" TEXT DEFAULT "", ' + '"options" TEXT DEFAULT "", ' + 'PRIMARY KEY (plugin, loginname) ON CONFLICT REPLACE' + ')' + ) #try to lower ids - self.c.execute('SELECT max(id) FROM LINKS') + self.c.execute('SELECT max(fid) FROM files') fid = self.c.fetchone()[0] - if fid: - fid = int(fid) - else: - fid = 0 - self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links")) + fid = int(fid) if fid else 0 + self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "files")) - self.c.execute('SELECT max(id) FROM packages') + self.c.execute('SELECT max(pid) FROM packages') pid = self.c.fetchone()[0] - if pid: - pid = int(pid) - else: - pid = 0 + pid = int(pid) if pid else 0 self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (pid, "packages")) self.c.execute('VACUUM') @@ -273,7 +380,8 @@ class DatabaseBackend(Thread): args = (self, ) + args job = DatabaseJob(f, *args, **kwargs) self.jobs.put(job) - job.wait() + # only wait when db is running + if self.running.isSet(): job.wait() return job.result @classmethod @@ -288,6 +396,7 @@ class DatabaseBackend(Thread): for sub in DatabaseBackend.subs: if hasattr(sub, attr): return getattr(sub, attr) + raise AttributeError(attr) if __name__ == "__main__": db = DatabaseBackend() diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index eb76f468b..08b18765d 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- + """ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -14,781 +16,331 @@ along with this program; if not, see . @author: RaNaN - @author: mkaay """ +from new_collections import OrderedDict +from module.Api import DownloadInfo, LinkStatus, FileInfo, PackageInfo, PackageStats +from module.database import DatabaseMethods, queue, async, inner -from threading import RLock -from time import time - -from module.utils import formatSize, lock -from module.PyPackage import PyPackage -from module.PyFile import PyFile -from module.database import DatabaseBackend, queue, async, inner - -try: - from pysqlite2 import dbapi2 as sqlite3 -except: - import sqlite3 - - -class FileHandler: - """Handles all request made to obtain information, - modify status or other request for links or packages""" - - def __init__(self, core): - """Constructor""" - self.core = core - self.evm = core.eventManager - - # translations - self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), _("downloading"), _("processing"), _("unknown")] - - self.cache = {} # holds instances for files - self.packageCache = {} # same for packages - #@TODO: purge the cache - - self.jobCache = {} - - self.lock = RLock() #@TODO should be a Lock w/o R - #self.lock._Verbose__verbose = True - - self.filecount = -1 # if an invalid value is set get current value from db - self.queuecount = -1 # number of package to be loaded - - self.db = self.core.db - - def change(func): - def new(*args): - args[0].filecount = -1 - args[0].queuecount = -1 - args[0].jobCache = {} - return func(*args) - return new - - #---------------------------------------------------------------------- - def save(self): - """saves all data to backend""" - self.db.commit() - - #---------------------------------------------------------------------- - def syncSave(self): - """saves all data to backend and waits until all data are written""" - pyfiles = self.cache.values() - for pyfile in pyfiles: - pyfile.sync() - - pypacks = self.packageCache.values() - for pypack in pypacks: - pypack.sync() - - self.db.syncSave() - - @lock - def getCompleteData(self, queue=1): - """gets a complete data representation""" - - data = self.db.getAllLinks(queue) - packs = self.db.getAllPackages(queue) - - data.update([(x.id, x.toDbDict()[x.id]) for x in self.cache.values()]) - - for x in self.packageCache.itervalues(): - if x.queue != queue or x.id not in packs: continue - packs[x.id].update(x.toDict()[x.id]) - - for key, value in data.iteritems(): - if value["package"] in packs: - packs[value["package"]]["links"][key] = value - - return packs - - @lock - def getInfoData(self, queue=1): - """gets a data representation without links""" - - packs = self.db.getAllPackages(queue) - for x in self.packageCache.itervalues(): - if x.queue != queue or x.id not in packs: continue - packs[x.id].update(x.toDict()[x.id]) - - return packs - - @lock - @change - def addLinks(self, data, package): - """Add links, data = (plugin, url) tuple. Internal method you should use API.""" - self.db.addLinks(data, package) - self.evm.dispatchEvent("packageUpdated", package) - - - @lock - @change - def addPackage(self, name, folder, queue=0, password=""): - """adds a package, default to link collector""" - pid = self.db.addPackage(name, folder, queue, password) - p = self.db.getPackage(pid) - - self.evm.dispatchEvent("packageInserted", pid, p.queue, p.order) - return pid - - - @lock - @change - def deletePackage(self, id): - """delete package and all contained links""" - - p = self.getPackage(id) - if not p: - if id in self.packageCache: del self.packageCache[id] - return - - oldorder = p.order - queue = p.queue - - - pyfiles = self.cache.values() - - for pyfile in pyfiles: - if pyfile.packageid == id: - pyfile.abortDownload() - pyfile.release() - - self.db.deletePackage(p) - self.evm.dispatchEvent("packageDeleted", id) - - if id in self.packageCache: - del self.packageCache[id] - - packs = self.packageCache.values() - for pack in packs: - if pack.queue == queue and pack.order > oldorder: - pack.order -= 1 - pack.notifyChange() - - - @lock - @change - def deleteLink(self, id): - """deletes links""" - - f = self.getFile(id) - if not f: - return None - - pid = f.packageid - oldorder = f.order - - if id in self.core.threadManager.processingIds(): - self.cache[id].abortDownload() - - if id in self.cache: - del self.cache[id] - - self.db.deleteLink(f) - - self.evm.dispatchEvent("linkDeleted", id, pid) - - p = self.getPackage(pid) - p.deleteIfEmpty() - - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid == pid and pyfile.order > oldorder: - pyfile.order -= 1 - pyfile.notifyChange() - - def releaseLink(self, id): - """removes pyfile from cache""" - if id in self.cache: - del self.cache[id] - - def releasePackage(self, id): - """removes package from cache""" - if id in self.packageCache: - del self.packageCache[id] - - def updateLink(self, pyfile): - """updates link""" - self.db.updateLink(pyfile) - self.evm.dispatchEvent("linkUpdated", pyfile.id, pyfile.packageid) - - def updatePackage(self, pypack): - """updates a package""" - self.db.updatePackage(pypack) - self.evm.dispatchEvent("packageUpdated", pypack.id) - - def getPackage(self, id): - """return package instance""" - - if id in self.packageCache: - return self.packageCache[id] - else: - return self.db.getPackage(id) - - def getPackageData(self, id): - """returns dict with package information""" - pack = self.getPackage(id) - - if not pack: - return None - - pack = pack.toDict()[id] - - data = self.db.getPackageData(id) - - tmplist = [] - - cache = self.cache.values() - for x in cache: - if int(x.toDbDict()[x.id]["package"]) == int(id): - tmplist.append((x.id, x.toDbDict()[x.id])) - data.update(tmplist) - - pack["links"] = data - - return pack - - - def getFileData(self, id): - """returns dict with file information""" - if id in self.cache: - return self.cache[id].toDbDict() - - return self.db.getLinkData(id) - - - def getFile(self, id): - """returns pyfile instance""" - if id in self.cache: - return self.cache[id] - else: - return self.db.getFile(id) - - - @lock - def getJob(self, occ): - """get suitable job""" - - #@TODO clean mess - #@TODO improve selection of valid jobs - - if occ in self.jobCache: - if self.jobCache[occ]: - id = self.jobCache[occ].pop() - if id == "empty": - pyfile = None - self.jobCache[occ].append("empty") - else: - pyfile = self.getFile(id) - else: - jobs = self.db.getJob(occ) - jobs.reverse() - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - self.jobCache[occ].extend(jobs) - pyfile = self.getFile(self.jobCache[occ].pop()) - - else: - self.jobCache = {} #better not caching to much - jobs = self.db.getJob(occ) - jobs.reverse() - self.jobCache[occ] = jobs - - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - pyfile = self.getFile(self.jobCache[occ].pop()) - - #@TODO: maybe the new job has to be approved... - - - #pyfile = self.getFile(self.jobCache[occ].pop()) - return pyfile - - - def getFileCount(self): - """returns number of files""" - - if self.filecount == -1: - self.filecount = self.db.filecount(1) - - return self.filecount - - def getQueueCount(self, force=False): - """number of files that have to be processed""" - if self.queuecount == -1 or force: - self.queuecount = self.db.queuecount(1) - - return self.queuecount - - def checkAllLinksFinished(self): - """checks if all files are finished and dispatch event""" - - if not self.getQueueCount(True): - self.core.hookManager.dispatchEvent("allDownloadsFinished") - self.core.log.debug("All downloads finished") - return True - - return False - - def checkAllLinksProcessed(self, fid): - """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" - - # reset count so statistic will update (this is called when dl was processed) - self.resetCount() - - if not self.db.processcount(1, fid): - self.core.hookManager.dispatchEvent("allDownloadsProcessed") - self.core.log.debug("All downloads processed") - return True - - return False - - def resetCount(self): - self.queuecount = -1 - - @lock - @change - def restartPackage(self, id): - """restart package""" - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid == id: - self.restartFile(pyfile.id) - - self.db.restartPackage(id) - - if id in self.packageCache: - self.packageCache[id].setFinished = False - - self.evm.dispatchEvent("packageUpdated", id) - - @lock - @change - def restartFile(self, id): - """ restart file""" - if id in self.cache: - self.cache[id].status = 3 - self.cache[id].name = self.cache[id].url - self.cache[id].error = "" - self.cache[id].abortDownload() - - - self.db.restartFile(id) - self.evm.dispatchEvent("linkUpdated", id) - - - @lock - @change - def setPackageLocation(self, id, queue): - """push package to queue""" - - p = self.db.getPackage(id) - oldorder = p.order - p.queue = queue +default = PackageStats(0, 0, 0, 0) - self.db.clearPackageOrder(p) - self.db.updatePackage(p) - self.db.reorderPackage(p, -1, True) - - packs = self.packageCache.values() - for pack in packs: - if pack.queue != queue and pack.order > oldorder: - pack.order -= 1 - pack.notifyChange() - - self.db.commit() - self.releasePackage(id) - - self.evm.dispatchEvent("packageDeleted", id) - self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) - - @lock - @change - def reorderPackage(self, id, position): - p = self.getPackage(id) - - self.db.reorderPackage(p, position) - - packs = self.packageCache.values() - for pack in packs: - if pack.queue != p.queue or pack.order < 0 or pack == p: continue - if p.order > position: - if position <= pack.order < p.order: - pack.order += 1 - pack.notifyChange() - elif p.order < position: - if position >= pack.order > p.order: - pack.order -= 1 - pack.notifyChange() - - p.order = position - self.db.commit() - - self.evm.dispatchEvent("packageDeleted", id) - self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) - - @lock - @change - def reorderFile(self, id, position): - f = self.getFileData(id) - f = f[id] - - self.db.reorderLink(f, position) - - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid != f["package"] or pyfile.order < 0: continue - if f["order"] > position: - if position <= pyfile.order < f["order"]: - pyfile.order += 1 - pyfile.notifyChange() - elif f["order"] < position: - if position >= pyfile.order > f["order"]: - pyfile.order -= 1 - pyfile.notifyChange() - - if id in self.cache: - self.cache[id].order = position - - self.db.commit() - - self.evm.dispatchEvent("packageUpdated", f["package"]) - - - @change - def updateFileInfo(self, data, pid): - """ updates file info (name, size, status, url)""" - ids = self.db.updateLinkInfo(data) - self.evm.dispatchEvent("packageUpdated", pid) - - def checkPackageFinished(self, pyfile): - """ checks if package is finished and calls hookmanager """ - - ids = self.db.getUnfinished(pyfile.packageid) - if not ids or (pyfile.id in ids and len(ids) == 1): - if not pyfile.package().setFinished: - self.core.log.info(_("Package finished: %s") % pyfile.package().name) - self.core.hookManager.packageFinished(pyfile.package()) - pyfile.package().setFinished = True - - - def reCheckPackage(self, pid): - """ recheck links in package """ - data = self.db.getPackageData(pid) - - urls = [] - - for pyfile in data.itervalues(): - if pyfile["status"] not in (0, 12, 13): - urls.append((pyfile["url"], pyfile["plugin"])) - - self.core.threadManager.createInfoThread(urls, pid) - - @lock - @change - def deleteFinishedLinks(self): - """ deletes finished links and packages, return deleted packages """ - - old_packs = self.getInfoData(0) - old_packs.update(self.getInfoData(1)) - - self.db.deleteFinished() - - new_packs = self.db.getAllPackages(0) - new_packs.update(self.db.getAllPackages(1)) - #get new packages only from db - - deleted = [] - for id in old_packs.iterkeys(): - if id not in new_packs: - deleted.append(id) - self.deletePackage(int(id)) - - return deleted - - @lock - @change - def restartFailed(self): - """ restart all failed links """ - self.db.restartFailed() - -class FileMethods(): +class FileMethods(DatabaseMethods): @queue - def filecount(self, queue): - """returns number of files in queue""" - self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue, )) + def filecount(self): + """returns number of files""" + self.c.execute("SELECT COUNT(*) FROM files") return self.c.fetchone()[0] @queue - def queuecount(self, queue): + def queuecount(self): """ number of files in queue not finished yet""" - self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0,4)", (queue, )) + # status not in NA, finished, skipped + self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus NOT IN (0,5,6)") return self.c.fetchone()[0] @queue - def processcount(self, queue, fid): + def processcount(self, fid): """ number of files which have to be proccessed """ - self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2,3,5,7,12) AND l.id != ?", (queue, str(fid))) + # status in online, queued, starting, waiting, downloading + self.c.execute("SELECT COUNT(*) FROM files as WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (str(fid), )) return self.c.fetchone()[0] - @inner - def _nextPackageOrder(self, queue=0): - self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) - max = self.c.fetchone()[0] - if max is not None: - return max + 1 - else: - return 0 - - @inner - def _nextFileOrder(self, package): - self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) - max = self.c.fetchone()[0] - if max is not None: - return max + 1 - else: - return 0 - @queue def addLink(self, url, name, plugin, package): - order = self._nextFileOrder(package) - self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', (url, name, plugin, package, order)) + # mark filestatus initially as missing, dlstatus - queued + self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', + (url, name, plugin, package)) return self.c.lastrowid - @queue + @async def addLinks(self, links, package): - """ links is a list of tupels (url,plugin)""" - order = self._nextFileOrder(package) - orders = [order + x for x in range(len(links))] - links = [(x[0], x[0], x[1], package, o) for x, o in zip(links, orders)] - self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) + """ links is a list of tupels (url, plugin)""" + links = [(x[0], x[0], x[1], package) for x in links] + self.c.executemany('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', links) @queue - def addPackage(self, name, folder, queue, password): - order = self._nextPackageOrder(queue) - self.c.execute('INSERT INTO packages(name, folder, queue, packageorder, password) VALUES(?,?,?,?,?)', (name, folder, queue, order, password)) + def addFile(self, name, size, media, package): + # filestatus - ok, dl status NA + self.c.execute('INSERT INTO files(name, size, media, package) VALUES(?,?,?,?)', + (name, size, media, package)) return self.c.lastrowid @queue - def deletePackage(self, p): + def addPackage(self, name, folder, root, password, site, comment, status): + self.c.execute('INSERT INTO packages(name, folder, root, password, site, comment, status) VALUES(?,?,?,?,?,?,?)' + , + (name, folder, root, password, site, comment, status)) + return self.c.lastrowid - self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) - self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue)) + @async + def deletePackage(self, pid): + # order updated by trigger + self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) - @queue - def deleteLink(self, f): + @async + def deleteFile(self, fid, order, package): + """ To delete a file order and package of it is needed """ + self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) + self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', + (order, package)) - self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) - self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid))) + @async + def addCollector(self, plugin, package, data): + """ fill collector, data as (name, size, status,[ hash,] url) list """ + if data and len(data[0]) == 4: + data = [(r[0], r[1], r[2], r[3], plugin, package) for r in data] + else: + data = [(r[0], r[1], r[2], r[4], plugin, package) for r in data] + self.c.executemany("INSERT INTO collector(name, size, status, url, plugin, packagename) VALUES (?,?,?,?,?,?)", + data) - @queue - def getAllLinks(self, q): - """return information about all links in queue q + @async + def deleteCollector(self, package=None, url=None): + qry = 'DELETE FROM collector' + if package: + self.c.execute(qry + " WHERE packagename=?", (package,)) + elif url: + self.c.execute(qry + " WHERE url=?", (url,)) + else: + self.c.execute(qry) - q0 queue - q1 collector + @queue + def getCollector(self, package=None): + """ get collector data, optionally filtered by package """ + qry = 'SELECT url, name, plugin, size, status, packagename FROM collector' + if package: + self.c.execute(qry + " WHERE packagename=?", (package,)) + else: + self.c.execute(qry) - format: + return [LinkStatus(*r) for r in self.c] - { - id: {'name': name, ... 'package': id }, ... - } + @queue + def getAllFiles(self, package=None, search=None, unfinished=False): + """ Return dict with file information + :param package: optional package to filter out + :param search: or search string for file name + :param unfinished: filter by dlstatus not finished """ - self.c.execute('SELECT l.id,l.url,l.name,l.size,l.status,l.error,l.plugin,l.package,l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', (q,)) - data = {} + qry = ('SELECT fid, name, size, status, media, added, fileorder, ' + 'url, plugin, hash, dlstatus, error, package FROM files') + + if unfinished: + qry += ' WHERE dlstatus NOT IN (0, 5, 6)' + + if package is not None: + qry += ' AND' if unfinished else ' WHERE' + self.c.execute(qry + ' package=? ORDER BY package, fileorder', (package,)) + elif search is not None: + qry += ' AND' if unfinished else ' WHERE' + search = "%%%s%%" % search.strip("%") + self.c.execute(qry + ' name LIKE ? ORDER BY package, fileorder', (search,)) + + else: + self.c.execute(qry) + + data = OrderedDict() for r in self.c: - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': r[6], - 'package': r[7], - 'order': r[8], - } + f = FileInfo(r[0], r[1], r[12], r[2], r[3], r[4], r[5], r[6]) + if r[10] > 0: # dl status != NA + f.download = DownloadInfo(r[7], r[8], r[9], r[10], self.manager.statusMsg[r[10]], r[11]) + + data[r[0]] = f return data @queue - def getAllPackages(self, q): - """return information about packages in queue q - (only useful in get all data) + def getAllPackages(self, root=None): + """ Return dict with package information + + :param root: optional root to filter + """ + qry = ('SELECT pid, name, folder, root, site, comment, password, added, status, packageorder ' + 'FROM packages%s ORDER BY root, packageorder') - q0 queue - q1 collector + if root is None: + stats = self.getPackageStats() + self.c.execute(qry % "") + else: + stats = self.getPackageStats(root=root) + self.c.execute(qry % ' WHERE root=? OR pid=?', (root, root)) - format: + data = OrderedDict() + for r in self.c: + data[r[0]] = PackageInfo( + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], stats.get(r[0], default) + ) - { - id: {'name': name ... 'links': {} }, ... - } - """ - self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ - FROM packages p LEFT OUTER JOIN pstats s ON p.id = s.id \ - WHERE p.queue=? ORDER BY p.packageorder', str(q)) + return data + + @inner + def getPackageStats(self, pid=None, root=None): + qry = ("SELECT p.pid, SUM(f.size) AS sizetotal, COUNT(f.fid) AS linkstotal, sizedone, linksdone " + "FROM packages p JOIN files f ON p.pid = f.package AND f.dlstatus > 0 %(sub)s LEFT OUTER JOIN " + "(SELECT p.pid AS pid, SUM(f.size) AS sizedone, COUNT(f.fid) AS linksdone " + "FROM packages p JOIN files f ON p.pid = f.package %(sub)s AND f.dlstatus in (5,6) GROUP BY p.pid) s ON s.pid = p.pid " + "GROUP BY p.pid") + + # status in (finished, skipped, processing) + + if root is not None: + self.c.execute(qry % {"sub": "AND (p.root=:root OR p.pid=:root)"}, locals()) + elif pid is not None: + self.c.execute(qry % {"sub": "AND p.pid=:pid"}, locals()) + else: + self.c.execute(qry % {"sub": ""}) data = {} for r in self.c: - data[r[0]] = { - 'id': r[0], - 'name': r[1], - 'folder': r[2], - 'site': r[3], - 'password': r[4], - 'queue': r[5], - 'order': r[6], - 'sizetotal': int(r[7]) if r[7] else 0, - 'sizedone': int(r[8]) if r[8] else 0, #these can be None - 'linksdone': r[9] if r[9] else 0, - 'linkstotal': r[10] if r[10] else 0, - 'links': {} - } + data[r[0]] = PackageStats( + r[2] if r[2] else 0, + r[4] if r[4] else 0, + int(r[1]) if r[1] else 0, + int(r[3]) if r[3] else 0, + ) return data - + @queue - def getLinkData(self, id): - """get link information as dict""" - self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE id=?', (str(id), )) - data = {} + def getStatsForPackage(self, pid): + return self.getPackageStats(pid=pid)[pid] + + @queue + def getFileInfo(self, fid, force=False): + """get data for specific file""" + self.c.execute('SELECT fid, name, size, status, media, added, fileorder, ' + 'url, plugin, hash, dlstatus, error, package FROM files ' + 'WHERE fid=?', (fid,)) r = self.c.fetchone() if not r: return None - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': r[6], - 'package': r[7], - 'order': r[8], - } + else: + f = FileInfo(r[0], r[1], r[12], r[2], r[3], r[4], r[5], r[6]) + if r[10] > 0 or force: + f.download = DownloadInfo(r[7], r[8], r[9], r[10], self.manager.statusMsg[r[10]], r[11]) - return data + return f @queue - def getPackageData(self, id): - """get data about links for a package""" - self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id), )) + def getPackageInfo(self, pid, stats=True): + """get data for specific package, optional with package stats""" + if stats: + stats = self.getPackageStats(pid=pid) - data = {} - for r in self.c: - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': r[6], - 'package': r[7], - 'order': r[8], - } + self.c.execute('SELECT pid, name, folder, root, site, comment, password, added, status, packageorder ' + 'FROM packages WHERE pid=?', (pid,)) - return data + r = self.c.fetchone() + if not r: + return None + else: + return PackageInfo( + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], stats.get(r[0], default) if stats else None + ) + @async + def updateLinkInfo(self, data): + """ data is list of tupels (name, size, status,[ hash,] url)""" + if data and len(data[0]) == 4: + self.c.executemany('UPDATE files SET name=?, size=?, dlstatus=? WHERE url=? AND dlstatus IN (0,1,2,3,14)', + data) + else: + self.c.executemany( + 'UPDATE files SET name=?, size=?, dlstatus=?, hash=? WHERE url=? AND dlstatus IN (0,1,2,3,14)', data) @async - def updateLink(self, f): - self.c.execute('UPDATE links SET url=?,name=?,size=?,status=?,error=?,package=? WHERE id=?', (f.url, f.name, f.size, f.status, f.error, str(f.packageid), str(f.id))) + def updateFile(self, f): + self.c.execute('UPDATE files SET name=?, size=?, status=?,' + 'media=?, url=?, hash=?, dlstatus=?, error=? WHERE fid=?', + (f.name, f.size, f.filestatus, f.media, f.url, + f.hash, f.status, f.error, f.fid)) - @queue + @async def updatePackage(self, p): - self.c.execute('UPDATE packages SET name=?,folder=?,site=?,password=?,queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) - - @queue - def updateLinkInfo(self, data): - """ data is list of tupels (name, size, status, url) """ - self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1,2,3,14)', data) - ids = [] - self.c.execute('SELECT id FROM links WHERE url IN (\'%s\')' % "','".join([x[3] for x in data])) - for r in self.c: - ids.append(int(r[0])) - return ids - - @queue - def reorderPackage(self, p, position, noMove=False): - if position == -1: - position = self._nextPackageOrder(p.queue) - if not noMove: - if p.order > position: - self.c.execute('UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND queue=? AND packageorder >= 0', (position, p.order, p.queue)) - elif p.order < position: - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND queue=? AND packageorder >= 0', (position, p.order, p.queue)) - - self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) - - @queue - def reorderLink(self, f, position): - """ reorder link with f as dict for pyfile """ - if f["order"] > position: - self.c.execute('UPDATE links SET linkorder=linkorder+1 WHERE linkorder >= ? AND linkorder < ? AND package=?', (position, f["order"], f["package"])) - elif f["order"] < position: - self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder <= ? AND linkorder > ? AND package=?', (position, f["order"], f["package"])) - - self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"])) - - - @queue - def clearPackageOrder(self, p): - self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) - + self.c.execute('UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, status=? WHERE pid=?', + (p.name, p.folder, p.site, p.comment, p.password, p.status, p.pid)) + @async - def restartFile(self, id): - self.c.execute('UPDATE links SET status=3,error="" WHERE id=?', (str(id),)) + def orderPackage(self, pid, root, oldorder, order): + if oldorder > order: # package moved upwards + self.c.execute( + 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND root=? AND packageorder >= 0' + , (order, oldorder, root)) + elif oldorder < order: # moved downwards + self.c.execute( + 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND root=? AND packageorder >= 0' + , (order, oldorder, root)) + + self.c.execute('UPDATE packages SET packageorder=? WHERE pid=?', (order, pid)) @async - def restartPackage(self, id): - self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) - - @queue - def getPackage(self, id): - """return package instance from id""" - self.c.execute("SELECT name,folder,site,password,queue,packageorder FROM packages WHERE id=?", (str(id), )) + def orderFiles(self, pid, fids, oldorder, order): + diff = len(fids) + data = [] + + if oldorder > order: # moved upwards + self.c.execute('UPDATE files SET fileorder=fileorder+? WHERE fileorder >= ? AND fileorder < ? AND package=?' + , (diff, order, oldorder, pid)) + data = [(order + i, fid) for i, fid in enumerate(fids)] + elif oldorder < order: + self.c.execute( + 'UPDATE files SET fileorder=fileorder-? WHERE fileorder <= ? AND fileorder >= ? AND package=?' + , (diff, order, oldorder + diff, pid)) + data = [(order - diff + i + 1, fid) for i, fid in enumerate(fids)] + + self.c.executemany('UPDATE files SET fileorder=? WHERE fid=?', data) + + @async + def moveFiles(self, pid, fids, package): + self.c.execute('SELECT max(fileorder) FROM files WHERE package=?', (package,)) r = self.c.fetchone() - if not r: return None - return PyPackage(self.manager, id, * r) + order = (r[0] if r[0] else 0) + 1 + self.c.execute('UPDATE files SET fileorder=fileorder-? WHERE fileorder > ? AND package=?', + (len(fids), order, pid)) - @queue - def getFile(self, id): - """return link instance from id""" - self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id), )) + data = [(package, order + i, fid) for i, fid in enumerate(fids)] + self.c.executemany('UPDATE files SET package=?, fileorder=? WHERE fid=?', data) + + @async + def movePackage(self, root, order, pid, dpid): + self.c.execute('SELECT max(packageorder) FROM packages WHERE root=?', (dpid,)) r = self.c.fetchone() - if not r: return None - return PyFile(self.manager, id, * r) + max = (r[0] if r[0] else 0) + 1 + print max + self.c.execute('SELECT pid, packageorder FROM packages WHERE root=?', (dpid,)) + for r in self.c: + print r + + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND root=?', + (order, root)) + + self.c.execute('UPDATE packages SET root=?, packageorder=? WHERE pid=?', (dpid, max, pid)) + + @async + def restartFile(self, fid): + # status -> queued + self.c.execute('UPDATE files SET dlstatus=3, error="" WHERE fid=?', (fid,)) + + @async + def restartPackage(self, pid): + # status -> queued + self.c.execute('UPDATE files SET status=3 WHERE package=?', (pid,)) @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" - cmd = "(" - for i, item in enumerate(occ): - if i: cmd += ", " - cmd += "'%s'" % item + cmd = "(%s)" % ", ".join(["'%s'" % x for x in occ]) + #TODO - cmd += ")" + # dlstatus in online, queued | package status = ok + cmd = ("SELECT f.fid FROM files as f INNER JOIN packages as p ON f.package=p.pid " + "WHERE f.plugin NOT IN %s AND f.dlstatus IN (2,3) AND p.status=0 " + "ORDER BY p.packageorder ASC, f.fileorder ASC LIMIT 5") % cmd - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=1 AND l.plugin NOT IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % cmd self.c.execute(cmd) # very bad! return [x[0] for x in self.c] @@ -796,79 +348,34 @@ class FileMethods(): @queue def getUnfinished(self, pid): """return list of max length 3 ids with pyfiles in package not finished or processed""" - - self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) - return [r[0] for r in self.c] - @queue - def deleteFinished(self): - self.c.execute("DELETE FROM links WHERE status IN (0,4)") - self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") + # status in finished, skipped, processing + self.c.execute("SELECT fid FROM files WHERE package=? AND dlstatus NOT IN (5, 6, 14) LIMIT 3", (pid,)) + return [r[0] for r in self.c] @queue def restartFailed(self): - self.c.execute("UPDATE links SET status=3,error='' WHERE status IN (8, 9)") + # status=queued, where status in failed, aborted, temp offline + self.c.execute("UPDATE files SET dlstatus=3, error='' WHERE dlstatus IN (7, 11, 12)") @queue def findDuplicates(self, id, folder, filename): """ checks if filename exists with different id and same package """ - self.c.execute("SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", (folder, id, filename)) + # TODO + self.c.execute( + "SELECT l.plugin FROM files f INNER JOIN packages as p ON f.package=p.pid AND p.folder=? WHERE f.fid!=? AND l.status=0 AND l.name=?" + , (folder, id, filename)) return self.c.fetchone() @queue def purgeLinks(self): - self.c.execute("DELETE FROM links;") - self.c.execute("DELETE FROM packages;") - -DatabaseBackend.registerSub(FileMethods) - -if __name__ == "__main__": - - pypath = "." - _ = lambda x: x - - db = FileHandler(None) - - #p = PyFile(db, 5) - #sleep(0.1) - - a = time() - - #print db.addPackage("package", "folder" , 1) - - pack = db.db.addPackage("package", "folder", 1) - - updates = [] - - - for x in range(0, 200): - x = str(x) - db.db.addLink("http://somehost.com/hoster/file/download?file_id=" + x, x, "BasePlugin", pack) - updates.append(("new name" + x, 0, 3, "http://somehost.com/hoster/file/download?file_id=" + x)) - - - for x in range(0, 100): - updates.append(("unimportant%s" % x, 0, 3, "a really long non existent url%s" % x)) - - db.db.commit() - - b = time() - print "adding 200 links, single sql execs, no commit", b-a - - print db.getCompleteData(1) - - c = time() - - - db.db.updateLinkInfo(updates) - - d = time() - - print "updates", d-c - - print db.getCompleteData(1) - - - e = time() - - print "complete data", e-d + # fstatus = missing + self.c.execute("DELETE FROM files WHERE status == 1") + + @queue + def purgeAll(self): # only used for debugging + self.c.execute("DELETE FROM packages") + self.c.execute("DELETE FROM files") + self.c.execute("DELETE FROM collector") + +FileMethods.register() \ No newline at end of file diff --git a/module/database/UserDatabase.py b/module/database/UserDatabase.py index 43fd93df3..6bfb02bbd 100644 --- a/module/database/UserDatabase.py +++ b/module/database/UserDatabase.py @@ -19,14 +19,13 @@ from hashlib import sha1 import random -from DatabaseBackend import DatabaseBackend, queue, async +from DatabaseBackend import DatabaseMethods, queue, async -class UserMethods(): +class UserMethods(DatabaseMethods): @queue - def checkAuth(db, user, password): - c = db.c - c.execute('SELECT rowid, name, password, role, permission, template, email FROM "users" WHERE name=?', (user, )) - r = c.fetchone() + def checkAuth(self, user, password): + self.c.execute('SELECT rowid, name, password, role, permission, template, email FROM "users" WHERE name=?', (user, )) + r = self.c.fetchone() if not r: return {} @@ -40,23 +39,22 @@ class UserMethods(): return {} @queue - def addUser(db, user, password): + def addUser(self, user, password): salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) h = sha1(salt + password) password = salt + h.hexdigest() - c = db.c - c.execute('SELECT name FROM users WHERE name=?', (user, )) - if c.fetchone() is not None: - c.execute('UPDATE users SET password=? WHERE name=?', (password, user)) + self.c.execute('SELECT name FROM users WHERE name=?', (user, )) + if self.c.fetchone() is not None: + self.c.execute('UPDATE users SET password=? WHERE name=?', (password, user)) else: - c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) + self.c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) @queue - def changePassword(db, user, oldpw, newpw): - db.c.execute('SELECT rowid, name, password FROM users WHERE name=?', (user, )) - r = db.c.fetchone() + def changePassword(self, user, oldpw, newpw): + self.c.execute('SELECT rowid, name, password FROM users WHERE name=?', (user, )) + r = self.c.fetchone() if not r: return False @@ -68,40 +66,40 @@ class UserMethods(): h = sha1(salt + newpw) password = salt + h.hexdigest() - db.c.execute("UPDATE users SET password=? WHERE name=?", (password, user)) + self.c.execute("UPDATE users SET password=? WHERE name=?", (password, user)) return True return False @async - def setPermission(db, user, perms): - db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) + def setPermission(self, user, perms): + self.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) @async - def setRole(db, user, role): - db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) + def setRole(self, user, role): + self.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) @queue - def listUsers(db): - db.c.execute('SELECT name FROM users') + def listUsers(self): + self.c.execute('SELECT name FROM users') users = [] - for row in db.c: + for row in self.c: users.append(row[0]) return users @queue - def getAllUserData(db): - db.c.execute("SELECT name, permission, role, template, email FROM users") + def getAllUserData(self): + self.c.execute("SELECT name, permission, role, template, email FROM users") user = {} - for r in db.c: + for r in self.c: user[r[0]] = {"permission": r[1], "role": r[2], "template": r[3], "email": r[4]} return user @queue - def removeUser(db, user): - db.c.execute('DELETE FROM users WHERE name=?', (user, )) + def removeUser(self, user): + self.c.execute('DELETE FROM users WHERE name=?', (user, )) -DatabaseBackend.registerSub(UserMethods) +UserMethods.register() diff --git a/module/database/__init__.py b/module/database/__init__.py index 39848ac58..bf4ead872 100644 --- a/module/database/__init__.py +++ b/module/database/__init__.py @@ -1,6 +1,6 @@ -from DatabaseBackend import DatabaseBackend, queue, async, inner +from DatabaseBackend import DatabaseMethods, DatabaseBackend, queue, async, inner -from FileDatabase import FileHandler +from FileDatabase import FileMethods from UserDatabase import UserMethods from StorageDatabase import StorageMethods from AccountDatabase import AccountMethods \ No newline at end of file -- cgit v1.2.3 From b40b32ee05f611323a7827fad2a25fa0a28dcb24 Mon Sep 17 00:00:00 2001 From: X3n0m0rph59 Date: Sun, 22 Apr 2012 19:56:17 +0200 Subject: a huge pile of spelling fixes --- module/database/DatabaseBackend.py | 6 +++--- module/database/FileDatabase.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'module/database') diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 8159446bd..97ecec3ab 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -131,7 +131,7 @@ class DatabaseBackend(Thread): Thread.__init__(self) self.setDaemon(True) self.core = core - self.manager = None # setted later + self.manager = None # set later self.running = Event() self.jobs = Queue() @@ -162,9 +162,9 @@ class DatabaseBackend(Thread): self.conn.close() try: - self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) + self.manager.core.log.warning(_("File database was deleted due to incompatible version.")) except: - print "Filedatabase was deleted due to incompatible version." + print "File database was deleted due to incompatible version." remove(self.VERSION_FILE) move(self.DB_FILE, self.DB_FILE + ".backup") diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 08b18765d..19dca84c7 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -40,27 +40,27 @@ class FileMethods(DatabaseMethods): @queue def processcount(self, fid): - """ number of files which have to be proccessed """ + """ number of files which have to be processed """ # status in online, queued, starting, waiting, downloading self.c.execute("SELECT COUNT(*) FROM files as WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (str(fid), )) return self.c.fetchone()[0] @queue def addLink(self, url, name, plugin, package): - # mark filestatus initially as missing, dlstatus - queued + # mark file status initially as missing, dlstatus - queued self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', (url, name, plugin, package)) return self.c.lastrowid @async def addLinks(self, links, package): - """ links is a list of tupels (url, plugin)""" + """ links is a list of tuples (url, plugin)""" links = [(x[0], x[0], x[1], package) for x in links] self.c.executemany('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', links) @queue def addFile(self, name, size, media, package): - # filestatus - ok, dl status NA + # file status - ok, dl status NA self.c.execute('INSERT INTO files(name, size, media, package) VALUES(?,?,?,?)', (name, size, media, package)) return self.c.lastrowid @@ -225,7 +225,7 @@ class FileMethods(DatabaseMethods): @queue def getPackageInfo(self, pid, stats=True): - """get data for specific package, optional with package stats""" + """get data for a specific package, optionally with package stats""" if stats: stats = self.getPackageStats(pid=pid) @@ -242,7 +242,7 @@ class FileMethods(DatabaseMethods): @async def updateLinkInfo(self, data): - """ data is list of tupels (name, size, status,[ hash,] url)""" + """ data is list of tuples (name, size, status,[ hash,] url)""" if data and len(data[0]) == 4: self.c.executemany('UPDATE files SET name=?, size=?, dlstatus=? WHERE url=? AND dlstatus IN (0,1,2,3,14)', data) -- cgit v1.2.3 From f36bb35cf296c74ff5a3676c038e2ef2a8be9068 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 6 May 2012 17:22:13 +0200 Subject: concept for multiuser api --- module/database/DatabaseBackend.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'module/database') diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 97ecec3ab..18898a93c 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -327,6 +327,7 @@ class DatabaseBackend(Thread): '"password" TEXT NOT NULL, ' '"role" INTEGER DEFAULT 0 NOT NULL, ' '"permission" INTEGER DEFAULT 0 NOT NULL, ' + '"folder" TEXT DEFAULT "" NOT NULL, ' '"template" TEXT DEFAULT "default" NOT NULL' ')' ) @@ -338,6 +339,8 @@ class DatabaseBackend(Thread): '"activated" INTEGER DEFAULT 1, ' '"password" TEXT DEFAULT "", ' '"options" TEXT DEFAULT "", ' +# '"user" TEXT NOT NULL, ' +# 'FOREIGN KEY(user) REFERENCES users(name)' 'PRIMARY KEY (plugin, loginname) ON CONFLICT REPLACE' ')' ) -- cgit v1.2.3 From 3d7ed0c367cf521b61ec1f78784dec73a3b7c32a Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 6 May 2012 20:40:39 +0200 Subject: some classes for multi user mode --- module/database/DatabaseBackend.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) (limited to 'module/database') diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 18898a93c..6373120ff 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -322,13 +322,36 @@ class DatabaseBackend(Thread): self.c.execute( 'CREATE TABLE IF NOT EXISTS "users" (' - '"name" TEXT PRIMARY KEY NOT NULL, ' + '"id" INTEGER PRIMARY KEY AUTOINCREMENT, ' + '"name" TEXT NOT NULL, ' '"email" TEXT DEFAULT "" NOT NULL, ' '"password" TEXT NOT NULL, ' '"role" INTEGER DEFAULT 0 NOT NULL, ' '"permission" INTEGER DEFAULT 0 NOT NULL, ' '"folder" TEXT DEFAULT "" NOT NULL, ' - '"template" TEXT DEFAULT "default" NOT NULL' + '"ratio" INTEGER DEFAULT -1 NOT NULL, ' + '"limit" INTEGER DEFAULT -1 NOT NULL, ' + '"template" TEXT DEFAULT "default" NOT NULL, ' + '"user" INTEGER DEFAULT -1 NOT NULL, ' + 'FOREIGN KEY(user) REFERENCES users(id)' + ')' + ) + + self.c.execute( + 'CREATE TRIGGER IF NOT EXISTS "insert_user" AFTER INSERT ON "users"' + 'BEGIN ' + 'UPDATE users SET user = new.id ' + 'WHERE rowid = new.rowid;' + 'END' + ) + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "settings" (' + '"plugin" TEXT NOT NULL, ' + '"owner" INTEGER NOT NULL, ' + '"configuration" TEXT NOT NULL, ' + 'FOREIGN KEY(owner) REFERENCES users(id), ' + 'PRIMARY KEY (plugin, owner) ON CONFLICT REPLACE' ')' ) @@ -339,8 +362,8 @@ class DatabaseBackend(Thread): '"activated" INTEGER DEFAULT 1, ' '"password" TEXT DEFAULT "", ' '"options" TEXT DEFAULT "", ' -# '"user" TEXT NOT NULL, ' -# 'FOREIGN KEY(user) REFERENCES users(name)' +# '"owner" INTEGER NOT NULL, ' TODO: shared, owner attribute +# 'FOREIGN KEY(owner) REFERENCES users(id)' 'PRIMARY KEY (plugin, loginname) ON CONFLICT REPLACE' ')' ) -- cgit v1.2.3 From 84efa9d5ccd46a0adddc256a5eba4f8e33c76afd Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 7 May 2012 18:42:29 +0200 Subject: new multiuser api methods --- module/database/DatabaseBackend.py | 34 +++++++++++++++++----------------- module/database/FileDatabase.py | 38 +++++++++++--------------------------- 2 files changed, 28 insertions(+), 44 deletions(-) (limited to 'module/database') diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 6373120ff..516aa981f 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -301,13 +301,10 @@ class DatabaseBackend(Thread): self.c.execute( 'CREATE TABLE IF NOT EXISTS "collector" (' - '"url" TEXT NOT NULL, ' - '"name" TEXT NOT NULL, ' - '"plugin" TEXT DEFAULT "BasePlugin" NOT NULL, ' - '"size" INTEGER DEFAULT 0 NOT NULL, ' - '"status" INTEGER DEFAULT 3 NOT NULL, ' - '"packagename" TEXT DEFAULT "" NOT NULL, ' - 'PRIMARY KEY (url, packagename) ON CONFLICT REPLACE' + '"owner" INTEGER NOT NULL, ' + '"data" TEXT NOT NULL, ' + 'FOREIGN KEY(owner) REFERENCES users(uid)' + 'PRIMARY KEY(owner) ON CONFLICT REPLACE' ') ' ) @@ -322,25 +319,27 @@ class DatabaseBackend(Thread): self.c.execute( 'CREATE TABLE IF NOT EXISTS "users" (' - '"id" INTEGER PRIMARY KEY AUTOINCREMENT, ' + '"uid" INTEGER PRIMARY KEY AUTOINCREMENT, ' '"name" TEXT NOT NULL, ' '"email" TEXT DEFAULT "" NOT NULL, ' '"password" TEXT NOT NULL, ' '"role" INTEGER DEFAULT 0 NOT NULL, ' '"permission" INTEGER DEFAULT 0 NOT NULL, ' '"folder" TEXT DEFAULT "" NOT NULL, ' - '"ratio" INTEGER DEFAULT -1 NOT NULL, ' - '"limit" INTEGER DEFAULT -1 NOT NULL, ' + '"traffic" INTEGER DEFAULT -1 NOT NULL, ' + '"dllimit" INTEGER DEFAULT -1 NOT NULL, ' '"template" TEXT DEFAULT "default" NOT NULL, ' - '"user" INTEGER DEFAULT -1 NOT NULL, ' - 'FOREIGN KEY(user) REFERENCES users(id)' + '"user" INTEGER DEFAULT -1 NOT NULL, ' # set by trigger to self + 'FOREIGN KEY(user) REFERENCES users(uid)' ')' ) + self.c.execute('CREATE INDEX IF NOT EXISTS "username_index" ON users(name)') + self.c.execute( 'CREATE TRIGGER IF NOT EXISTS "insert_user" AFTER INSERT ON "users"' 'BEGIN ' - 'UPDATE users SET user = new.id ' + 'UPDATE users SET user = new.uid, folder=new.name ' 'WHERE rowid = new.rowid;' 'END' ) @@ -350,7 +349,7 @@ class DatabaseBackend(Thread): '"plugin" TEXT NOT NULL, ' '"owner" INTEGER NOT NULL, ' '"configuration" TEXT NOT NULL, ' - 'FOREIGN KEY(owner) REFERENCES users(id), ' + 'FOREIGN KEY(owner) REFERENCES users(uid), ' 'PRIMARY KEY (plugin, owner) ON CONFLICT REPLACE' ')' ) @@ -359,12 +358,13 @@ class DatabaseBackend(Thread): 'CREATE TABLE IF NOT EXISTS "accounts" (' '"plugin" TEXT NOT NULL, ' '"loginname" TEXT NOT NULL, ' + '"owner", INTEGER NOT NULL, ' '"activated" INTEGER DEFAULT 1, ' '"password" TEXT DEFAULT "", ' + '"shared" INTEGER DEFAULT 0, ' '"options" TEXT DEFAULT "", ' -# '"owner" INTEGER NOT NULL, ' TODO: shared, owner attribute -# 'FOREIGN KEY(owner) REFERENCES users(id)' - 'PRIMARY KEY (plugin, loginname) ON CONFLICT REPLACE' + 'FOREIGN KEY(owner) REFERENCES users(uid)' + 'PRIMARY KEY (plugin, loginname, owner) ON CONFLICT REPLACE' ')' ) diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 19dca84c7..b783d15d9 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -85,36 +85,20 @@ class FileMethods(DatabaseMethods): (order, package)) @async - def addCollector(self, plugin, package, data): - """ fill collector, data as (name, size, status,[ hash,] url) list """ - if data and len(data[0]) == 4: - data = [(r[0], r[1], r[2], r[3], plugin, package) for r in data] - else: - data = [(r[0], r[1], r[2], r[4], plugin, package) for r in data] - - self.c.executemany("INSERT INTO collector(name, size, status, url, plugin, packagename) VALUES (?,?,?,?,?,?)", - data) - - @async - def deleteCollector(self, package=None, url=None): - qry = 'DELETE FROM collector' - if package: - self.c.execute(qry + " WHERE packagename=?", (package,)) - elif url: - self.c.execute(qry + " WHERE url=?", (url,)) - else: - self.c.execute(qry) + def saveCollector(self, owner, data): + """ simply save the json string to database """ + self.c.execute("INSERT INTO collector(owner, data) VALUES (?,?)", (owner, data)) @queue - def getCollector(self, package=None): - """ get collector data, optionally filtered by package """ - qry = 'SELECT url, name, plugin, size, status, packagename FROM collector' - if package: - self.c.execute(qry + " WHERE packagename=?", (package,)) - else: - self.c.execute(qry) + def retrieveCollector(self, owner): + """ retrieve the saved string """ + self.c.execute('SELECT data FROM collector owner=?', (owner,)) + return self.c.fetchone()[0] - return [LinkStatus(*r) for r in self.c] + @async + def deleteCollector(self, owner): + """ drop saved user collector """ + self.c.execute('DELETE FROM collector WHERE owner=?', (owner,)) @queue def getAllFiles(self, package=None, search=None, unfinished=False): -- cgit v1.2.3 From 829244a6140763712d50ed046c33f415f2b04301 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 15 May 2012 19:22:34 +0200 Subject: some multiuser db changes --- module/database/DatabaseBackend.py | 48 +++++----- module/database/FileDatabase.py | 184 +++++++++++++++++++++---------------- 2 files changed, 132 insertions(+), 100 deletions(-) (limited to 'module/database') diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 516aa981f..ec39e3fd9 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -1,21 +1,21 @@ #!/usr/bin/env python -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN - @author: mkaay -""" +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +# @author: mkaay +############################################################################### + from threading import Thread, Event from shutil import move @@ -245,8 +245,10 @@ class DatabaseBackend(Thread): '"added" INTEGER DEFAULT 0 NOT NULL,' # set by trigger '"status" INTEGER DEFAULT 0 NOT NULL,' '"packageorder" INTEGER DEFAULT -1 NOT NULL,' #incremented by trigger - '"root" INTEGER DEFAULT -1 NOT NULL,' - 'CHECK (root != pid) ' + '"root" INTEGER DEFAULT -1 NOT NULL, ' + '"owner" INTEGER NOT NULL, ' + 'FOREIGN KEY(owner) REFERENCES users(uid), ' + 'CHECK (root != pid)' ')' ) @@ -267,7 +269,8 @@ class DatabaseBackend(Thread): 'END' ) - self.c.execute('CREATE INDEX IF NOT EXISTS "root_index" ON packages(root)') + self.c.execute('CREATE INDEX IF NOT EXISTS "package_index" ON packages(root, owner)') + self.c.execute('CREATE INDEX IF NOT EXISTS "package_owner" ON packages(owner)') self.c.execute( 'CREATE TABLE IF NOT EXISTS "files" (' @@ -284,11 +287,14 @@ class DatabaseBackend(Thread): '"dlstatus" INTEGER DEFAULT 0 NOT NULL, ' '"error" TEXT DEFAULT "" NOT NULL, ' '"package" INTEGER NOT NULL, ' + '"owner" INTEGER NOT NULL, ' + 'FOREIGN KEY(owner) REFERENCES users(uid), ' 'FOREIGN KEY(package) REFERENCES packages(id)' ')' ) - self.c.execute('CREATE INDEX IF NOT EXISTS "package_index" ON files(package)') + self.c.execute('CREATE INDEX IF NOT EXISTS "file_index" ON files(package, owner)') + self.c.execute('CREATE INDEX IF NOT EXISTS "file_owner" ON files(owner)') self.c.execute( 'CREATE TRIGGER IF NOT EXISTS "insert_file" AFTER INSERT ON "files"' diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index b783d15d9..80da775c7 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -1,88 +1,97 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + from new_collections import OrderedDict -from module.Api import DownloadInfo, LinkStatus, FileInfo, PackageInfo, PackageStats +from module.Api import DownloadInfo, FileInfo, PackageInfo, PackageStats from module.database import DatabaseMethods, queue, async, inner -default = PackageStats(0, 0, 0, 0) +zero_stats = PackageStats(0, 0, 0, 0) class FileMethods(DatabaseMethods): @queue - def filecount(self): + def filecount(self, user=None): """returns number of files""" self.c.execute("SELECT COUNT(*) FROM files") return self.c.fetchone()[0] @queue - def queuecount(self): + def queuecount(self, user=None): """ number of files in queue not finished yet""" # status not in NA, finished, skipped self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus NOT IN (0,5,6)") return self.c.fetchone()[0] @queue - def processcount(self, fid): + def processcount(self, fid, user=None): """ number of files which have to be processed """ # status in online, queued, starting, waiting, downloading self.c.execute("SELECT COUNT(*) FROM files as WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (str(fid), )) return self.c.fetchone()[0] + # TODO: think about multiuser side effects on *count methods + @queue - def addLink(self, url, name, plugin, package): + def addLink(self, url, name, plugin, package, owner): # mark file status initially as missing, dlstatus - queued - self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', - (url, name, plugin, package)) + self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', + (url, name, plugin, package, owner)) return self.c.lastrowid @async - def addLinks(self, links, package): + def addLinks(self, links, package, owner): """ links is a list of tuples (url, plugin)""" - links = [(x[0], x[0], x[1], package) for x in links] - self.c.executemany('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', links) + links = [(x[0], x[0], x[1], package, owner) for x in links] + self.c.executemany('INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', + links) @queue - def addFile(self, name, size, media, package): + def addFile(self, name, size, media, package, owner): # file status - ok, dl status NA - self.c.execute('INSERT INTO files(name, size, media, package) VALUES(?,?,?,?)', - (name, size, media, package)) + self.c.execute('INSERT INTO files(name, size, media, package, owner) VALUES(?,?,?,?,?)', + (name, size, media, package, owner)) return self.c.lastrowid @queue - def addPackage(self, name, folder, root, password, site, comment, status): - self.c.execute('INSERT INTO packages(name, folder, root, password, site, comment, status) VALUES(?,?,?,?,?,?,?)' - , - (name, folder, root, password, site, comment, status)) + def addPackage(self, name, folder, root, password, site, comment, status, owner): + self.c.execute( + 'INSERT INTO packages(name, folder, root, password, site, comment, status, owner) VALUES(?,?,?,?,?,?,?,?)' + , (name, folder, root, password, site, comment, status, owner)) return self.c.lastrowid @async - def deletePackage(self, pid): - # order updated by trigger - self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) + def deletePackage(self, pid, owner=None): + # order updated by trigge + if owner is None: + self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) + else: + self.c.execute('DELETE FROM packages WHERE pid=? AND owner=?', (pid, owner)) @async - def deleteFile(self, fid, order, package): + def deleteFile(self, fid, order, package, owner=None): """ To delete a file order and package of it is needed """ - self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) - self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', - (order, package)) + if owner is None: + self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) + self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', + (order, package)) + else: + self.c.execute('DELETE FROM files WHERE fid=? AND owner=?', (fid, owner)) + self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=? AND owner=?', + (order, package, owner)) @async def saveCollector(self, owner, data): @@ -92,8 +101,10 @@ class FileMethods(DatabaseMethods): @queue def retrieveCollector(self, owner): """ retrieve the saved string """ - self.c.execute('SELECT data FROM collector owner=?', (owner,)) - return self.c.fetchone()[0] + self.c.execute('SELECT data FROM collector WHERE owner=?', (owner,)) + r = self.c.fetchone() + if not r: return None + return r[0] @async def deleteCollector(self, owner): @@ -101,66 +112,81 @@ class FileMethods(DatabaseMethods): self.c.execute('DELETE FROM collector WHERE owner=?', (owner,)) @queue - def getAllFiles(self, package=None, search=None, unfinished=False): + def getAllFiles(self, package=None, search=None, unfinished=False, owner=None): """ Return dict with file information :param package: optional package to filter out :param search: or search string for file name :param unfinished: filter by dlstatus not finished + :param owner: only specific owner """ - qry = ('SELECT fid, name, size, status, media, added, fileorder, ' - 'url, plugin, hash, dlstatus, error, package FROM files') + qry = ('SELECT fid, name, owner, size, status, media, added, fileorder, ' + 'url, plugin, hash, dlstatus, error, package FROM files WHERE ') + + arg = [] if unfinished: - qry += ' WHERE dlstatus NOT IN (0, 5, 6)' + qry += 'dlstatus NOT IN (0, 5, 6) AND ' + if owner is not None: + qry += 'owner=? AND ' + arg.append(owner) if package is not None: - qry += ' AND' if unfinished else ' WHERE' - self.c.execute(qry + ' package=? ORDER BY package, fileorder', (package,)) - elif search is not None: - qry += ' AND' if unfinished else ' WHERE' + arg.append(package) + qry += 'package=? AND ' + if search is not None: search = "%%%s%%" % search.strip("%") - self.c.execute(qry + ' name LIKE ? ORDER BY package, fileorder', (search,)) + arg.append(search) + qry += "name LIKE ? " - else: - self.c.execute(qry) + # make qry valid + if qry.endswith("WHERE "): qry = qry[:-6] + if qry.endswith("AND "): qry = qry[:-4] + + self.c.execute(qry + "ORDER BY package, fileorder", arg) data = OrderedDict() for r in self.c: - f = FileInfo(r[0], r[1], r[12], r[2], r[3], r[4], r[5], r[6]) - if r[10] > 0: # dl status != NA - f.download = DownloadInfo(r[7], r[8], r[9], r[10], self.manager.statusMsg[r[10]], r[11]) + f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) + if r[11] > 0: # dl status != NA + f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) data[r[0]] = f return data @queue - def getAllPackages(self, root=None): + def getAllPackages(self, root=None, owner=None): """ Return dict with package information :param root: optional root to filter """ - qry = ('SELECT pid, name, folder, root, site, comment, password, added, status, packageorder ' + qry = ('SELECT pid, name, folder, root, owner, site, comment, password, added, status, packageorder ' 'FROM packages%s ORDER BY root, packageorder') if root is None: - stats = self.getPackageStats() - self.c.execute(qry % "") + stats = self.getPackageStats(owner=owner) + if owner is None: + self.c.execute(qry % "") + else: + self.c.execute(qry % " WHERE owner=?", (owner,)) else: - stats = self.getPackageStats(root=root) - self.c.execute(qry % ' WHERE root=? OR pid=?', (root, root)) + stats = self.getPackageStats(root=root, owner=owner) + if owner is None: + self.c.execute(qry % ' WHERE root=? OR pid=?', (root, root)) + else: + self.c.execute(qry % ' WHERE (root=? OR pid=?) AND owner=?', (root, root, owner)) data = OrderedDict() for r in self.c: data[r[0]] = PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], stats.get(r[0], default) + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], stats.get(r[0], zero_stats) ) return data @inner - def getPackageStats(self, pid=None, root=None): + def getPackageStats(self, pid=None, root=None, owner=None): qry = ("SELECT p.pid, SUM(f.size) AS sizetotal, COUNT(f.fid) AS linkstotal, sizedone, linksdone " "FROM packages p JOIN files f ON p.pid = f.package AND f.dlstatus > 0 %(sub)s LEFT OUTER JOIN " "(SELECT p.pid AS pid, SUM(f.size) AS sizedone, COUNT(f.fid) AS linksdone " @@ -173,6 +199,8 @@ class FileMethods(DatabaseMethods): self.c.execute(qry % {"sub": "AND (p.root=:root OR p.pid=:root)"}, locals()) elif pid is not None: self.c.execute(qry % {"sub": "AND p.pid=:pid"}, locals()) + elif owner is not None: + self.c.execute(qry % {"sub": "AND p.owner=:owner"}, locals()) else: self.c.execute(qry % {"sub": ""}) @@ -193,17 +221,17 @@ class FileMethods(DatabaseMethods): @queue def getFileInfo(self, fid, force=False): - """get data for specific file""" - self.c.execute('SELECT fid, name, size, status, media, added, fileorder, ' + """get data for specific file, when force is true download info will be appended""" + self.c.execute('SELECT fid, name, owner, size, status, media, added, fileorder, ' 'url, plugin, hash, dlstatus, error, package FROM files ' 'WHERE fid=?', (fid,)) r = self.c.fetchone() if not r: return None else: - f = FileInfo(r[0], r[1], r[12], r[2], r[3], r[4], r[5], r[6]) - if r[10] > 0 or force: - f.download = DownloadInfo(r[7], r[8], r[9], r[10], self.manager.statusMsg[r[10]], r[11]) + f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) + if r[11] > 0 or force: + f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) return f @@ -213,7 +241,7 @@ class FileMethods(DatabaseMethods): if stats: stats = self.getPackageStats(pid=pid) - self.c.execute('SELECT pid, name, folder, root, site, comment, password, added, status, packageorder ' + self.c.execute('SELECT pid, name, folder, root, owner, site, comment, password, added, status, packageorder ' 'FROM packages WHERE pid=?', (pid,)) r = self.c.fetchone() @@ -221,11 +249,11 @@ class FileMethods(DatabaseMethods): return None else: return PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], stats.get(r[0], default) if stats else None + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], stats.get(r[0], zero_stats) if stats else None ) @async - def updateLinkInfo(self, data): + def updateLinkInfo(self, data, owner): """ data is list of tuples (name, size, status,[ hash,] url)""" if data and len(data[0]) == 4: self.c.executemany('UPDATE files SET name=?, size=?, dlstatus=? WHERE url=? AND dlstatus IN (0,1,2,3,14)', @@ -246,6 +274,7 @@ class FileMethods(DatabaseMethods): self.c.execute('UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, status=? WHERE pid=?', (p.name, p.folder, p.site, p.comment, p.password, p.status, p.pid)) + # TODO: most modifying methods needs owner argument to avoid checking beforehand @async def orderPackage(self, pid, root, oldorder, order): if oldorder > order: # package moved upwards @@ -293,11 +322,6 @@ class FileMethods(DatabaseMethods): self.c.execute('SELECT max(packageorder) FROM packages WHERE root=?', (dpid,)) r = self.c.fetchone() max = (r[0] if r[0] else 0) + 1 - print max - - self.c.execute('SELECT pid, packageorder FROM packages WHERE root=?', (dpid,)) - for r in self.c: - print r self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND root=?', (order, root)) @@ -314,6 +338,8 @@ class FileMethods(DatabaseMethods): # status -> queued self.c.execute('UPDATE files SET status=3 WHERE package=?', (pid,)) + + # TODO: multi user approach @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" @@ -338,7 +364,7 @@ class FileMethods(DatabaseMethods): return [r[0] for r in self.c] @queue - def restartFailed(self): + def restartFailed(self, owner): # status=queued, where status in failed, aborted, temp offline self.c.execute("UPDATE files SET dlstatus=3, error='' WHERE dlstatus IN (7, 11, 12)") -- cgit v1.2.3 From 0d2d6daef850ac6bcc7fafccd230e52d2a862c2c Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 3 Jun 2012 17:45:10 +0200 Subject: updates for database + api --- module/database/DatabaseBackend.py | 30 ++++++--- module/database/StatisticDatabase.py | 13 ++++ module/database/UserDatabase.py | 117 ++++++++++++++++++++--------------- 3 files changed, 100 insertions(+), 60 deletions(-) create mode 100644 module/database/StatisticDatabase.py (limited to 'module/database') diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index ec39e3fd9..2c494e520 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -5,15 +5,15 @@ # Copyright(c) 2008-2012 pyLoad Team # http://www.pyload.org # -# This program is free software: you can redistribute it and/or modify +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # # Subjected to the terms and conditions in LICENSE # -# @author: RaNaN -# @author: mkaay +# @author: RaNaN, mkaay ############################################################################### from threading import Thread, Event @@ -268,7 +268,6 @@ class DatabaseBackend(Thread): 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > old.packageorder AND root=old.pid;' 'END' ) - self.c.execute('CREATE INDEX IF NOT EXISTS "package_index" ON packages(root, owner)') self.c.execute('CREATE INDEX IF NOT EXISTS "package_owner" ON packages(owner)') @@ -292,7 +291,6 @@ class DatabaseBackend(Thread): 'FOREIGN KEY(package) REFERENCES packages(id)' ')' ) - self.c.execute('CREATE INDEX IF NOT EXISTS "file_index" ON files(package, owner)') self.c.execute('CREATE INDEX IF NOT EXISTS "file_owner" ON files(owner)') @@ -309,7 +307,7 @@ class DatabaseBackend(Thread): 'CREATE TABLE IF NOT EXISTS "collector" (' '"owner" INTEGER NOT NULL, ' '"data" TEXT NOT NULL, ' - 'FOREIGN KEY(owner) REFERENCES users(uid)' + 'FOREIGN KEY(owner) REFERENCES users(uid), ' 'PRIMARY KEY(owner) ON CONFLICT REPLACE' ') ' ) @@ -326,7 +324,7 @@ class DatabaseBackend(Thread): self.c.execute( 'CREATE TABLE IF NOT EXISTS "users" (' '"uid" INTEGER PRIMARY KEY AUTOINCREMENT, ' - '"name" TEXT NOT NULL, ' + '"name" TEXT NOT NULL UNIQUE, ' '"email" TEXT DEFAULT "" NOT NULL, ' '"password" TEXT NOT NULL, ' '"role" INTEGER DEFAULT 0 NOT NULL, ' @@ -334,12 +332,13 @@ class DatabaseBackend(Thread): '"folder" TEXT DEFAULT "" NOT NULL, ' '"traffic" INTEGER DEFAULT -1 NOT NULL, ' '"dllimit" INTEGER DEFAULT -1 NOT NULL, ' + '"dlquota" TEXT DEFAULT "" NOT NULL, ' + '"hddquota" INTEGER DEFAULT -1 NOT NULL, ' '"template" TEXT DEFAULT "default" NOT NULL, ' '"user" INTEGER DEFAULT -1 NOT NULL, ' # set by trigger to self 'FOREIGN KEY(user) REFERENCES users(uid)' ')' ) - self.c.execute('CREATE INDEX IF NOT EXISTS "username_index" ON users(name)') self.c.execute( @@ -369,11 +368,24 @@ class DatabaseBackend(Thread): '"password" TEXT DEFAULT "", ' '"shared" INTEGER DEFAULT 0, ' '"options" TEXT DEFAULT "", ' - 'FOREIGN KEY(owner) REFERENCES users(uid)' + 'FOREIGN KEY(owner) REFERENCES users(uid), ' 'PRIMARY KEY (plugin, loginname, owner) ON CONFLICT REPLACE' ')' ) + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "stats" (' + '"user" INTEGER NOT NULL, ' + '"plugin" TEXT NOT NULL, ' + '"time" INTEGER NOT NULL, ' + '"premium" INTEGER DEFAULT 0 NOT NULL, ' + '"amount" INTEGER DEFAULT 0 NOT NULL, ' + 'FOREIGN KEY(user) REFERENCES users(uid), ' + 'PRIMARY KEY(user, plugin, time)' + ')' + ) + self.c.execute('CREATE INDEX IF NOT EXISTS "stats_time" ON stats(time)') + #try to lower ids self.c.execute('SELECT max(fid) FROM files') fid = self.c.fetchone()[0] diff --git a/module/database/StatisticDatabase.py b/module/database/StatisticDatabase.py new file mode 100644 index 000000000..10619eb5b --- /dev/null +++ b/module/database/StatisticDatabase.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.database import DatabaseMethods, queue, async, inner + +# TODO + +class StatisticMethods(DatabaseMethods): + pass + + + +StatisticMethods.register() \ No newline at end of file diff --git a/module/database/UserDatabase.py b/module/database/UserDatabase.py index 6bfb02bbd..bed4e94a9 100644 --- a/module/database/UserDatabase.py +++ b/module/database/UserDatabase.py @@ -1,42 +1,28 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### from hashlib import sha1 import random +from module.Api import UserData + from DatabaseBackend import DatabaseMethods, queue, async class UserMethods(DatabaseMethods): - @queue - def checkAuth(self, user, password): - self.c.execute('SELECT rowid, name, password, role, permission, template, email FROM "users" WHERE name=?', (user, )) - r = self.c.fetchone() - if not r: - return {} - - salt = r[2][:5] - pw = r[2][5:] - h = sha1(salt + password) - if h.hexdigest() == pw: - return {"id": r[0], "name": r[1], "role": r[3], - "permission": r[4], "template": r[5], "email": r[6]} - else: - return {} @queue def addUser(self, user, password): @@ -50,8 +36,53 @@ class UserMethods(DatabaseMethods): else: self.c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) + @queue + def getUserData(self, name=None, uid=None): + qry = ('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' + 'hddquota, user, template FROM "users" WHERE ') + + if name is not None: + self.c.execute(qry + "name=?", (name,)) + r = self.c.fetchone() + if r: + return UserData(*r) + + elif uid is not None: + self.c.execute(qry + "uid=?", (uid,)) + r = self.c.fetchone() + if r: + return UserData(*r) + + return None @queue + def getAllUserData(self): + self.c.execute('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' + 'hddquota, user, template FROM "users"') + user = {} + for r in self.c: + user[r[0]] = UserData(*r) + + return user + + + @queue + def checkAuth(self, user, password): + self.c.execute('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' + 'hddquota, user, template password FROM "users" WHERE name=?', (user, )) + r = self.c.fetchone() + if not r: + return None + + salt = r[-1][:5] + pw = r[-1][5:] + h = sha1(salt + password) + if h.hexdigest() == pw: + return UserData(*r[:-1]) + else: + return None + + @queue #TODO def changePassword(self, user, oldpw, newpw): self.c.execute('SELECT rowid, name, password FROM users WHERE name=?', (user, )) r = self.c.fetchone() @@ -71,7 +102,6 @@ class UserMethods(DatabaseMethods): return False - @async def setPermission(self, user, perms): self.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) @@ -80,26 +110,11 @@ class UserMethods(DatabaseMethods): def setRole(self, user, role): self.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) + # TODO update methods - @queue - def listUsers(self): - self.c.execute('SELECT name FROM users') - users = [] - for row in self.c: - users.append(row[0]) - return users - - @queue - def getAllUserData(self): - self.c.execute("SELECT name, permission, role, template, email FROM users") - user = {} - for r in self.c: - user[r[0]] = {"permission": r[1], "role": r[2], "template": r[3], "email": r[4]} - - return user - - @queue - def removeUser(self, user): - self.c.execute('DELETE FROM users WHERE name=?', (user, )) + @async + def removeUser(self, uid=None): + # deletes user and all associated accounts + self.c.execute('DELETE FROM users WHERE user=?', (uid, )) UserMethods.register() -- cgit v1.2.3 From 941e3021000e59020f66419cc2156aee30972121 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 13 Aug 2012 17:40:10 +0200 Subject: working login --- module/database/UserDatabase.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'module/database') diff --git a/module/database/UserDatabase.py b/module/database/UserDatabase.py index bed4e94a9..0df94e0eb 100644 --- a/module/database/UserDatabase.py +++ b/module/database/UserDatabase.py @@ -16,17 +16,23 @@ ############################################################################### from hashlib import sha1 -import random +from string import letters, digits +from random import choice + +alphnum = letters+digits from module.Api import UserData from DatabaseBackend import DatabaseMethods, queue, async +def random_salt(): + return "".join(choice(alphnum) for x in range(0,5)) + class UserMethods(DatabaseMethods): @queue def addUser(self, user, password): - salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) + salt = random_salt() h = sha1(salt + password) password = salt + h.hexdigest() @@ -69,11 +75,10 @@ class UserMethods(DatabaseMethods): @queue def checkAuth(self, user, password): self.c.execute('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' - 'hddquota, user, template password FROM "users" WHERE name=?', (user, )) + 'hddquota, user, template, password FROM "users" WHERE name=?', (user, )) r = self.c.fetchone() if not r: return None - salt = r[-1][:5] pw = r[-1][5:] h = sha1(salt + password) @@ -93,7 +98,7 @@ class UserMethods(DatabaseMethods): pw = r[2][5:] h = sha1(salt + oldpw) if h.hexdigest() == pw: - salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) + salt = random_salt() h = sha1(salt + newpw) password = salt + h.hexdigest() -- cgit v1.2.3 From cbd4f4b5375f89362733e10a9b98e5a74c2a5734 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Fri, 31 Aug 2012 23:25:26 +0200 Subject: first js models/views --- module/database/FileDatabase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/database') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 80da775c7..ab681dc7f 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -40,7 +40,7 @@ class FileMethods(DatabaseMethods): def processcount(self, fid, user=None): """ number of files which have to be processed """ # status in online, queued, starting, waiting, downloading - self.c.execute("SELECT COUNT(*) FROM files as WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (str(fid), )) + self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (fid, )) return self.c.fetchone()[0] # TODO: think about multiuser side effects on *count methods -- cgit v1.2.3 From 560958b70043ea5b7e0e32d41cb51bd44696d775 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 9 Sep 2012 15:39:50 +0200 Subject: new config api --- module/database/ConfigDatabase.py | 28 ++++++++++++++++++++++++++++ module/database/DatabaseBackend.py | 6 +++--- module/database/FileDatabase.py | 2 +- 3 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 module/database/ConfigDatabase.py (limited to 'module/database') diff --git a/module/database/ConfigDatabase.py b/module/database/ConfigDatabase.py new file mode 100644 index 000000000..cc24f6785 --- /dev/null +++ b/module/database/ConfigDatabase.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.database import DatabaseMethods, queue, async, inner + +# TODO + +class ConfigMethods(DatabaseMethods): + + @async + def saveConfig(self, plugin, user, config): + pass + + @queue + def loadConfig(self, plugin, user): + pass + + @async + def deleteConfig(self, plugin, user): + pass + + @queue + def loadAllConfigs(self): + pass + + + +ConfigMethods.register() \ No newline at end of file diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 2c494e520..3e6b059c0 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -352,10 +352,10 @@ class DatabaseBackend(Thread): self.c.execute( 'CREATE TABLE IF NOT EXISTS "settings" (' '"plugin" TEXT NOT NULL, ' - '"owner" INTEGER NOT NULL, ' - '"configuration" TEXT NOT NULL, ' + '"user" INTEGER NOT NULL, ' + '"config" TEXT NOT NULL, ' 'FOREIGN KEY(owner) REFERENCES users(uid), ' - 'PRIMARY KEY (plugin, owner) ON CONFLICT REPLACE' + 'PRIMARY KEY (plugin, user) ON CONFLICT REPLACE' ')' ) diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index ab681dc7f..e065b84e2 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -75,7 +75,7 @@ class FileMethods(DatabaseMethods): @async def deletePackage(self, pid, owner=None): - # order updated by trigge + # order updated by trigger, as well as links deleted if owner is None: self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) else: -- cgit v1.2.3 From 54bc92b4c5e0b3543a313f497cbc2276403c5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 10 Sep 2012 11:49:35 +0200 Subject: changed config + progress api --- module/database/ConfigDatabase.py | 41 ++++++++++++++++++++++++++++---------- module/database/DatabaseBackend.py | 6 +++--- module/database/__init__.py | 4 +++- 3 files changed, 37 insertions(+), 14 deletions(-) (limited to 'module/database') diff --git a/module/database/ConfigDatabase.py b/module/database/ConfigDatabase.py index cc24f6785..198ae0173 100644 --- a/module/database/ConfigDatabase.py +++ b/module/database/ConfigDatabase.py @@ -1,28 +1,49 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from module.database import DatabaseMethods, queue, async, inner - -# TODO +from module.database import DatabaseMethods, queue, async class ConfigMethods(DatabaseMethods): @async - def saveConfig(self, plugin, user, config): - pass + def saveConfig(self, plugin, config, user=None): + if user is None: + self.c.execute('INSERT INTO settings(plugin, config) VALUES(?,?)', (plugin, config)) + else: + self.c.execute('INSERT INTO settings(plugin, config, user) VALUES(?,?,?)', (plugin, config, user)) + @queue - def loadConfig(self, plugin, user): - pass + def loadConfig(self, plugin, user=None): + if user is None: + self.c.execute('SELECT config FROM settings WHERE plugin=?', (plugin, )) + else: + self.c.execute('SELECT config FROM settings WHERE plugin=? AND user=?', (plugin, user)) + + return self.c.fetchone()[0] @async - def deleteConfig(self, plugin, user): - pass + def deleteConfig(self, plugin, user=None): + if user is None: + self.c.execute('DELETE FROM settings WHERE plugin=?', (plugin, )) + else: + self.c.execute('DELETE FROM settings WHERE plugin=? AND user=?', (plugin, user)) @queue def loadAllConfigs(self): - pass + self.c.execute('SELECT user, plugin, config FROM settings') + configs = {} + for r in self.c: + if r[0] in configs: + configs[r[0]][r[1]] = r[2] + else: + configs[r[0]] = {r[1]: r[2]} + return configs + + @async + def clearAllConfigs(self): + self.c.execute('DELETE FROM settings') ConfigMethods.register() \ No newline at end of file diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 3e6b059c0..58e1e74d8 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -139,7 +139,7 @@ class DatabaseBackend(Thread): set_DB(self) def setup(self): - + """ *MUST* be called before db can be used !""" self.start() self.running.wait() @@ -352,9 +352,9 @@ class DatabaseBackend(Thread): self.c.execute( 'CREATE TABLE IF NOT EXISTS "settings" (' '"plugin" TEXT NOT NULL, ' - '"user" INTEGER NOT NULL, ' + '"user" INTEGER DEFAULT -1 NOT NULL, ' '"config" TEXT NOT NULL, ' - 'FOREIGN KEY(owner) REFERENCES users(uid), ' + 'FOREIGN KEY(user) REFERENCES users(uid), ' 'PRIMARY KEY (plugin, user) ON CONFLICT REPLACE' ')' ) diff --git a/module/database/__init__.py b/module/database/__init__.py index bf4ead872..d3f97fb53 100644 --- a/module/database/__init__.py +++ b/module/database/__init__.py @@ -3,4 +3,6 @@ from DatabaseBackend import DatabaseMethods, DatabaseBackend, queue, async, inne from FileDatabase import FileMethods from UserDatabase import UserMethods from StorageDatabase import StorageMethods -from AccountDatabase import AccountMethods \ No newline at end of file +from AccountDatabase import AccountMethods +from ConfigDatabase import ConfigMethods +from StatisticDatabase import StatisticMethods \ No newline at end of file -- cgit v1.2.3