summaryrefslogtreecommitdiffstats
path: root/pyload/Database
diff options
context:
space:
mode:
Diffstat (limited to 'pyload/Database')
-rw-r--r--pyload/Database/Backend.py329
-rw-r--r--pyload/Database/File.py985
-rw-r--r--pyload/Database/Storage.py36
-rw-r--r--pyload/Database/User.py93
-rw-r--r--pyload/Database/__init__.py6
5 files changed, 1449 insertions, 0 deletions
diff --git a/pyload/Database/Backend.py b/pyload/Database/Backend.py
new file mode 100644
index 000000000..0fc961973
--- /dev/null
+++ b/pyload/Database/Backend.py
@@ -0,0 +1,329 @@
+# -*- coding: utf-8 -*-
+# @author: RaNaN, mkaay
+
+from __future__ import with_statement
+
+try:
+ from pysqlite2 import dbapi2 as sqlite3
+except Exception:
+ import sqlite3
+
+import shutil
+import threading
+import traceback
+
+from Queue import Queue
+
+from pyload.utils import chmod
+
+
+DB_VERSION = 4
+
+
+class style(object):
+ db = None
+
+
+ @classmethod
+ def setDB(cls, db):
+ cls.db = db
+
+
+ @classmethod
+ def inner(cls, f):
+
+
+ @staticmethod
+ def x(*args, **kwargs):
+ if cls.db:
+ return f(cls.db, *args, **kwargs)
+
+ return x
+
+
+ @classmethod
+ def queue(cls, f):
+
+
+ @staticmethod
+ def x(*args, **kwargs):
+ if cls.db:
+ return cls.db.queue(f, *args, **kwargs)
+
+ return x
+
+
+ @classmethod
+ def async(cls, f):
+
+
+ @staticmethod
+ def x(*args, **kwargs):
+ if cls.db:
+ return cls.db.async(f, *args, **kwargs)
+ return x
+
+
+class DatabaseJob(object):
+
+ def __init__(self, f, *args, **kwargs):
+ self.done = threading.Event()
+
+ self.f = f
+ self.args = args
+ self.kwargs = kwargs
+
+ self.result = None
+ self.exception = False
+
+ # import inspect
+ # self.frame = inspect.currentframe()
+
+
+ def __repr__(self):
+ import os
+
+ frame = self.frame.f_back
+ output = ""
+ for _i in xrange(5):
+ output += "\t%s:%s, %s\n" % (os.path.basename(frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name)
+ frame = frame.f_back
+ del frame
+ del self.frame
+
+ return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result)
+
+
+ def processJob(self):
+ try:
+ self.result = self.f(*self.args, **self.kwargs)
+ except Exception, e:
+ traceback.print_exc()
+ try:
+ print "Database Error @", self.f.__name__, self.args[1:], self.kwargs, e
+ except Exception:
+ pass
+
+ self.exception = e
+ finally:
+ self.done.set()
+
+
+ def wait(self):
+ self.done.wait()
+
+
+class DatabaseBackend(threading.Thread):
+ subs = []
+
+
+ def __init__(self, core):
+ threading.Thread.__init__(self)
+ self.setDaemon(True)
+ self.core = core
+
+ self.jobs = Queue()
+
+ self.setuplock = threading.Event()
+
+ style.setDB(self)
+
+
+ def setup(self):
+ self.start()
+ self.setuplock.wait()
+
+
+ def run(self):
+ """main loop, which executes commands"""
+ convert = self._checkVersion() #: returns None or current version
+
+ self.conn = sqlite3.connect("files.db")
+ os.chmod("files.db", 0600)
+
+ self.c = self.conn.cursor() #: compatibility
+
+ if convert is not None:
+ self._convertDB(convert)
+
+ self._createTables()
+ self._migrateUser()
+
+ self.conn.commit()
+
+ self.setuplock.set()
+
+ while True:
+ j = self.jobs.get()
+ if j == "quit":
+ self.c.close()
+ self.conn.close()
+ break
+ j.processJob()
+
+
+ @style.queue
+ def shutdown(self):
+ self.conn.commit()
+ self.jobs.put("quit")
+
+
+ def _checkVersion(self):
+ """ check db version and delete it if needed"""
+ if not os.path.exists("files.version"):
+ with open("files.version", "wb") as f:
+ f.write(str(DB_VERSION))
+ return
+
+ with open("files.version", "rb") as f:
+ v = int(f.read().strip() or 0)
+
+ if v < DB_VERSION:
+ if v < 2:
+ try:
+ self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version."))
+ except Exception:
+ print "Filedatabase was deleted due to incompatible version."
+ reshutil.move("files.version")
+ shutil.move("files.db", "files.backup.db")
+
+ with open("files.version", "wb") as f:
+ f.write(str(DB_VERSION))
+
+ return v
+
+
+ def _convertDB(self, v):
+ try:
+ getattr(self, "_convertV%i" % v)()
+ except Exception:
+ try:
+ self.core.log.error(_("Filedatabase could NOT be converted."))
+ except Exception:
+ print "Filedatabase could NOT be converted."
+
+ # convert scripts start ---------------------------------------------------
+
+
+ def _convertV2(self):
+ self.c.execute(
+ 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")')
+ try:
+ self.manager.core.log.info(_("Database was converted from v2 to v3."))
+ except Exception:
+ print "Database was converted from v2 to v3."
+ self._convertV3()
+
+
+ def _convertV3(self):
+ self.c.execute(
+ 'CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)')
+ try:
+ self.manager.core.log.info(_("Database was converted from v3 to v4."))
+ except Exception:
+ print "Database was converted from v3 to v4."
+
+ # convert scripts end -----------------------------------------------------
+
+
+ def _createTables(self):
+ """create tables for database"""
+
+ self.c.execute(
+ 'CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)')
+ self.c.execute(
+ 'CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))')
+ self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)')
+ self.c.execute(
+ 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")')
+ self.c.execute(
+ 'CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)')
+
+ self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \
+ SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\
+ FROM packages p JOIN links l ON p.id = l.package LEFT OUTER JOIN\
+ (SELECT p.id AS id, COUNT(*) AS linksdone, SUM(l.size) AS sizedone \
+ FROM packages p JOIN links l ON p.id = l.package AND l.status IN (0, 4, 13) GROUP BY p.id) s ON s.id = p.id \
+ GROUP BY p.id')
+
+ # try to lower ids
+ self.c.execute('SELECT max(id) FROM LINKS')
+ fid = self.c.fetchone()[0]
+ fid = int(fid) if fid else 0
+ self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links"))
+
+ self.c.execute('SELECT max(id) FROM packages')
+ pid = self.c.fetchone()[0]
+ pid = int(pid) if pid else 0
+ self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (pid, "packages"))
+
+ self.c.execute('VACUUM')
+
+
+ def _migrateUser(self):
+ if os.path.exists("pyload.db"):
+ try:
+ self.core.log.info(_("Converting old Django DB"))
+ except Exception:
+ print "Converting old Django DB"
+ conn = sqlite3.connect('pyload.db')
+ c = conn.cursor()
+ c.execute("SELECT username, password, email FROM auth_user WHERE is_superuser")
+ users = []
+ for r in c:
+ pw = r[1].split("$")
+ users.append((r[0], pw[1] + pw[2], r[2]))
+ c.close()
+ conn.close()
+
+ self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users)
+ shutil.move("pyload.db", "pyload.old.db")
+
+
+ def createCursor(self):
+ return self.conn.cursor()
+
+
+ @style.async
+ def commit(self):
+ self.conn.commit()
+
+
+ @style.queue
+ def syncSave(self):
+ self.conn.commit()
+
+
+ @style.async
+ def rollback(self):
+ self.conn.rollback()
+
+
+ def async(self, f, *args, **kwargs):
+ args = (self,) + args
+ job = DatabaseJob(f, *args, **kwargs)
+ self.jobs.put(job)
+
+
+ def queue(self, f, *args, **kwargs):
+ args = (self,) + args
+ job = DatabaseJob(f, *args, **kwargs)
+ self.jobs.put(job)
+ job.wait()
+ return job.result
+
+
+ @classmethod
+ def registerSub(cls, klass):
+ cls.subs.append(klass)
+
+
+ @classmethod
+ def unregisterSub(cls, klass):
+ cls.subs.remove(klass)
+
+
+ def __getattr__(self, attr):
+ for sub in DatabaseBackend.subs:
+ if hasattr(sub, attr):
+ return getattr(sub, attr)
diff --git a/pyload/Database/File.py b/pyload/Database/File.py
new file mode 100644
index 000000000..f28535ffa
--- /dev/null
+++ b/pyload/Database/File.py
@@ -0,0 +1,985 @@
+# -*- coding: utf-8 -*-
+# @author: RaNaN, mkaay
+
+import threading
+
+from pyload.Datatype import PyFile, PyPackage
+from pyload.Database import DatabaseBackend, style
+from pyload.manager.Event import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent
+from pyload.utils import formatSize, lock
+
+try:
+ from pysqlite2 import dbapi2 as sqlite3
+except Exception:
+ import sqlite3
+
+
+class FileHandler(object):
+ """Handles all request made to obtain information,
+ modify status or other request for links or packages"""
+
+ def __init__(self, core):
+ """Constructor"""
+ self.core = core
+
+ # translations
+ self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"),
+ _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"),
+ _("downloading"), _("processing"), _("unknown")]
+
+ self.cache = {} #: holds instances for files
+ self.packageCache = {} #: same for packages
+ #@TODO: purge the cache
+
+ self.jobCache = {}
+
+ self.lock = threading.RLock() #@TODO: should be a Lock w/o R
+ # self.lock._Verbose__verbose = True
+
+ self.filecount = -1 #: if an invalid value is set get current value from db
+ self.queuecount = -1 #: number of package to be loaded
+ self.unchanged = False #: determines if any changes was made since last call
+
+ self.db = self.core.db
+
+
+ def change(func):
+
+
+ def new(*args):
+ args[0].unchanged = False
+ args[0].filecount = -1
+ args[0].queuecount = -1
+ args[0].jobCache = {}
+ return func(*args)
+
+ return new
+
+
+ #--------------------------------------------------------------------------
+
+ def save(self):
+ """saves all data to backend"""
+ self.db.commit()
+
+
+ #--------------------------------------------------------------------------
+
+ def syncSave(self):
+ """saves all data to backend and waits until all data are written"""
+ pyfiles = self.cache.values()
+ for pyfile in pyfiles:
+ pyfile.sync()
+
+ pypacks = self.packageCache.values()
+ for pypack in pypacks:
+ pypack.sync()
+
+ self.db.syncSave()
+
+
+ @lock
+ def getCompleteData(self, queue=1):
+ """gets a complete data representation"""
+
+ data = self.db.getAllLinks(queue)
+ packs = self.db.getAllPackages(queue)
+
+ data.update([(x.id, x.toDbDict()[x.id]) for x in self.cache.values()])
+
+ for x in self.packageCache.itervalues():
+ if x.queue != queue or x.id not in packs:
+ continue
+ packs[x.id].update(x.toDict()[x.id])
+
+ for key, value in data.iteritems():
+ if value['package'] in packs:
+ packs[value['package']]['links'][key] = value
+
+ return packs
+
+
+ @lock
+ def getInfoData(self, queue=1):
+ """gets a data representation without links"""
+
+ packs = self.db.getAllPackages(queue)
+ for x in self.packageCache.itervalues():
+ if x.queue != queue or x.id not in packs:
+ continue
+ packs[x.id].update(x.toDict()[x.id])
+
+ return packs
+
+
+ @lock
+ @change
+ def addLinks(self, urls, package):
+ """adds links"""
+
+ self.core.addonManager.dispatchEvent("links-added", urls, package)
+
+ data = self.core.pluginManager.parseUrls(urls)
+
+ self.db.addLinks(data, package)
+ self.core.threadManager.createInfoThread(data, package)
+
+ #@TODO: change from reloadAll event to package update event
+ self.core.pullManager.addEvent(ReloadAllEvent("collector"))
+
+
+ #--------------------------------------------------------------------------
+
+ @lock
+ @change
+ def addPackage(self, name, folder, queue=0):
+ """adds a package, default to link collector"""
+ lastID = self.db.addPackage(name, folder, queue)
+ p = self.db.getPackage(lastID)
+ e = InsertEvent("pack", lastID, p.order, "collector" if not queue else "queue")
+ self.core.pullManager.addEvent(e)
+ return lastID
+
+
+ #--------------------------------------------------------------------------
+
+ @lock
+ @change
+ def deletePackage(self, id):
+ """delete package and all contained links"""
+
+ p = self.getPackage(id)
+ if not p:
+ if id in self.packageCache:
+ del self.packageCache[id]
+ return
+
+ oldorder = p.order
+ queue = p.queue
+
+ e = RemoveEvent("pack", id, "collector" if not p.queue else "queue")
+
+ pyfiles = self.cache.values()
+
+ for pyfile in pyfiles:
+ if pyfile.packageid == id:
+ pyfile.abortDownload()
+ pyfile.release()
+
+ self.db.deletePackage(p)
+ self.core.pullManager.addEvent(e)
+ self.core.addonManager.dispatchEvent("package-deleted", id)
+
+ if id in self.packageCache:
+ del self.packageCache[id]
+
+ packs = self.packageCache.values()
+ for pack in packs:
+ if pack.queue == queue and pack.order > oldorder:
+ pack.order -= 1
+ pack.notifyChange()
+
+
+ #--------------------------------------------------------------------------
+
+ @lock
+ @change
+ def deleteLink(self, id):
+ """deletes links"""
+
+ f = self.getFile(id)
+ if not f:
+ return None
+
+ pid = f.packageid
+ e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue")
+
+ oldorder = f.order
+
+ if id in self.core.threadManager.processingIds():
+ self.cache[id].abortDownload()
+
+ if id in self.cache:
+ del self.cache[id]
+
+ self.db.deleteLink(f)
+
+ self.core.pullManager.addEvent(e)
+
+ p = self.getPackage(pid)
+ if not len(p.getChildren()):
+ p.delete()
+
+ pyfiles = self.cache.values()
+ for pyfile in pyfiles:
+ if pyfile.packageid == pid and pyfile.order > oldorder:
+ pyfile.order -= 1
+ pyfile.notifyChange()
+
+
+ #--------------------------------------------------------------------------
+
+ def releaseLink(self, id):
+ """removes pyfile from cache"""
+ if id in self.cache:
+ del self.cache[id]
+
+
+ #--------------------------------------------------------------------------
+
+ def releasePackage(self, id):
+ """removes package from cache"""
+ if id in self.packageCache:
+ del self.packageCache[id]
+
+
+ #--------------------------------------------------------------------------
+
+ def updateLink(self, pyfile):
+ """updates link"""
+ self.db.updateLink(pyfile)
+
+ e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue")
+ self.core.pullManager.addEvent(e)
+
+
+ #--------------------------------------------------------------------------
+
+ def updatePackage(self, pypack):
+ """updates a package"""
+ self.db.updatePackage(pypack)
+
+ e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue")
+ self.core.pullManager.addEvent(e)
+
+
+ #--------------------------------------------------------------------------
+
+ def getPackage(self, id):
+ """return package instance"""
+
+ if id in self.packageCache:
+ return self.packageCache[id]
+ else:
+ return self.db.getPackage(id)
+
+
+ #--------------------------------------------------------------------------
+
+ def getPackageData(self, id):
+ """returns dict with package information"""
+ pack = self.getPackage(id)
+
+ if not pack:
+ return None
+
+ pack = pack.toDict()[id]
+
+ data = self.db.getPackageData(id)
+
+ tmplist = []
+
+ cache = self.cache.values()
+ for x in cache:
+ if int(x.toDbDict()[x.id]['package']) == int(id):
+ tmplist.append((x.id, x.toDbDict()[x.id]))
+ data.update(tmplist)
+
+ pack['links'] = data
+
+ return pack
+
+
+ #--------------------------------------------------------------------------
+
+ def getFileData(self, id):
+ """returns dict with file information"""
+ if id in self.cache:
+ return self.cache[id].toDbDict()
+
+ return self.db.getLinkData(id)
+
+
+ #--------------------------------------------------------------------------
+
+ def getFile(self, id):
+ """returns pyfile instance"""
+ if id in self.cache:
+ return self.cache[id]
+ else:
+ return self.db.getFile(id)
+
+
+ #--------------------------------------------------------------------------
+
+ @lock
+ def getJob(self, occ):
+ """get suitable job"""
+
+ #@TODO: clean mess
+ #@TODO: improve selection of valid jobs
+
+ if occ in self.jobCache:
+ if self.jobCache[occ]:
+ id = self.jobCache[occ].pop()
+ if id == "empty":
+ pyfile = None
+ self.jobCache[occ].append("empty")
+ else:
+ pyfile = self.getFile(id)
+ else:
+ jobs = self.db.getJob(occ)
+ jobs.reverse()
+ if not jobs:
+ self.jobCache[occ].append("empty")
+ pyfile = None
+ else:
+ self.jobCache[occ].extend(jobs)
+ pyfile = self.getFile(self.jobCache[occ].pop())
+
+ else:
+ self.jobCache = {} #: better not caching to much
+ jobs = self.db.getJob(occ)
+ jobs.reverse()
+ self.jobCache[occ] = jobs
+
+ if not jobs:
+ self.jobCache[occ].append("empty")
+ pyfile = None
+ else:
+ pyfile = self.getFile(self.jobCache[occ].pop())
+
+ #@TODO: maybe the new job has to be approved...
+
+ # pyfile = self.getFile(self.jobCache[occ].pop())
+ return pyfile
+
+
+ @lock
+ def getDecryptJob(self):
+ """return job for decrypting"""
+ if "decrypt" in self.jobCache:
+ return None
+
+ plugins = self.core.pluginManager.crypterPlugins.keys() + self.core.pluginManager.containerPlugins.keys()
+ plugins = str(tuple(plugins))
+
+ jobs = self.db.getPluginJob(plugins)
+ if jobs:
+ return self.getFile(jobs[0])
+ else:
+ self.jobCache['decrypt'] = "empty"
+ return None
+
+
+ def getFileCount(self):
+ """returns number of files"""
+
+ if self.filecount == -1:
+ self.filecount = self.db.filecount(1)
+
+ return self.filecount
+
+
+ def getQueueCount(self, force=False):
+ """number of files that have to be processed"""
+ if self.queuecount == -1 or force:
+ self.queuecount = self.db.queuecount(1)
+
+ return self.queuecount
+
+
+ def checkAllLinksFinished(self):
+ """checks if all files are finished and dispatch event"""
+
+ if not self.getQueueCount(True):
+ self.core.addonManager.dispatchEvent("all_downloads-finished")
+ self.core.log.debug("All downloads finished")
+ return True
+
+ return False
+
+
+ def checkAllLinksProcessed(self, fid):
+ """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting"""
+
+ # reset count so statistic will update (this is called when dl was processed)
+ self.resetCount()
+
+ if not self.db.processcount(1, fid):
+ self.core.addonManager.dispatchEvent("all_downloads-processed")
+ self.core.log.debug("All downloads processed")
+ return True
+
+ return False
+
+
+ def resetCount(self):
+ self.queuecount = -1
+
+
+ @lock
+ @change
+ def restartPackage(self, id):
+ """restart package"""
+ pyfiles = self.cache.values()
+ for pyfile in pyfiles:
+ if pyfile.packageid == id:
+ self.restartFile(pyfile.id)
+
+ self.db.restartPackage(id)
+
+ if id in self.packageCache:
+ self.packageCache[id].setFinished = False
+
+ e = UpdateEvent("pack", id, "collector" if not self.getPackage(id).queue else "queue")
+ self.core.pullManager.addEvent(e)
+
+
+ @lock
+ @change
+ def restartFile(self, id):
+ """ restart file"""
+ if id in self.cache:
+ self.cache[id].status = 3
+ self.cache[id].name = self.cache[id].url
+ self.cache[id].error = ""
+ self.cache[id].abortDownload()
+
+ self.db.restartFile(id)
+
+ e = UpdateEvent("file", id, "collector" if not self.getFile(id).package().queue else "queue")
+ self.core.pullManager.addEvent(e)
+
+
+ @lock
+ @change
+ def setPackageLocation(self, id, queue):
+ """push package to queue"""
+
+ p = self.db.getPackage(id)
+ oldorder = p.order
+
+ e = RemoveEvent("pack", id, "collector" if not p.queue else "queue")
+ self.core.pullManager.addEvent(e)
+
+ self.db.clearPackageOrder(p)
+
+ p = self.db.getPackage(id)
+
+ p.queue = queue
+ self.db.updatePackage(p)
+
+ self.db.reorderPackage(p, -1, True)
+
+ packs = self.packageCache.values()
+ for pack in packs:
+ if pack.queue != queue and pack.order > oldorder:
+ pack.order -= 1
+ pack.notifyChange()
+
+ self.db.commit()
+ self.releasePackage(id)
+ p = self.getPackage(id)
+
+ e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue")
+ self.core.pullManager.addEvent(e)
+
+
+ @lock
+ @change
+ def reorderPackage(self, id, position):
+ p = self.getPackage(id)
+
+ e = RemoveEvent("pack", id, "collector" if not p.queue else "queue")
+ self.core.pullManager.addEvent(e)
+ self.db.reorderPackage(p, position)
+
+ packs = self.packageCache.values()
+ for pack in packs:
+ if pack.queue != p.queue or pack.order < 0 or pack == p:
+ continue
+ if p.order > position:
+ if position <= pack.order < p.order:
+ pack.order += 1
+ pack.notifyChange()
+ elif p.order < position:
+ if position >= pack.order > p.order:
+ pack.order -= 1
+ pack.notifyChange()
+
+ p.order = position
+ self.db.commit()
+
+ e = InsertEvent("pack", id, position, "collector" if not p.queue else "queue")
+ self.core.pullManager.addEvent(e)
+
+
+ @lock
+ @change
+ def reorderFile(self, id, position):
+ f = self.getFileData(id)
+ f = f[id]
+
+ e = RemoveEvent("file", id, "collector" if not self.getPackage(f['package']).queue else "queue")
+ self.core.pullManager.addEvent(e)
+
+ self.db.reorderLink(f, position)
+
+ pyfiles = self.cache.values()
+ for pyfile in pyfiles:
+ if pyfile.packageid != f['package'] or pyfile.order < 0:
+ continue
+ if f['order'] > position:
+ if position <= pyfile.order < f['order']:
+ pyfile.order += 1
+ pyfile.notifyChange()
+ elif f['order'] < position:
+ if position >= pyfile.order > f['order']:
+ pyfile.order -= 1
+ pyfile.notifyChange()
+
+ if id in self.cache:
+ self.cache[id].order = position
+
+ self.db.commit()
+
+ e = InsertEvent("file", id, position, "collector" if not self.getPackage(f['package']).queue else "queue")
+ self.core.pullManager.addEvent(e)
+
+
+ @change
+ def updateFileInfo(self, data, pid):
+ """ updates file info (name, size, status, url)"""
+ ids = self.db.updateLinkInfo(data)
+ e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue")
+ self.core.pullManager.addEvent(e)
+
+
+ def checkPackageFinished(self, pyfile):
+ """ checks if package is finished and calls AddonManager """
+
+ ids = self.db.getUnfinished(pyfile.packageid)
+ if not ids or (pyfile.id in ids and len(ids) == 1):
+ if not pyfile.package().setFinished:
+ self.core.log.info(_("Package finished: %s") % pyfile.package().name)
+ self.core.addonManager.packageFinished(pyfile.package())
+ pyfile.package().setFinished = True
+
+
+ def reCheckPackage(self, pid):
+ """ recheck links in package """
+ data = self.db.getPackageData(pid)
+
+ urls = []
+
+ for pyfile in data.itervalues():
+ if pyfile['status'] not in (0, 12, 13):
+ urls.append((pyfile['url'], pyfile['plugin']))
+
+ self.core.threadManager.createInfoThread(urls, pid)
+
+
+ @lock
+ @change
+ def deleteFinishedLinks(self):
+ """ deletes finished links and packages, return deleted packages """
+
+ old_packs = self.getInfoData(0)
+ old_packs.update(self.getInfoData(1))
+
+ self.db.deleteFinished()
+
+ new_packs = self.db.getAllPackages(0)
+ new_packs.update(self.db.getAllPackages(1))
+ # get new packages only from db
+
+ deleted = [id for id in old_packs.iterkeys() if id not in new_packs]
+ for id_deleted in deleted:
+ self.deletePackage(int(id_deleted))
+
+ return deleted
+
+
+ @lock
+ @change
+ def restartFailed(self):
+ """ restart all failed links """
+ self.db.restartFailed()
+
+
+class FileMethods(object):
+
+
+ @style.queue
+ def filecount(self, queue):
+ """returns number of files in queue"""
+ self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?",
+ (queue,))
+ return self.c.fetchone()[0]
+
+
+ @style.queue
+ def queuecount(self, queue):
+ """ number of files in queue not finished yet"""
+ self.c.execute(
+ "SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0, 4)",
+ (queue,))
+ return self.c.fetchone()[0]
+
+
+ @style.queue
+ def processcount(self, queue, fid):
+ """ number of files which have to be proccessed """
+ self.c.execute(
+ "SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2, 3, 5, 7, 12) AND l.id != ?",
+ (queue, str(fid)))
+ return self.c.fetchone()[0]
+
+
+ @style.inner
+ def _nextPackageOrder(self, queue=0):
+ self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,))
+ max = self.c.fetchone()[0]
+ if max is not None:
+ return max + 1
+ else:
+ return 0
+
+
+ @style.inner
+ def _nextFileOrder(self, package):
+ self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,))
+ max = self.c.fetchone()[0]
+ if max is not None:
+ return max + 1
+ else:
+ return 0
+
+
+ @style.queue
+ def addLink(self, url, name, plugin, package):
+ order = self._nextFileOrder(package)
+ self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)',
+ (url, name, ".".join(plugintype, pluginname), package, order))
+ return self.c.lastrowid
+
+
+ @style.queue
+ def addLinks(self, links, package):
+ """ links is a list of tupels (url, plugin)"""
+ order = self._nextFileOrder(package)
+ orders = [order + x for x in xrange(len(links))]
+ links = [(x[0], x[0], ".".join((x[1], x[2])), package, o) for x, o in zip(links, orders)]
+ self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links)
+
+
+ @style.queue
+ def addPackage(self, name, folder, queue):
+ order = self._nextPackageOrder(queue)
+ self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)',
+ (name, folder, queue, order))
+ return self.c.lastrowid
+
+
+ @style.queue
+ def deletePackage(self, p):
+ self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),))
+ self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),))
+ self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?',
+ (p.order, p.queue))
+
+
+ @style.queue
+ def deleteLink(self, f):
+ self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),))
+ self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?',
+ (f.order, str(f.packageid)))
+
+
+ @style.queue
+ def getAllLinks(self, q):
+ """return information about all links in queue q
+
+ q0 queue
+ q1 collector
+
+ format:
+
+ {
+ id: {'name': name, ... 'package': id }, ...
+ }
+
+ """
+ self.c.execute(
+ 'SELECT l.id, l.url, l.name, l.size, l.status, l.error, l.plugin, l.package, l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder',
+ (q,))
+ data = {}
+ for r in self.c:
+ data[r[0]] = {
+ 'id': r[0],
+ 'url': r[1],
+ 'name': r[2],
+ 'size': r[3],
+ 'format_size': formatSize(r[3]),
+ 'status': r[4],
+ 'statusmsg': self.manager.statusMsg[r[4]],
+ 'error': r[5],
+ 'plugin': tuple(r[6].split('.')),
+ 'package': r[7],
+ 'order': r[8],
+ }
+
+ return data
+
+
+ @style.queue
+ def getAllPackages(self, q):
+ """return information about packages in queue q
+ (only useful in get all data)
+
+ q0 queue
+ q1 collector
+
+ format:
+
+ {
+ id: {'name': name ... 'links': {}}, ...
+ }
+ """
+ self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \
+ FROM packages p JOIN pstats s ON p.id = s.id \
+ WHERE p.queue=? ORDER BY p.packageorder', str(q))
+
+ data = {}
+ for r in self.c:
+ data[r[0]] = {
+ 'id': r[0],
+ 'name': r[1],
+ 'folder': r[2],
+ 'site': r[3],
+ 'password': r[4],
+ 'queue': r[5],
+ 'order': r[6],
+ 'sizetotal': int(r[7]),
+ 'sizedone': r[8] if r[8] else 0, #: these can be None
+ 'linksdone': r[9] if r[9] else 0,
+ 'linkstotal': r[10],
+ 'links': {}
+ }
+
+ return data
+
+
+ @style.queue
+ def getLinkData(self, id):
+ """get link information as dict"""
+ self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?',
+ (str(id),))
+ data = {}
+ r = self.c.fetchone()
+ if not r:
+ return None
+ data[r[0]] = {
+ 'id': r[0],
+ 'url': r[1],
+ 'name': r[2],
+ 'size': r[3],
+ 'format_size': formatSize(r[3]),
+ 'status': r[4],
+ 'statusmsg': self.manager.statusMsg[r[4]],
+ 'error': r[5],
+ 'plugin': tuple(r[6].split('.')),
+ 'package': r[7],
+ 'order': r[8],
+ }
+
+ return data
+
+
+ @style.queue
+ def getPackageData(self, id):
+ """get data about links for a package"""
+ self.c.execute(
+ 'SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE package=? ORDER BY linkorder',
+ (str(id),))
+
+ data = {}
+ for r in self.c:
+ data[r[0]] = {
+ 'id': r[0],
+ 'url': r[1],
+ 'name': r[2],
+ 'size': r[3],
+ 'format_size': formatSize(r[3]),
+ 'status': r[4],
+ 'statusmsg': self.manager.statusMsg[r[4]],
+ 'error': r[5],
+ 'plugin': tuple(r[6].split('.')),
+ 'package': r[7],
+ 'order': r[8],
+ }
+
+ return data
+
+
+ @style.async
+ def updateLink(self, f):
+ self.c.execute('UPDATE links SET url=?, name=?, size=?, status=?, error=?, package=? WHERE id=?',
+ (f.url, f.name, f.size, f.status, str(f.error), str(f.packageid), str(f.id)))
+
+
+ @style.queue
+ def updatePackage(self, p):
+ self.c.execute('UPDATE packages SET name=?, folder=?, site=?, password=?, queue=? WHERE id=?',
+ (p.name, p.folder, p.site, p.password, p.queue, str(p.id)))
+
+ @style.queue
+ def updateLinkInfo(self, data):
+ """ data is list of tupels (name, size, status, url) """
+ self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1, 2, 3, 14)', data)
+ ids = []
+ self.c.execute('SELECT id FROM links WHERE url IN (\'%s\')' % "','".join([x[3] for x in data]))
+ for r in self.c:
+ ids.append(int(r[0]))
+ return ids
+
+
+ @style.queue
+ def reorderPackage(self, p, position, noMove=False):
+ if position == -1:
+ position = self._nextPackageOrder(p.queue)
+ if not noMove:
+ if p.order > position:
+ self.c.execute(
+ 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND queue=? AND packageorder >= 0',
+ (position, p.order, p.queue))
+ elif p.order < position:
+ self.c.execute(
+ 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND queue=? AND packageorder >= 0',
+ (position, p.order, p.queue))
+
+ self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id)))
+
+
+ @style.queue
+ def reorderLink(self, f, position):
+ """ reorder link with f as dict for pyfile """
+ if f['order'] > position:
+ self.c.execute('UPDATE links SET linkorder=linkorder+1 WHERE linkorder >= ? AND linkorder < ? AND package=?',
+ (position, f['order'], f['package']))
+ elif f['order'] < position:
+ self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder <= ? AND linkorder > ? AND package=?',
+ (position, f['order'], f['package']))
+
+ self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f['id']))
+
+
+ @style.queue
+ def clearPackageOrder(self, p):
+ self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id)))
+ self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?',
+ (p.order, p.queue, str(p.id)))
+
+
+ @style.async
+ def restartFile(self, id):
+ self.c.execute('UPDATE links SET status=3, error="" WHERE id=?', (str(id),))
+
+
+ @style.async
+ def restartPackage(self, id):
+ self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),))
+
+
+ @style.queue
+ def getPackage(self, id):
+ """return package instance from id"""
+ self.c.execute("SELECT name, folder, site, password, queue, packageorder FROM packages WHERE id=?", (str(id),))
+ r = self.c.fetchone()
+ if not r:
+ return None
+ return PyPackage(self.manager, id, *r)
+
+
+ #--------------------------------------------------------------------------
+
+ @style.queue
+ def getFile(self, id):
+ """return link instance from id"""
+ self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?",
+ (str(id),))
+ r = self.c.fetchone()
+ if not r:
+ return None
+ r = list(r)
+ r[5] = tuple(r[5].split('.'))
+ return PyFile(self.manager, id, *r)
+
+
+ @style.queue
+ def getJob(self, occ):
+ """return pyfile ids, which are suitable for download and dont use a occupied plugin"""
+
+ #@TODO: improve this hardcoded method
+ pre = "('CCF', 'DLC', 'LinkList', 'RSDF', 'TXT')" #: plugins which are processed in collector
+
+ cmd = "("
+ for i, item in enumerate(occ):
+ if i: cmd += ", "
+ cmd += "'%s'" % item
+
+ cmd += ")"
+
+ cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre)
+
+ self.c.execute(cmd) #: very bad!
+
+ return [x[0] for x in self.c]
+
+
+ @style.queue
+ def getPluginJob(self, plugins):
+ """returns pyfile ids with suited plugins"""
+ cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins
+
+ self.c.execute(cmd) #: very bad!
+
+ return [x[0] for x in self.c]
+
+
+ @style.queue
+ def getUnfinished(self, pid):
+ """return list of max length 3 ids with pyfiles in package not finished or processed"""
+
+ self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),))
+ return [r[0] for r in self.c]
+
+
+ @style.queue
+ def deleteFinished(self):
+ self.c.execute("DELETE FROM links WHERE status IN (0, 4)")
+ self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)")
+
+
+ @style.queue
+ def restartFailed(self):
+ self.c.execute("UPDATE links SET status=3, error='' WHERE status IN (6, 8, 9)")
+
+
+ @style.queue
+ def findDuplicates(self, id, folder, filename):
+ """ checks if filename exists with different id and same package """
+ self.c.execute(
+ "SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?",
+ (folder, id, filename))
+ return self.c.fetchone()
+
+
+ @style.queue
+ def purgeLinks(self):
+ self.c.execute("DELETE FROM links;")
+ self.c.execute("DELETE FROM packages;")
+
+
+DatabaseBackend.registerSub(FileMethods)
diff --git a/pyload/Database/Storage.py b/pyload/Database/Storage.py
new file mode 100644
index 000000000..a19f67606
--- /dev/null
+++ b/pyload/Database/Storage.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# @author: mkaay
+
+from pyload.Database import style
+from pyload.Database import DatabaseBackend
+
+
+class StorageMethods(object):
+
+ @style.queue
+ def setStorage(db, identifier, key, value):
+ db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key))
+ if db.c.fetchone() is not None:
+ db.c.execute("UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key))
+ else:
+ db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value))
+
+
+ @style.queue
+ def getStorage(db, identifier, key=None):
+ if key is not None:
+ db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key))
+ row = db.c.fetchone()
+ if row is not None:
+ return row[0]
+ else:
+ db.c.execute("SELECT key, value FROM storage WHERE identifier=?", (identifier,))
+ return {row[0]: row[1] for row in db.c}
+
+
+ @style.queue
+ def delStorage(db, identifier, key):
+ db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key))
+
+
+DatabaseBackend.registerSub(StorageMethods)
diff --git a/pyload/Database/User.py b/pyload/Database/User.py
new file mode 100644
index 000000000..dc60ce23a
--- /dev/null
+++ b/pyload/Database/User.py
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+# @author: mkaay
+
+from hashlib import sha1
+import random
+
+from pyload.Database import DatabaseBackend, style
+
+
+class UserMethods(object):
+
+
+ @style.queue
+ def checkAuth(db, user, password):
+ c = db.c
+ c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user,))
+ r = c.fetchone()
+ if not r:
+ return {}
+
+ salt = r[2][:5]
+ pw = r[2][5:]
+ h = sha1(salt + password)
+ if h.hexdigest() == pw:
+ return {"id": r[0], "name": r[1], "role": r[3],
+ "permission": r[4], "template": r[5], "email": r[6]}
+ else:
+ return {}
+
+
+ @style.queue
+ def addUser(db, user, password):
+ salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for _i in xrange(0, 5)])
+ h = sha1(salt + password)
+ password = salt + h.hexdigest()
+
+ c = db.c
+ c.execute('SELECT name FROM users WHERE name=?', (user,))
+ if c.fetchone() is not None:
+ c.execute('UPDATE users SET password=? WHERE name=?', (password, user))
+ else:
+ c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password))
+
+
+ @style.queue
+ def changePassword(db, user, oldpw, newpw):
+ db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user,))
+ r = db.c.fetchone()
+ if not r:
+ return False
+
+ salt = r[2][:5]
+ pw = r[2][5:]
+ h = sha1(salt + oldpw)
+ if h.hexdigest() == pw:
+ salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for _i in xrange(0, 5)])
+ h = sha1(salt + newpw)
+ password = salt + h.hexdigest()
+
+ db.c.execute("UPDATE users SET password=? WHERE name=?", (password, user))
+ return True
+
+ return False
+
+
+ @style.async
+ def setPermission(db, user, perms):
+ db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user))
+
+
+ @style.async
+ def setRole(db, user, role):
+ db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user))
+
+
+ @style.queue
+ def listUsers(db):
+ db.c.execute('SELECT name FROM users')
+ return [row[0] for row in db.c]
+
+
+ @style.queue
+ def getAllUserData(db):
+ db.c.execute("SELECT name, permission, role, template, email FROM users")
+ return {r[0]: {"permission": r[1], "role": r[2], "template": r[3], "email": r[4]} for r in db.c}
+
+
+ @style.queue
+ def removeUser(db, user):
+ db.c.execute('DELETE FROM users WHERE name=?', (user,))
+
+
+DatabaseBackend.registerSub(UserMethods)
diff --git a/pyload/Database/__init__.py b/pyload/Database/__init__.py
new file mode 100644
index 000000000..b4200a698
--- /dev/null
+++ b/pyload/Database/__init__.py
@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from pyload.Database.Backend import DatabaseBackend, style
+from pyload.Database.File import FileHandler
+from pyload.Database.Storage import StorageMethods
+from pyload.Database.User import UserMethods
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-09 09:50:24 +0100