summaryrefslogtreecommitdiffstats
path: root/pyload/database
diff options
context:
space:
mode:
Diffstat (limited to 'pyload/database')
-rw-r--r--pyload/database/Backend.py43
-rw-r--r--pyload/database/File.py124
-rw-r--r--pyload/database/Storage.py6
-rw-r--r--pyload/database/User.py16
4 files changed, 189 insertions, 0 deletions
diff --git a/pyload/database/Backend.py b/pyload/database/Backend.py
index 4b63dd284..48c1fcb0d 100644
--- a/pyload/database/Backend.py
+++ b/pyload/database/Backend.py
@@ -22,34 +22,49 @@ class style(object):
db = None
@classmethod
+
+
def setDB(cls, db):
cls.db = db
@classmethod
+
+
def inner(cls, f):
@staticmethod
+
+
def x(*args, **kwargs):
if cls.db:
return f(cls.db, *args, **kwargs)
return x
@classmethod
+
+
def queue(cls, f):
@staticmethod
+
+
def x(*args, **kwargs):
if cls.db:
return cls.db.queue(f, *args, **kwargs)
return x
@classmethod
+
+
def async(cls, f):
@staticmethod
+
+
def x(*args, **kwargs):
if cls.db:
return cls.db.async(f, *args, **kwargs)
return x
class DatabaseJob(object):
+
def __init__(self, f, *args, **kwargs):
self.done = Event()
@@ -63,6 +78,7 @@ class DatabaseJob(object):
# import inspect
# self.frame = inspect.currentframe()
+
def __repr__(self):
from os.path import basename
frame = self.frame.f_back
@@ -75,6 +91,7 @@ class DatabaseJob(object):
return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result)
+
def processJob(self):
try:
self.result = self.f(*self.args, **self.kwargs)
@@ -89,11 +106,14 @@ class DatabaseJob(object):
finally:
self.done.set()
+
def wait(self):
self.done.wait()
class DatabaseBackend(Thread):
subs = []
+
+
def __init__(self, core):
Thread.__init__(self)
self.setDaemon(True)
@@ -105,10 +125,12 @@ class DatabaseBackend(Thread):
style.setDB(self)
+
def setup(self):
self.start()
self.setuplock.wait()
+
def run(self):
"""main loop, which executes commands"""
convert = self._checkVersion() #returns None or current version
@@ -137,10 +159,13 @@ class DatabaseBackend(Thread):
j.processJob()
@style.queue
+
+
def shutdown(self):
self.conn.commit()
self.jobs.put("quit")
+
def _checkVersion(self):
""" check db version and delete it if needed"""
if not exists("files.version"):
@@ -165,6 +190,7 @@ class DatabaseBackend(Thread):
f.close()
return v
+
def _convertDB(self, v):
try:
getattr(self, "_convertV%i" % v)()
@@ -176,6 +202,7 @@ class DatabaseBackend(Thread):
#convert scripts start-----------------------------------------------------
+
def _convertV2(self):
self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")')
try:
@@ -184,6 +211,7 @@ class DatabaseBackend(Thread):
print "Database was converted from v2 to v3."
self._convertV3()
+
def _convertV3(self):
self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)')
try:
@@ -193,6 +221,7 @@ class DatabaseBackend(Thread):
#convert scripts end-------------------------------------------------------
+
def _createTables(self):
"""create tables for database"""
@@ -249,26 +278,35 @@ class DatabaseBackend(Thread):
self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users)
move("pyload.db", "pyload.old.db")
+
def createCursor(self):
return self.conn.cursor()
@style.async
+
+
def commit(self):
self.conn.commit()
@style.queue
+
+
def syncSave(self):
self.conn.commit()
@style.async
+
+
def rollback(self):
self.conn.rollback()
+
def async(self, f, *args, **kwargs):
args = (self,) + args
job = DatabaseJob(f, *args, **kwargs)
self.jobs.put(job)
+
def queue(self, f, *args, **kwargs):
args = (self,) + args
job = DatabaseJob(f, *args, **kwargs)
@@ -277,13 +315,18 @@ class DatabaseBackend(Thread):
return job.result
@classmethod
+
+
def registerSub(cls, klass):
cls.subs.append(klass)
@classmethod
+
+
def unregisterSub(cls, klass):
cls.subs.remove(klass)
+
def __getattr__(self, attr):
for sub in DatabaseBackend.subs:
if hasattr(sub, attr):
diff --git a/pyload/database/File.py b/pyload/database/File.py
index 857da1ff9..0ff520623 100644
--- a/pyload/database/File.py
+++ b/pyload/database/File.py
@@ -42,7 +42,10 @@ class FileHandler(object):
self.db = self.core.db
+
def change(func):
+
+
def new(*args):
args[0].unchanged = False
args[0].filecount = -1
@@ -51,12 +54,16 @@ class FileHandler(object):
return func(*args)
return new
+
#--------------------------------------------------------------------------
+
def save(self):
"""saves all data to backend"""
self.db.commit()
+
#--------------------------------------------------------------------------
+
def syncSave(self):
"""saves all data to backend and waits until all data are written"""
pyfiles = self.cache.values()
@@ -70,6 +77,8 @@ class FileHandler(object):
self.db.syncSave()
@lock
+
+
def getCompleteData(self, queue=1):
"""gets a complete data representation"""
@@ -89,6 +98,8 @@ class FileHandler(object):
return packs
@lock
+
+
def getInfoData(self, queue=1):
"""gets a data representation without links"""
@@ -101,6 +112,8 @@ class FileHandler(object):
@lock
@change
+
+
def addLinks(self, urls, package):
"""adds links"""
@@ -114,7 +127,9 @@ class FileHandler(object):
#@TODO: change from reloadAll event to package update event
self.core.pullManager.addEvent(ReloadAllEvent("collector"))
+
#--------------------------------------------------------------------------
+
@lock
@change
def addPackage(self, name, folder, queue=0):
@@ -125,7 +140,9 @@ class FileHandler(object):
self.core.pullManager.addEvent(e)
return lastID
+
#--------------------------------------------------------------------------
+
@lock
@change
def deletePackage(self, id):
@@ -161,7 +178,9 @@ class FileHandler(object):
pack.order -= 1
pack.notifyChange()
+
#--------------------------------------------------------------------------
+
@lock
@change
def deleteLink(self, id):
@@ -197,18 +216,24 @@ class FileHandler(object):
pyfile.notifyChange()
#--------------------------------------------------------------------------
+
+
def releaseLink(self, id):
"""removes pyfile from cache"""
if id in self.cache:
del self.cache[id]
+
#--------------------------------------------------------------------------
+
def releasePackage(self, id):
"""removes package from cache"""
if id in self.packageCache:
del self.packageCache[id]
+
#--------------------------------------------------------------------------
+
def updateLink(self, pyfile):
"""updates link"""
self.db.updateLink(pyfile)
@@ -216,7 +241,9 @@ class FileHandler(object):
e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue")
self.core.pullManager.addEvent(e)
+
#--------------------------------------------------------------------------
+
def updatePackage(self, pypack):
"""updates a package"""
self.db.updatePackage(pypack)
@@ -224,7 +251,9 @@ class FileHandler(object):
e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue")
self.core.pullManager.addEvent(e)
+
#--------------------------------------------------------------------------
+
def getPackage(self, id):
"""return package instance"""
@@ -233,7 +262,9 @@ class FileHandler(object):
else:
return self.db.getPackage(id)
+
#--------------------------------------------------------------------------
+
def getPackageData(self, id):
"""returns dict with package information"""
pack = self.getPackage(id)
@@ -257,7 +288,9 @@ class FileHandler(object):
return pack
+
#--------------------------------------------------------------------------
+
def getFileData(self, id):
"""returns dict with file information"""
if id in self.cache:
@@ -265,7 +298,9 @@ class FileHandler(object):
return self.db.getLinkData(id)
+
#--------------------------------------------------------------------------
+
def getFile(self, id):
"""returns pyfile instance"""
if id in self.cache:
@@ -273,7 +308,9 @@ class FileHandler(object):
else:
return self.db.getFile(id)
+
#--------------------------------------------------------------------------
+
@lock
def getJob(self, occ):
"""get suitable job"""
@@ -317,6 +354,8 @@ class FileHandler(object):
return pyfile
@lock
+
+
def getDecryptJob(self):
"""return job for decrypting"""
if "decrypt" in self.jobCache:
@@ -332,6 +371,7 @@ class FileHandler(object):
self.jobCache["decrypt"] = "empty"
return None
+
def getFileCount(self):
"""returns number of files"""
@@ -340,6 +380,7 @@ class FileHandler(object):
return self.filecount
+
def getQueueCount(self, force=False):
"""number of files that have to be processed"""
if self.queuecount == -1 or force:
@@ -347,6 +388,7 @@ class FileHandler(object):
return self.queuecount
+
def checkAllLinksFinished(self):
"""checks if all files are finished and dispatch event"""
@@ -357,6 +399,7 @@ class FileHandler(object):
return False
+
def checkAllLinksProcessed(self, fid):
"""checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting"""
@@ -370,11 +413,14 @@ class FileHandler(object):
return False
+
def resetCount(self):
self.queuecount = -1
@lock
@change
+
+
def restartPackage(self, id):
"""restart package"""
pyfiles = self.cache.values()
@@ -392,6 +438,8 @@ class FileHandler(object):
@lock
@change
+
+
def restartFile(self, id):
""" restart file"""
if id in self.cache:
@@ -407,6 +455,8 @@ class FileHandler(object):
@lock
@change
+
+
def setPackageLocation(self, id, queue):
"""push package to queue"""
@@ -440,6 +490,8 @@ class FileHandler(object):
@lock
@change
+
+
def reorderPackage(self, id, position):
p = self.getPackage(id)
@@ -467,6 +519,8 @@ class FileHandler(object):
@lock
@change
+
+
def reorderFile(self, id, position):
f = self.getFileData(id)
f = f[id]
@@ -497,12 +551,15 @@ class FileHandler(object):
self.core.pullManager.addEvent(e)
@change
+
+
def updateFileInfo(self, data, pid):
""" updates file info (name, size, status, url)"""
ids = self.db.updateLinkInfo(data)
e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue")
self.core.pullManager.addEvent(e)
+
def checkPackageFinished(self, pyfile):
""" checks if package is finished and calls AddonManager """
@@ -513,6 +570,7 @@ class FileHandler(object):
self.core.addonManager.packageFinished(pyfile.package())
pyfile.package().setFinished = True
+
def reCheckPackage(self, pid):
""" recheck links in package """
data = self.db.getPackageData(pid)
@@ -527,6 +585,8 @@ class FileHandler(object):
@lock
@change
+
+
def deleteFinishedLinks(self):
""" deletes finished links and packages, return deleted packages """
@@ -549,6 +609,8 @@ class FileHandler(object):
@lock
@change
+
+
def restartFailed(self):
""" restart all failed links """
self.db.restartFailed()
@@ -556,24 +618,32 @@ class FileHandler(object):
class FileMethods(object):
@style.queue
+
+
def filecount(self, queue):
"""returns number of files in queue"""
self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue,))
return self.c.fetchone()[0]
@style.queue
+
+
def queuecount(self, queue):
""" number of files in queue not finished yet"""
self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0, 4)", (queue,))
return self.c.fetchone()[0]
@style.queue
+
+
def processcount(self, queue, fid):
""" number of files which have to be proccessed """
self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2, 3, 5, 7, 12) AND l.id != ?", (queue, str(fid)))
return self.c.fetchone()[0]
@style.inner
+
+
def _nextPackageOrder(self, queue=0):
self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,))
max = self.c.fetchone()[0]
@@ -583,6 +653,8 @@ class FileMethods(object):
return 0
@style.inner
+
+
def _nextFileOrder(self, package):
self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,))
max = self.c.fetchone()[0]
@@ -592,12 +664,16 @@ class FileMethods(object):
return 0
@style.queue
+
+
def addLink(self, url, name, plugin, package):
order = self._nextFileOrder(package)
self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', (url, name, ".".join(plugintype, pluginname), package, order))
return self.c.lastrowid
@style.queue
+
+
def addLinks(self, links, package):
""" links is a list of tupels (url, plugin)"""
order = self._nextFileOrder(package)
@@ -606,23 +682,31 @@ class FileMethods(object):
self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links)
@style.queue
+
+
def addPackage(self, name, folder, queue):
order = self._nextPackageOrder(queue)
self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', (name, folder, queue, order))
return self.c.lastrowid
@style.queue
+
+
def deletePackage(self, p):
self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),))
self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),))
self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue))
@style.queue
+
+
def deleteLink(self, f):
self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),))
self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid)))
@style.queue
+
+
def getAllLinks(self, q):
"""return information about all links in queue q
@@ -656,6 +740,8 @@ class FileMethods(object):
return data
@style.queue
+
+
def getAllPackages(self, q):
"""return information about packages in queue q
(only useful in get all data)
@@ -693,6 +779,8 @@ class FileMethods(object):
return data
@style.queue
+
+
def getLinkData(self, id):
"""get link information as dict"""
self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?', (str(id),))
@@ -717,6 +805,8 @@ class FileMethods(object):
return data
@style.queue
+
+
def getPackageData(self, id):
"""get data about links for a package"""
self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id),))
@@ -740,14 +830,20 @@ class FileMethods(object):
return data
@style.async
+
+
def updateLink(self, f):
self.c.execute('UPDATE links SET url=?, name=?, size=?, status=?, error=?, package=? WHERE id=?', (f.url, f.name, f.size, f.status, str(f.error), str(f.packageid), str(f.id)))
@style.queue
+
+
def updatePackage(self, p):
self.c.execute('UPDATE packages SET name=?, folder=?, site=?, password=?, queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id)))
@style.queue
+
+
def updateLinkInfo(self, data):
""" data is list of tupels (name, size, status, url) """
self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1, 2, 3, 14)', data)
@@ -758,6 +854,8 @@ class FileMethods(object):
return ids
@style.queue
+
+
def reorderPackage(self, p, position, noMove=False):
if position == -1:
position = self._nextPackageOrder(p.queue)
@@ -770,6 +868,8 @@ class FileMethods(object):
self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id)))
@style.queue
+
+
def reorderLink(self, f, position):
""" reorder link with f as dict for pyfile """
if f["order"] > position:
@@ -780,19 +880,27 @@ class FileMethods(object):
self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"]))
@style.queue
+
+
def clearPackageOrder(self, p):
self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id)))
self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id)))
@style.async
+
+
def restartFile(self, id):
self.c.execute('UPDATE links SET status=3, error="" WHERE id=?', (str(id),))
@style.async
+
+
def restartPackage(self, id):
self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),))
@style.queue
+
+
def getPackage(self, id):
"""return package instance from id"""
self.c.execute("SELECT name, folder, site, password, queue, packageorder FROM packages WHERE id=?", (str(id),))
@@ -800,7 +908,9 @@ class FileMethods(object):
if not r: return None
return PyPackage(self.manager, id, * r)
+
#--------------------------------------------------------------------------
+
@style.queue
def getFile(self, id):
"""return link instance from id"""
@@ -812,6 +922,8 @@ class FileMethods(object):
return PyFile(self.manager, id, * r)
@style.queue
+
+
def getJob(self, occ):
"""return pyfile ids, which are suitable for download and dont use a occupied plugin"""
@@ -832,6 +944,8 @@ class FileMethods(object):
return [x[0] for x in self.c]
@style.queue
+
+
def getPluginJob(self, plugins):
"""returns pyfile ids with suited plugins"""
cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins
@@ -841,6 +955,8 @@ class FileMethods(object):
return [x[0] for x in self.c]
@style.queue
+
+
def getUnfinished(self, pid):
"""return list of max length 3 ids with pyfiles in package not finished or processed"""
@@ -848,21 +964,29 @@ class FileMethods(object):
return [r[0] for r in self.c]
@style.queue
+
+
def deleteFinished(self):
self.c.execute("DELETE FROM links WHERE status IN (0, 4)")
self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)")
@style.queue
+
+
def restartFailed(self):
self.c.execute("UPDATE links SET status=3, error='' WHERE status IN (6, 8, 9)")
@style.queue
+
+
def findDuplicates(self, id, folder, filename):
""" checks if filename exists with different id and same package """
self.c.execute("SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", (folder, id, filename))
return self.c.fetchone()
@style.queue
+
+
def purgeLinks(self):
self.c.execute("DELETE FROM links;")
self.c.execute("DELETE FROM packages;")
diff --git a/pyload/database/Storage.py b/pyload/database/Storage.py
index 75e166d39..bd7844d8d 100644
--- a/pyload/database/Storage.py
+++ b/pyload/database/Storage.py
@@ -6,6 +6,8 @@ from pyload.database import DatabaseBackend
class StorageMethods(object):
@style.queue
+
+
def setStorage(db, identifier, key, value):
db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key))
if db.c.fetchone() is not None:
@@ -14,6 +16,8 @@ class StorageMethods(object):
db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value))
@style.queue
+
+
def getStorage(db, identifier, key=None):
if key is not None:
db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key))
@@ -28,6 +32,8 @@ class StorageMethods(object):
return d
@style.queue
+
+
def delStorage(db, identifier, key):
db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key))
diff --git a/pyload/database/User.py b/pyload/database/User.py
index 54545e588..52b05749f 100644
--- a/pyload/database/User.py
+++ b/pyload/database/User.py
@@ -8,6 +8,8 @@ from pyload.database import DatabaseBackend, style
class UserMethods(object):
@style.queue
+
+
def checkAuth(db, user, password):
c = db.c
c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user,))
@@ -25,6 +27,8 @@ class UserMethods(object):
return {}
@style.queue
+
+
def addUser(db, user, password):
salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)])
h = sha1(salt + password)
@@ -39,6 +43,8 @@ class UserMethods(object):
@style.queue
+
+
def changePassword(db, user, oldpw, newpw):
db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user,))
r = db.c.fetchone()
@@ -60,15 +66,21 @@ class UserMethods(object):
@style.async
+
+
def setPermission(db, user, perms):
db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user))
@style.async
+
+
def setRole(db, user, role):
db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user))
@style.queue
+
+
def listUsers(db):
db.c.execute('SELECT name FROM users')
users = []
@@ -77,6 +89,8 @@ class UserMethods(object):
return users
@style.queue
+
+
def getAllUserData(db):
db.c.execute("SELECT name, permission, role, template, email FROM users")
user = {}
@@ -86,6 +100,8 @@ class UserMethods(object):
return user
@style.queue
+
+
def removeUser(db, user):
db.c.execute('DELETE FROM users WHERE name=?', (user,))
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-23 05:10:41 +0100