summaryrefslogtreecommitdiffstats
path: root/module/database
diff options
context:
space:
mode:
authorGravatar Walter Purcaro <vuolter@gmail.com> 2014-03-28 22:32:14 +0100
committerGravatar Walter Purcaro <vuolter@gmail.com> 2014-06-28 02:47:08 +0200
commitb1fffc3a1b2dbbb807213b85f538e59251b9bf35 (patch)
treec373d3234dcb474bb424371a3d89341bed8a9e07 /module/database
parentPlugins licensing doc (diff)
downloadpyload-b1fffc3a1b2dbbb807213b85f538e59251b9bf35.tar.xz
Remove bad whitespaces
Merged vuolter/pyload@00288e6
Diffstat (limited to 'module/database')
-rw-r--r--module/database/DatabaseBackend.py74
-rw-r--r--module/database/FileDatabase.py59
-rw-r--r--module/database/StorageDatabase.py4
3 files changed, 67 insertions, 70 deletions
diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py
index 9530390c3..642289647 100644
--- a/module/database/DatabaseBackend.py
+++ b/module/database/DatabaseBackend.py
@@ -36,11 +36,11 @@ DB_VERSION = 4
class style():
db = None
-
+
@classmethod
def setDB(cls, db):
cls.db = db
-
+
@classmethod
def inner(cls, f):
@staticmethod
@@ -48,7 +48,7 @@ class style():
if cls.db:
return f(cls.db, *args, **kwargs)
return x
-
+
@classmethod
def queue(cls, f):
@staticmethod
@@ -56,7 +56,7 @@ class style():
if cls.db:
return cls.db.queue(f, *args, **kwargs)
return x
-
+
@classmethod
def async(cls, f):
@staticmethod
@@ -68,11 +68,11 @@ class style():
class DatabaseJob():
def __init__(self, f, *args, **kwargs):
self.done = Event()
-
+
self.f = f
self.args = args
self.kwargs = kwargs
-
+
self.result = None
self.exception = False
@@ -104,7 +104,7 @@ class DatabaseJob():
self.exception = e
finally:
self.done.set()
-
+
def wait(self):
self.done.wait()
@@ -116,34 +116,34 @@ class DatabaseBackend(Thread):
self.core = core
self.jobs = Queue()
-
+
self.setuplock = Event()
-
+
style.setDB(self)
-
+
def setup(self):
self.start()
self.setuplock.wait()
-
+
def run(self):
"""main loop, which executes commands"""
convert = self._checkVersion() #returns None or current version
-
+
self.conn = sqlite3.connect("files.db")
chmod("files.db", 0600)
self.c = self.conn.cursor() #compatibility
-
+
if convert is not None:
self._convertDB(convert)
-
+
self._createTables()
self._migrateUser()
self.conn.commit()
-
+
self.setuplock.set()
-
+
while True:
j = self.jobs.get()
if j == "quit":
@@ -164,7 +164,7 @@ class DatabaseBackend(Thread):
f.write(str(DB_VERSION))
f.close()
return
-
+
f = open("files.version", "rb")
v = int(f.read().strip())
f.close()
@@ -180,7 +180,7 @@ class DatabaseBackend(Thread):
f.write(str(DB_VERSION))
f.close()
return v
-
+
def _convertDB(self, v):
try:
getattr(self, "_convertV%i" % v)()
@@ -189,9 +189,9 @@ class DatabaseBackend(Thread):
self.core.log.error(_("Filedatabase could NOT be converted."))
except:
print "Filedatabase could NOT be converted."
-
+
#--convert scripts start
-
+
def _convertV2(self):
self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")')
try:
@@ -199,16 +199,16 @@ class DatabaseBackend(Thread):
except:
print "Database was converted from v2 to v3."
self._convertV3()
-
+
def _convertV3(self):
self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)')
try:
self.manager.core.log.info(_("Database was converted from v3 to v4."))
except:
print "Database was converted from v3 to v4."
-
+
#--convert scripts end
-
+
def _createTables(self):
"""create tables for database"""
@@ -264,10 +264,10 @@ class DatabaseBackend(Thread):
self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users)
move("pyload.db", "pyload.old.db")
-
+
def createCursor(self):
return self.conn.cursor()
-
+
@style.async
def commit(self):
self.conn.commit()
@@ -275,31 +275,31 @@ class DatabaseBackend(Thread):
@style.queue
def syncSave(self):
self.conn.commit()
-
+
@style.async
def rollback(self):
self.conn.rollback()
-
+
def async(self, f, *args, **kwargs):
args = (self, ) + args
job = DatabaseJob(f, *args, **kwargs)
self.jobs.put(job)
-
+
def queue(self, f, *args, **kwargs):
args = (self, ) + args
job = DatabaseJob(f, *args, **kwargs)
self.jobs.put(job)
job.wait()
return job.result
-
+
@classmethod
def registerSub(cls, klass):
cls.subs.append(klass)
-
+
@classmethod
def unregisterSub(cls, klass):
cls.subs.remove(klass)
-
+
def __getattr__(self, attr):
for sub in DatabaseBackend.subs:
if hasattr(sub, attr):
@@ -308,7 +308,7 @@ class DatabaseBackend(Thread):
if __name__ == "__main__":
db = DatabaseBackend()
db.setup()
-
+
class Test():
@style.queue
def insert(db):
@@ -320,21 +320,21 @@ if __name__ == "__main__":
c = db.createCursor()
for i in range(1000*1000):
c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar"))
-
+
@style.queue
def select(db):
c = db.createCursor()
for i in range(10):
res = c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", ("foo", i))
print res.fetchone()
-
+
@style.queue
def error(db):
c = db.createCursor()
print "a"
c.execute("SELECT myerror FROM storage WHERE identifier=? AND key=?", ("foo", i))
print "e"
-
+
db.registerSub(Test)
from time import time
start = time()
@@ -342,11 +342,11 @@ if __name__ == "__main__":
db.insert()
end = time()
print end-start
-
+
start = time()
db.insert2()
end = time()
print end-start
-
+
db.error()
diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py
index 7e7efb028..2f7a4a82d 100644
--- a/module/database/FileDatabase.py
+++ b/module/database/FileDatabase.py
@@ -190,7 +190,7 @@ class FileHandler:
pid = f.packageid
e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue")
-
+
oldorder = f.order
if id in self.core.threadManager.processingIds():
@@ -206,7 +206,7 @@ class FileHandler:
p = self.getPackage(pid)
if not len(p.getChildren()):
p.delete()
-
+
pyfiles = self.cache.values()
for pyfile in pyfiles:
if pyfile.packageid == pid and pyfile.order > oldorder:
@@ -434,7 +434,7 @@ class FileHandler:
e = RemoveEvent("pack", id, "collector" if not p.queue else "queue")
self.core.pullManager.addEvent(e)
-
+
self.db.clearPackageOrder(p)
p = self.db.getPackage(id)
@@ -443,7 +443,7 @@ class FileHandler:
self.db.updatePackage(p)
self.db.reorderPackage(p, -1, True)
-
+
packs = self.packageCache.values()
for pack in packs:
if pack.queue != queue and pack.order > oldorder:
@@ -453,7 +453,7 @@ class FileHandler:
self.db.commit()
self.releasePackage(id)
p = self.getPackage(id)
-
+
e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue")
self.core.pullManager.addEvent(e)
@@ -600,7 +600,7 @@ class FileMethods():
return max + 1
else:
return 0
-
+
@style.inner
def _nextFileOrder(self, package):
self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,))
@@ -609,7 +609,7 @@ class FileMethods():
return max + 1
else:
return 0
-
+
@style.queue
def addLink(self, url, name, plugin, package):
order = self._nextFileOrder(package)
@@ -713,7 +713,7 @@ class FileMethods():
}
return data
-
+
@style.queue
def getLinkData(self, id):
"""get link information as dict"""
@@ -769,7 +769,7 @@ class FileMethods():
@style.queue
def updatePackage(self, p):
self.c.execute('UPDATE packages SET name=?,folder=?,site=?,password=?,queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id)))
-
+
@style.queue
def updateLinkInfo(self, data):
""" data is list of tupels (name, size, status, url) """
@@ -779,7 +779,7 @@ class FileMethods():
for r in self.c:
ids.append(int(r[0]))
return ids
-
+
@style.queue
def reorderPackage(self, p, position, noMove=False):
if position == -1:
@@ -791,7 +791,7 @@ class FileMethods():
self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND queue=? AND packageorder >= 0', (position, p.order, p.queue))
self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id)))
-
+
@style.queue
def reorderLink(self, f, position):
""" reorder link with f as dict for pyfile """
@@ -801,13 +801,12 @@ class FileMethods():
self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder <= ? AND linkorder > ? AND package=?', (position, f["order"], f["package"]))
self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"]))
-
-
+
@style.queue
def clearPackageOrder(self, p):
self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id)))
self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id)))
-
+
@style.async
def restartFile(self, id):
self.c.execute('UPDATE links SET status=3,error="" WHERE id=?', (str(id),))
@@ -815,7 +814,7 @@ class FileMethods():
@style.async
def restartPackage(self, id):
self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),))
-
+
@style.queue
def getPackage(self, id):
"""return package instance from id"""
@@ -845,11 +844,11 @@ class FileMethods():
for i, item in enumerate(occ):
if i: cmd += ", "
cmd += "'%s'" % item
-
+
cmd += ")"
cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre)
-
+
self.c.execute(cmd) # very bad!
return [x[0] for x in self.c]
@@ -866,7 +865,7 @@ class FileMethods():
@style.queue
def getUnfinished(self, pid):
"""return list of max length 3 ids with pyfiles in package not finished or processed"""
-
+
self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),))
return [r[0] for r in self.c]
@@ -896,7 +895,7 @@ if __name__ == "__main__":
pypath = "."
_ = lambda x: x
-
+
db = FileHandler(None)
#p = PyFile(db, 5)
@@ -905,12 +904,11 @@ if __name__ == "__main__":
a = time()
#print db.addPackage("package", "folder" , 1)
-
+
pack = db.db.addPackage("package", "folder", 1)
-
+
updates = []
-
-
+
for x in range(0, 200):
x = str(x)
db.db.addLink("http://somehost.com/hoster/file/download?file_id=" + x, x, "BasePlugin", pack)
@@ -919,26 +917,25 @@ if __name__ == "__main__":
for x in range(0, 100):
updates.append(("unimportant%s" % x, 0, 3, "a really long non existent url%s" % x))
-
+
db.db.commit()
b = time()
print "adding 200 links, single sql execs, no commit", b-a
-
+
print db.getCompleteData(1)
c = time()
-
+
db.db.updateLinkInfo(updates)
-
+
d = time()
-
+
print "updates", d-c
print db.getCompleteData(1)
-
-
+
e = time()
-
+
print "complete data", e-d
diff --git a/module/database/StorageDatabase.py b/module/database/StorageDatabase.py
index 3ed29625f..a27866632 100644
--- a/module/database/StorageDatabase.py
+++ b/module/database/StorageDatabase.py
@@ -27,7 +27,7 @@ class StorageMethods():
db.c.execute("UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key))
else:
db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value))
-
+
@style.queue
def getStorage(db, identifier, key=None):
if key is not None:
@@ -41,7 +41,7 @@ class StorageMethods():
for row in db.c:
d[row[0]] = row[1]
return d
-
+
@style.queue
def delStorage(db, identifier, key):
db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key))