diff options
Diffstat (limited to 'module')
| -rw-r--r-- | module/FileManager.py | 26 | ||||
| -rw-r--r-- | module/PyFile.py | 40 | ||||
| -rw-r--r-- | module/PyPackage.py | 42 | ||||
| -rw-r--r-- | module/database/DatabaseBackend.py | 48 | ||||
| -rw-r--r-- | module/database/FileDatabase.py | 184 | ||||
| -rw-r--r-- | module/lib/hg_tool.py | 133 | ||||
| -rw-r--r-- | module/remote/socketbackend/ttypes.py | 16 | ||||
| -rw-r--r-- | module/remote/thriftbackend/pyload.thrift | 48 | ||||
| -rw-r--r-- | module/remote/thriftbackend/thriftgen/pyload/Pyload.py | 2 | ||||
| -rw-r--r-- | module/remote/thriftbackend/thriftgen/pyload/ttypes.py | 52 | ||||
| -rw-r--r-- | module/web/pyload_app.py | 6 | ||||
| -rw-r--r-- | module/web/utils.py | 2 |
12 files changed, 395 insertions, 204 deletions
diff --git a/module/FileManager.py b/module/FileManager.py index cb3b2ae96..5a0d2b958 100644 --- a/module/FileManager.py +++ b/module/FileManager.py @@ -1,6 +1,20 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + from time import time from threading import RLock from module.utils import lock @@ -19,6 +33,8 @@ def invalidate(func): return new +# TODO: needs to be replaced later +OWNER = 0 class FileManager: """Handles all request made to obtain information, @@ -78,7 +94,7 @@ class FileManager: @invalidate def addLinks(self, data, package): """Add links, data = (plugin, url) tuple. Internal method should use API.""" - self.db.addLinks(data, package) + self.db.addLinks(data, package, OWNER) self.evm.dispatchEvent("packageUpdated", package) @@ -86,7 +102,7 @@ class FileManager: def addPackage(self, name, folder, root, password, site, comment, paused): """Adds a package to database""" pid = self.db.addPackage(name, folder, root, password, site, comment, - PackageStatus.Paused if paused else PackageStatus.Ok) + PackageStatus.Paused if paused else PackageStatus.Ok, OWNER) p = self.db.getPackageInfo(pid) self.evm.dispatchEvent("packageInserted", pid, p.root, p.packageorder) @@ -97,7 +113,7 @@ class FileManager: def getPackage(self, pid): """return package instance""" if pid == self.ROOT_PACKAGE: - return RootPackage(self) + return RootPackage(self, OWNER) elif pid in self.packages: pack = self.packages[pid] pack.timestamp = time() @@ -115,7 +131,7 @@ class FileManager: def getPackageInfo(self, pid): """returns dict with package information""" if pid == self.ROOT_PACKAGE: - pack = RootPackage(self).toInfoData() + pack = RootPackage(self, OWNER).toInfoData() elif pid in self.packages: pack = self.packages[pid].toInfoData() pack.stats = self.db.getStatsForPackage(pid) @@ -183,7 +199,7 @@ class FileManager: # root package is not in database, create an instance if pid == self.ROOT_PACKAGE: - view.root = RootPackage(self).toInfoData() + view.root = RootPackage(self, OWNER).toInfoData() packs[self.ROOT_PACKAGE] = view.root elif pid in packs: view.root = packs[pid] diff --git a/module/PyFile.py b/module/PyFile.py index 5e6a3fae3..4cd0488a0 100644 --- a/module/PyFile.py +++ b/module/PyFile.py @@ -1,20 +1,19 @@ #!/usr/bin/env python -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### from time import sleep, time from threading import RLock @@ -48,14 +47,14 @@ class PyFile(object): Represents a file object at runtime """ __slots__ = ("m", "fid", "_name", "_size", "filestatus", "media", "added", "fileorder", - "url", "pluginname", "hash", "status", "error", "packageid", + "url", "pluginname", "hash", "status", "error", "packageid", "ownerid", "lock", "plugin", "waitUntil", "active", "abort", "statusname", "reconnected", "progress", "maxprogress", "pluginclass") @staticmethod def fromInfoData(m, info): f = PyFile(m, info.fid, info.name, info.size, info.status, info.media, info.added, info.fileorder, - "", "", "", DownloadStatus.NA, "", info.package) + "", "", "", DownloadStatus.NA, "", info.package, info.owner) if info.download: f.url = info.download.url f.pluginname = info.download.plugin @@ -66,7 +65,7 @@ class PyFile(object): return f def __init__(self, manager, fid, name, size, filestatus, media, added, fileorder, - url, pluginname, hash, status, error, package): + url, pluginname, hash, status, error, package, owner): self.m = manager @@ -82,6 +81,7 @@ class PyFile(object): self.hash = hash self.status = status self.error = error + self.ownerid = owner self.packageid = package #should not be used, use package() instead # database information ends here @@ -183,7 +183,7 @@ class PyFile(object): def toInfoData(self): - return FileInfo(self.fid, self.getName(), self.packageid, self.getSize(), self.filestatus, + return FileInfo(self.fid, self.getName(), self.packageid, self.ownerid, self.getSize(), self.filestatus, self.media, self.added, self.fileorder, DownloadInfo( self.url, self.pluginname, self.hash, self.status, self.getStatusName(), self.error ) diff --git a/module/PyPackage.py b/module/PyPackage.py index d0739124f..1dc2754ef 100644 --- a/module/PyPackage.py +++ b/module/PyPackage.py @@ -1,20 +1,19 @@ #!/usr/bin/env python -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### from time import time @@ -29,16 +28,17 @@ class PyPackage: @staticmethod def fromInfoData(m, info): - return PyPackage(m, info.pid, info.name, info.folder, info.root, + return PyPackage(m, info.pid, info.name, info.folder, info.root, info.owner, info.site, info.comment, info.password, info.added, info.status, info.packageorder) - def __init__(self, manager, pid, name, folder, root, site, comment, password, added, status, packageorder): + def __init__(self, manager, pid, name, folder, root, owner, site, comment, password, added, status, packageorder): self.m = manager self.pid = pid self.name = name self.folder = folder self.root = root + self.ownerid = owner self.site = site self.comment = comment self.password = password @@ -56,7 +56,7 @@ class PyPackage: return self.timestamp + 30 * 60 > time() def toInfoData(self): - return PackageInfo(self.pid, self.name, self.folder, self.root, self.site, + return PackageInfo(self.pid, self.name, self.folder, self.root, self.ownerid, self.site, self.comment, self.password, self.added, self.status, self.packageorder ) @@ -92,8 +92,8 @@ class PyPackage: class RootPackage(PyPackage): - def __init__(self, m): - PyPackage.__init__(self, m, -1, "root", "", -2, "", "", "", 0, PackageStatus.Ok, 0) + def __init__(self, m, owner): + PyPackage.__init__(self, m, -1, "root", "", owner, -2, "", "", "", 0, PackageStatus.Ok, 0) def getPath(self, name=""): return join(self.m.core.config["general"]["download_folder"], name) diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 516aa981f..ec39e3fd9 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -1,21 +1,21 @@ #!/usr/bin/env python -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN - @author: mkaay -""" +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +# @author: mkaay +############################################################################### + from threading import Thread, Event from shutil import move @@ -245,8 +245,10 @@ class DatabaseBackend(Thread): '"added" INTEGER DEFAULT 0 NOT NULL,' # set by trigger '"status" INTEGER DEFAULT 0 NOT NULL,' '"packageorder" INTEGER DEFAULT -1 NOT NULL,' #incremented by trigger - '"root" INTEGER DEFAULT -1 NOT NULL,' - 'CHECK (root != pid) ' + '"root" INTEGER DEFAULT -1 NOT NULL, ' + '"owner" INTEGER NOT NULL, ' + 'FOREIGN KEY(owner) REFERENCES users(uid), ' + 'CHECK (root != pid)' ')' ) @@ -267,7 +269,8 @@ class DatabaseBackend(Thread): 'END' ) - self.c.execute('CREATE INDEX IF NOT EXISTS "root_index" ON packages(root)') + self.c.execute('CREATE INDEX IF NOT EXISTS "package_index" ON packages(root, owner)') + self.c.execute('CREATE INDEX IF NOT EXISTS "package_owner" ON packages(owner)') self.c.execute( 'CREATE TABLE IF NOT EXISTS "files" (' @@ -284,11 +287,14 @@ class DatabaseBackend(Thread): '"dlstatus" INTEGER DEFAULT 0 NOT NULL, ' '"error" TEXT DEFAULT "" NOT NULL, ' '"package" INTEGER NOT NULL, ' + '"owner" INTEGER NOT NULL, ' + 'FOREIGN KEY(owner) REFERENCES users(uid), ' 'FOREIGN KEY(package) REFERENCES packages(id)' ')' ) - self.c.execute('CREATE INDEX IF NOT EXISTS "package_index" ON files(package)') + self.c.execute('CREATE INDEX IF NOT EXISTS "file_index" ON files(package, owner)') + self.c.execute('CREATE INDEX IF NOT EXISTS "file_owner" ON files(owner)') self.c.execute( 'CREATE TRIGGER IF NOT EXISTS "insert_file" AFTER INSERT ON "files"' diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index b783d15d9..80da775c7 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -1,88 +1,97 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + from new_collections import OrderedDict -from module.Api import DownloadInfo, LinkStatus, FileInfo, PackageInfo, PackageStats +from module.Api import DownloadInfo, FileInfo, PackageInfo, PackageStats from module.database import DatabaseMethods, queue, async, inner -default = PackageStats(0, 0, 0, 0) +zero_stats = PackageStats(0, 0, 0, 0) class FileMethods(DatabaseMethods): @queue - def filecount(self): + def filecount(self, user=None): """returns number of files""" self.c.execute("SELECT COUNT(*) FROM files") return self.c.fetchone()[0] @queue - def queuecount(self): + def queuecount(self, user=None): """ number of files in queue not finished yet""" # status not in NA, finished, skipped self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus NOT IN (0,5,6)") return self.c.fetchone()[0] @queue - def processcount(self, fid): + def processcount(self, fid, user=None): """ number of files which have to be processed """ # status in online, queued, starting, waiting, downloading self.c.execute("SELECT COUNT(*) FROM files as WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (str(fid), )) return self.c.fetchone()[0] + # TODO: think about multiuser side effects on *count methods + @queue - def addLink(self, url, name, plugin, package): + def addLink(self, url, name, plugin, package, owner): # mark file status initially as missing, dlstatus - queued - self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', - (url, name, plugin, package)) + self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', + (url, name, plugin, package, owner)) return self.c.lastrowid @async - def addLinks(self, links, package): + def addLinks(self, links, package, owner): """ links is a list of tuples (url, plugin)""" - links = [(x[0], x[0], x[1], package) for x in links] - self.c.executemany('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', links) + links = [(x[0], x[0], x[1], package, owner) for x in links] + self.c.executemany('INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', + links) @queue - def addFile(self, name, size, media, package): + def addFile(self, name, size, media, package, owner): # file status - ok, dl status NA - self.c.execute('INSERT INTO files(name, size, media, package) VALUES(?,?,?,?)', - (name, size, media, package)) + self.c.execute('INSERT INTO files(name, size, media, package, owner) VALUES(?,?,?,?,?)', + (name, size, media, package, owner)) return self.c.lastrowid @queue - def addPackage(self, name, folder, root, password, site, comment, status): - self.c.execute('INSERT INTO packages(name, folder, root, password, site, comment, status) VALUES(?,?,?,?,?,?,?)' - , - (name, folder, root, password, site, comment, status)) + def addPackage(self, name, folder, root, password, site, comment, status, owner): + self.c.execute( + 'INSERT INTO packages(name, folder, root, password, site, comment, status, owner) VALUES(?,?,?,?,?,?,?,?)' + , (name, folder, root, password, site, comment, status, owner)) return self.c.lastrowid @async - def deletePackage(self, pid): - # order updated by trigger - self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) + def deletePackage(self, pid, owner=None): + # order updated by trigge + if owner is None: + self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) + else: + self.c.execute('DELETE FROM packages WHERE pid=? AND owner=?', (pid, owner)) @async - def deleteFile(self, fid, order, package): + def deleteFile(self, fid, order, package, owner=None): """ To delete a file order and package of it is needed """ - self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) - self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', - (order, package)) + if owner is None: + self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) + self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', + (order, package)) + else: + self.c.execute('DELETE FROM files WHERE fid=? AND owner=?', (fid, owner)) + self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=? AND owner=?', + (order, package, owner)) @async def saveCollector(self, owner, data): @@ -92,8 +101,10 @@ class FileMethods(DatabaseMethods): @queue def retrieveCollector(self, owner): """ retrieve the saved string """ - self.c.execute('SELECT data FROM collector owner=?', (owner,)) - return self.c.fetchone()[0] + self.c.execute('SELECT data FROM collector WHERE owner=?', (owner,)) + r = self.c.fetchone() + if not r: return None + return r[0] @async def deleteCollector(self, owner): @@ -101,66 +112,81 @@ class FileMethods(DatabaseMethods): self.c.execute('DELETE FROM collector WHERE owner=?', (owner,)) @queue - def getAllFiles(self, package=None, search=None, unfinished=False): + def getAllFiles(self, package=None, search=None, unfinished=False, owner=None): """ Return dict with file information :param package: optional package to filter out :param search: or search string for file name :param unfinished: filter by dlstatus not finished + :param owner: only specific owner """ - qry = ('SELECT fid, name, size, status, media, added, fileorder, ' - 'url, plugin, hash, dlstatus, error, package FROM files') + qry = ('SELECT fid, name, owner, size, status, media, added, fileorder, ' + 'url, plugin, hash, dlstatus, error, package FROM files WHERE ') + + arg = [] if unfinished: - qry += ' WHERE dlstatus NOT IN (0, 5, 6)' + qry += 'dlstatus NOT IN (0, 5, 6) AND ' + if owner is not None: + qry += 'owner=? AND ' + arg.append(owner) if package is not None: - qry += ' AND' if unfinished else ' WHERE' - self.c.execute(qry + ' package=? ORDER BY package, fileorder', (package,)) - elif search is not None: - qry += ' AND' if unfinished else ' WHERE' + arg.append(package) + qry += 'package=? AND ' + if search is not None: search = "%%%s%%" % search.strip("%") - self.c.execute(qry + ' name LIKE ? ORDER BY package, fileorder', (search,)) + arg.append(search) + qry += "name LIKE ? " - else: - self.c.execute(qry) + # make qry valid + if qry.endswith("WHERE "): qry = qry[:-6] + if qry.endswith("AND "): qry = qry[:-4] + + self.c.execute(qry + "ORDER BY package, fileorder", arg) data = OrderedDict() for r in self.c: - f = FileInfo(r[0], r[1], r[12], r[2], r[3], r[4], r[5], r[6]) - if r[10] > 0: # dl status != NA - f.download = DownloadInfo(r[7], r[8], r[9], r[10], self.manager.statusMsg[r[10]], r[11]) + f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) + if r[11] > 0: # dl status != NA + f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) data[r[0]] = f return data @queue - def getAllPackages(self, root=None): + def getAllPackages(self, root=None, owner=None): """ Return dict with package information :param root: optional root to filter """ - qry = ('SELECT pid, name, folder, root, site, comment, password, added, status, packageorder ' + qry = ('SELECT pid, name, folder, root, owner, site, comment, password, added, status, packageorder ' 'FROM packages%s ORDER BY root, packageorder') if root is None: - stats = self.getPackageStats() - self.c.execute(qry % "") + stats = self.getPackageStats(owner=owner) + if owner is None: + self.c.execute(qry % "") + else: + self.c.execute(qry % " WHERE owner=?", (owner,)) else: - stats = self.getPackageStats(root=root) - self.c.execute(qry % ' WHERE root=? OR pid=?', (root, root)) + stats = self.getPackageStats(root=root, owner=owner) + if owner is None: + self.c.execute(qry % ' WHERE root=? OR pid=?', (root, root)) + else: + self.c.execute(qry % ' WHERE (root=? OR pid=?) AND owner=?', (root, root, owner)) data = OrderedDict() for r in self.c: data[r[0]] = PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], stats.get(r[0], default) + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], stats.get(r[0], zero_stats) ) return data @inner - def getPackageStats(self, pid=None, root=None): + def getPackageStats(self, pid=None, root=None, owner=None): qry = ("SELECT p.pid, SUM(f.size) AS sizetotal, COUNT(f.fid) AS linkstotal, sizedone, linksdone " "FROM packages p JOIN files f ON p.pid = f.package AND f.dlstatus > 0 %(sub)s LEFT OUTER JOIN " "(SELECT p.pid AS pid, SUM(f.size) AS sizedone, COUNT(f.fid) AS linksdone " @@ -173,6 +199,8 @@ class FileMethods(DatabaseMethods): self.c.execute(qry % {"sub": "AND (p.root=:root OR p.pid=:root)"}, locals()) elif pid is not None: self.c.execute(qry % {"sub": "AND p.pid=:pid"}, locals()) + elif owner is not None: + self.c.execute(qry % {"sub": "AND p.owner=:owner"}, locals()) else: self.c.execute(qry % {"sub": ""}) @@ -193,17 +221,17 @@ class FileMethods(DatabaseMethods): @queue def getFileInfo(self, fid, force=False): - """get data for specific file""" - self.c.execute('SELECT fid, name, size, status, media, added, fileorder, ' + """get data for specific file, when force is true download info will be appended""" + self.c.execute('SELECT fid, name, owner, size, status, media, added, fileorder, ' 'url, plugin, hash, dlstatus, error, package FROM files ' 'WHERE fid=?', (fid,)) r = self.c.fetchone() if not r: return None else: - f = FileInfo(r[0], r[1], r[12], r[2], r[3], r[4], r[5], r[6]) - if r[10] > 0 or force: - f.download = DownloadInfo(r[7], r[8], r[9], r[10], self.manager.statusMsg[r[10]], r[11]) + f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) + if r[11] > 0 or force: + f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) return f @@ -213,7 +241,7 @@ class FileMethods(DatabaseMethods): if stats: stats = self.getPackageStats(pid=pid) - self.c.execute('SELECT pid, name, folder, root, site, comment, password, added, status, packageorder ' + self.c.execute('SELECT pid, name, folder, root, owner, site, comment, password, added, status, packageorder ' 'FROM packages WHERE pid=?', (pid,)) r = self.c.fetchone() @@ -221,11 +249,11 @@ class FileMethods(DatabaseMethods): return None else: return PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], stats.get(r[0], default) if stats else None + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], stats.get(r[0], zero_stats) if stats else None ) @async - def updateLinkInfo(self, data): + def updateLinkInfo(self, data, owner): """ data is list of tuples (name, size, status,[ hash,] url)""" if data and len(data[0]) == 4: self.c.executemany('UPDATE files SET name=?, size=?, dlstatus=? WHERE url=? AND dlstatus IN (0,1,2,3,14)', @@ -246,6 +274,7 @@ class FileMethods(DatabaseMethods): self.c.execute('UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, status=? WHERE pid=?', (p.name, p.folder, p.site, p.comment, p.password, p.status, p.pid)) + # TODO: most modifying methods needs owner argument to avoid checking beforehand @async def orderPackage(self, pid, root, oldorder, order): if oldorder > order: # package moved upwards @@ -293,11 +322,6 @@ class FileMethods(DatabaseMethods): self.c.execute('SELECT max(packageorder) FROM packages WHERE root=?', (dpid,)) r = self.c.fetchone() max = (r[0] if r[0] else 0) + 1 - print max - - self.c.execute('SELECT pid, packageorder FROM packages WHERE root=?', (dpid,)) - for r in self.c: - print r self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND root=?', (order, root)) @@ -314,6 +338,8 @@ class FileMethods(DatabaseMethods): # status -> queued self.c.execute('UPDATE files SET status=3 WHERE package=?', (pid,)) + + # TODO: multi user approach @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" @@ -338,7 +364,7 @@ class FileMethods(DatabaseMethods): return [r[0] for r in self.c] @queue - def restartFailed(self): + def restartFailed(self, owner): # status=queued, where status in failed, aborted, temp offline self.c.execute("UPDATE files SET dlstatus=3, error='' WHERE dlstatus IN (7, 11, 12)") diff --git a/module/lib/hg_tool.py b/module/lib/hg_tool.py new file mode 100644 index 000000000..cd97833df --- /dev/null +++ b/module/lib/hg_tool.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from subprocess import Popen, PIPE +from time import time, gmtime, strftime + +aliases = {"zoidber": "zoidberg", "zoidberg10": "zoidberg", "webmaster": "dhmh", "mast3rranan": "ranan", + "ranan2": "ranan"} +exclude = ["locale/*", "module/lib/*"] +date_format = "%Y-%m-%d" +line_re = re.compile(r" (\d+) \**", re.I) + +def add_exclude_flags(args): + for dir in exclude: + args.extend(["-X", dir]) + +# remove small percentages +def wipe(data, perc=1): + s = (sum(data.values()) * perc) / 100 + for k, v in data.items(): + if v < s: del data[k] + + return data + +# remove aliases +def de_alias(data): + for k, v in aliases.iteritems(): + if k not in data: continue + alias = aliases[k] + + if alias in data: data[alias] += data[k] + else: data[alias] = data[k] + + del data[k] + + return data + + +def output(data): + s = float(sum(data.values())) + print "Total Lines: %d" % s + for k, v in data.iteritems(): + print "%15s: %.1f%% | %d" % (k, (v * 100) / s, v) + print + + +def file_list(): + args = ["hg", "status", "-A"] + add_exclude_flags(args) + p = Popen(args, stdout=PIPE) + out, err = p.communicate() + return [x.split()[1] for x in out.splitlines() if x.split()[0] in "CMA"] + + +def hg_annotate(path): + args = ["hg", "annotate", "-u", path] + p = Popen(args, stdout=PIPE) + out, err = p.communicate() + + data = {} + + for line in out.splitlines(): + author, non, line = line.partition(":") + + # probably binary file + if author == path: return {} + + author = author.strip().lower() + if not line.strip(): continue # don't count blank lines + + if author in data: data[author] += 1 + else: data[author] = 1 + + return de_alias(data) + + +def hg_churn(days=None): + args = ["hg", "churn"] + if days: + args.append("-d") + t = time() - 60 * 60 * 24 * days + args.append("%s to %s" % (strftime(date_format, gmtime(t)), strftime(date_format))) + + add_exclude_flags(args) + p = Popen(args, stdout=PIPE) + out, err = p.communicate() + + data = {} + + for line in out.splitlines(): + m = line_re.search(line) + author = line.split()[0] + lines = int(m.group(1)) + + if "@" in author: + author, n, email = author.partition("@") + + author = author.strip().lower() + + if author in data: data[author] += lines + else: data[author] = lines + + return de_alias(data) + + +def complete_annotate(): + files = file_list() + data = {} + for f in files: + tmp = hg_annotate(f) + for k, v in tmp.iteritems(): + if k in data: data[k] += v + else: data[k] = v + + return data + + +if __name__ == "__main__": + for d in (30, 90, 180): + c = wipe(hg_churn(d)) + print "Changes in %d days:" % d + output(c) + + c = wipe(hg_churn()) + print "Total changes:" + output(c) + + print "Current source code version:" + data = wipe(complete_annotate()) + output(data) + + diff --git a/module/remote/socketbackend/ttypes.py b/module/remote/socketbackend/ttypes.py index 36f2b01ef..127098ec3 100644 --- a/module/remote/socketbackend/ttypes.py +++ b/module/remote/socketbackend/ttypes.py @@ -159,12 +159,13 @@ class FileDoesNotExists(Exception): self.fid = fid class FileInfo(BaseObject): - __slots__ = ['fid', 'name', 'package', 'size', 'status', 'media', 'added', 'fileorder', 'download'] + __slots__ = ['fid', 'name', 'package', 'owner', 'size', 'status', 'media', 'added', 'fileorder', 'download'] - def __init__(self, fid=None, name=None, package=None, size=None, status=None, media=None, added=None, fileorder=None, download=None): + def __init__(self, fid=None, name=None, package=None, owner=None, size=None, status=None, media=None, added=None, fileorder=None, download=None): self.fid = fid self.name = name self.package = package + self.owner = owner self.size = size self.status = status self.media = media @@ -210,13 +211,14 @@ class PackageDoesNotExists(Exception): self.pid = pid class PackageInfo(BaseObject): - __slots__ = ['pid', 'name', 'folder', 'root', 'site', 'comment', 'password', 'added', 'status', 'packageorder', 'stats', 'fids', 'pids'] + __slots__ = ['pid', 'name', 'folder', 'root', 'owner', 'site', 'comment', 'password', 'added', 'status', 'packageorder', 'stats', 'fids', 'pids'] - def __init__(self, pid=None, name=None, folder=None, root=None, site=None, comment=None, password=None, added=None, status=None, packageorder=None, stats=None, fids=None, pids=None): + def __init__(self, pid=None, name=None, folder=None, root=None, owner=None, site=None, comment=None, password=None, added=None, status=None, packageorder=None, stats=None, fids=None, pids=None): self.pid = pid self.name = name self.folder = folder self.root = root + self.owner = owner self.site = site self.comment = comment self.password = password @@ -291,9 +293,9 @@ class ServiceException(Exception): self.msg = msg class UserData(BaseObject): - __slots__ = ['uid', 'name', 'email', 'role', 'permission', 'folder', 'traffic', 'limit', 'user', 'templateName'] + __slots__ = ['uid', 'name', 'email', 'role', 'permission', 'folder', 'traffic', 'dllimit', 'user', 'templateName'] - def __init__(self, uid=None, name=None, email=None, role=None, permission=None, folder=None, traffic=None, limit=None, user=None, templateName=None): + def __init__(self, uid=None, name=None, email=None, role=None, permission=None, folder=None, traffic=None, dllimit=None, user=None, templateName=None): self.uid = uid self.name = name self.email = email @@ -301,7 +303,7 @@ class UserData(BaseObject): self.permission = permission self.folder = folder self.traffic = traffic - self.limit = limit + self.dllimit = dllimit self.user = user self.templateName = templateName diff --git a/module/remote/thriftbackend/pyload.thrift b/module/remote/thriftbackend/pyload.thrift index 254f41068..181ca4204 100644 --- a/module/remote/thriftbackend/pyload.thrift +++ b/module/remote/thriftbackend/pyload.thrift @@ -84,13 +84,12 @@ enum Permission { All = 0, // requires no permission, but login Add = 1, // can add packages Delete = 2, // can delete packages - Status = 4, // see and change server status - List = 16, // see listed downloads - Modify = 32, // modify some attribute of downloads - Download = 64, // can download from webinterface - Accounts = 128, // can access accounts - Interaction = 256, // can interact with plugins - Addons = 512 // user can activate addons + Modify = 4, // modify some attribute of downloads + Status = 8, // see and change server status + Download = 16, // can download from webinterface + Accounts = 32, // can access accounts + Interaction = 64, // can interact with plugins + Addons = 128 // user can activate addons } enum Role { @@ -141,12 +140,13 @@ struct FileInfo { 1: FileID fid, 2: string name, 3: PackageID package, - 4: ByteCount size, - 5: FileStatus status, - 6: MediaType media, - 7: UTCDate added, - 8: i16 fileorder, - 9: optional DownloadInfo download, + 4: UserID owner, + 5: ByteCount size, + 6: FileStatus status, + 7: MediaType media, + 8: UTCDate added, + 9: i16 fileorder, + 10: optional DownloadInfo download, } struct PackageStats { @@ -161,15 +161,16 @@ struct PackageInfo { 2: string name, 3: string folder, 4: PackageID root, - 5: string site, - 6: string comment, - 7: string password, - 8: UTCDate added, - 9: PackageStatus status, - 10: i16 packageorder, - 11: PackageStats stats, - 12: list<FileID> fids, - 13: list<PackageID> pids, + 5: UserID owner, + 6: string site, + 7: string comment, + 8: string password, + 9: UTCDate added, + 10: PackageStatus status, + 11: i16 packageorder, + 12: PackageStats stats, + 13: list<FileID> fids, + 14: list<PackageID> pids, } // thrift does not allow recursive datatypes, so all data is accumulated and mapped with id @@ -266,10 +267,9 @@ struct AddonService { struct OnlineCheck { 1: ResultID rid, // -1 -> nothing more to get - 2: map<string, LinkStatus> data, //url to result + 2: map<string, LinkStatus> data, // url to result } - // exceptions exception PackageDoesNotExists { diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py index c45663d25..dd446cfc3 100644 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py @@ -9,7 +9,7 @@ from thrift.Thrift import TType, TMessageType, TException from ttypes import * from thrift.Thrift import TProcessor -from thrift.protocol.TBase import TBase, TExceptionBase, TApplicationException +from thrift.protocol.TBase import TBase, TExceptionBase class Iface(object): diff --git a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py index c177a9dd2..d170f4688 100644 --- a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py +++ b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py @@ -411,6 +411,7 @@ class FileInfo(TBase): - fid - name - package + - owner - size - status - media @@ -423,6 +424,7 @@ class FileInfo(TBase): 'fid', 'name', 'package', + 'owner', 'size', 'status', 'media', @@ -436,18 +438,20 @@ class FileInfo(TBase): (1, TType.I32, 'fid', None, None, ), # 1 (2, TType.STRING, 'name', None, None, ), # 2 (3, TType.I32, 'package', None, None, ), # 3 - (4, TType.I64, 'size', None, None, ), # 4 - (5, TType.I32, 'status', None, None, ), # 5 - (6, TType.I32, 'media', None, None, ), # 6 - (7, TType.I64, 'added', None, None, ), # 7 - (8, TType.I16, 'fileorder', None, None, ), # 8 - (9, TType.STRUCT, 'download', (DownloadInfo, DownloadInfo.thrift_spec), None, ), # 9 + (4, TType.I32, 'owner', None, None, ), # 4 + (5, TType.I64, 'size', None, None, ), # 5 + (6, TType.I32, 'status', None, None, ), # 6 + (7, TType.I32, 'media', None, None, ), # 7 + (8, TType.I64, 'added', None, None, ), # 8 + (9, TType.I16, 'fileorder', None, None, ), # 9 + (10, TType.STRUCT, 'download', (DownloadInfo, DownloadInfo.thrift_spec), None, ), # 10 ) - def __init__(self, fid=None, name=None, package=None, size=None, status=None, media=None, added=None, fileorder=None, download=None,): + def __init__(self, fid=None, name=None, package=None, owner=None, size=None, status=None, media=None, added=None, fileorder=None, download=None,): self.fid = fid self.name = name self.package = package + self.owner = owner self.size = size self.status = status self.media = media @@ -494,6 +498,7 @@ class PackageInfo(TBase): - name - folder - root + - owner - site - comment - password @@ -510,6 +515,7 @@ class PackageInfo(TBase): 'name', 'folder', 'root', + 'owner', 'site', 'comment', 'password', @@ -527,22 +533,24 @@ class PackageInfo(TBase): (2, TType.STRING, 'name', None, None, ), # 2 (3, TType.STRING, 'folder', None, None, ), # 3 (4, TType.I32, 'root', None, None, ), # 4 - (5, TType.STRING, 'site', None, None, ), # 5 - (6, TType.STRING, 'comment', None, None, ), # 6 - (7, TType.STRING, 'password', None, None, ), # 7 - (8, TType.I64, 'added', None, None, ), # 8 - (9, TType.I32, 'status', None, None, ), # 9 - (10, TType.I16, 'packageorder', None, None, ), # 10 - (11, TType.STRUCT, 'stats', (PackageStats, PackageStats.thrift_spec), None, ), # 11 - (12, TType.LIST, 'fids', (TType.I32,None), None, ), # 12 - (13, TType.LIST, 'pids', (TType.I32,None), None, ), # 13 + (5, TType.I32, 'owner', None, None, ), # 5 + (6, TType.STRING, 'site', None, None, ), # 6 + (7, TType.STRING, 'comment', None, None, ), # 7 + (8, TType.STRING, 'password', None, None, ), # 8 + (9, TType.I64, 'added', None, None, ), # 9 + (10, TType.I32, 'status', None, None, ), # 10 + (11, TType.I16, 'packageorder', None, None, ), # 11 + (12, TType.STRUCT, 'stats', (PackageStats, PackageStats.thrift_spec), None, ), # 12 + (13, TType.LIST, 'fids', (TType.I32,None), None, ), # 13 + (14, TType.LIST, 'pids', (TType.I32,None), None, ), # 14 ) - def __init__(self, pid=None, name=None, folder=None, root=None, site=None, comment=None, password=None, added=None, status=None, packageorder=None, stats=None, fids=None, pids=None,): + def __init__(self, pid=None, name=None, folder=None, root=None, owner=None, site=None, comment=None, password=None, added=None, status=None, packageorder=None, stats=None, fids=None, pids=None,): self.pid = pid self.name = name self.folder = folder self.root = root + self.owner = owner self.site = site self.comment = comment self.password = password @@ -809,7 +817,7 @@ class UserData(TBase): - permission - folder - traffic - - limit + - dllimit - user - templateName """ @@ -822,7 +830,7 @@ class UserData(TBase): 'permission', 'folder', 'traffic', - 'limit', + 'dllimit', 'user', 'templateName', ] @@ -836,12 +844,12 @@ class UserData(TBase): (5, TType.I16, 'permission', None, None, ), # 5 (6, TType.STRING, 'folder', None, None, ), # 6 (7, TType.I64, 'traffic', None, None, ), # 7 - (8, TType.I16, 'limit', None, None, ), # 8 + (8, TType.I16, 'dllimit', None, None, ), # 8 (9, TType.I32, 'user', None, None, ), # 9 (10, TType.STRING, 'templateName', None, None, ), # 10 ) - def __init__(self, uid=None, name=None, email=None, role=None, permission=None, folder=None, traffic=None, limit=None, user=None, templateName=None,): + def __init__(self, uid=None, name=None, email=None, role=None, permission=None, folder=None, traffic=None, dllimit=None, user=None, templateName=None,): self.uid = uid self.name = name self.email = email @@ -849,7 +857,7 @@ class UserData(TBase): self.permission = permission self.folder = folder self.traffic = traffic - self.limit = limit + self.dllimit = dllimit self.user = user self.templateName = templateName diff --git a/module/web/pyload_app.py b/module/web/pyload_app.py index 4c448d2cd..2234b76c6 100644 --- a/module/web/pyload_app.py +++ b/module/web/pyload_app.py @@ -36,7 +36,7 @@ from utils import render_to_response, parse_permissions, parse_userdata, \ from filters import relpath, unquotepath -from module.Api import Output +from module.Api import Output, Permission from module.utils import format_size from module.utils.fs import save_join, fs_encode, fs_decode, listdir @@ -150,7 +150,7 @@ def logout(): @route("/") @route("/home") -@login_required("LIST") +@login_required("List") def home(): try: res = [toDict(x) for x in PYLOAD.getProgressInfo()] @@ -168,7 +168,7 @@ def home(): @route("/queue") -@login_required("LIST") +@login_required("List") def queue(): queue = PYLOAD.getQueue() diff --git a/module/web/utils.py b/module/web/utils.py index ac4bdd4f8..1641fdbba 100644 --- a/module/web/utils.py +++ b/module/web/utils.py @@ -50,7 +50,7 @@ def parse_permissions(session): def permlist(): - return [x for x in dir(Permission) if not x.startswith("_") and x != "ALL"] + return [x for x in dir(Permission) if not x.startswith("_") and x != "All"] def get_permission(perms, p): |