From c22d2aadb41b6014516b31c978634c4979d9a43c Mon Sep 17 00:00:00 2001 From: RaNaN Date: Fri, 21 Dec 2012 17:09:01 +0100 Subject: added tags attribute to packages --- module/Api.py | 39 +++--- module/database/FileDatabase.py | 23 ++-- module/remote/create_ttypes.py | 3 + module/remote/pyload.thrift | 5 +- module/remote/ttypes.py | 16 +-- module/remote/ttypes_debug.py | 6 +- module/web/static/js/models/Package.js | 1 + tests/other/test_database.py | 192 ----------------------------- tests/other/test_filedatabase.py | 212 +++++++++++++++++++++++++++++++++ 9 files changed, 266 insertions(+), 231 deletions(-) delete mode 100644 tests/other/test_database.py create mode 100644 tests/other/test_filedatabase.py diff --git a/module/Api.py b/module/Api.py index d1ccf43c0..0b777a659 100644 --- a/module/Api.py +++ b/module/Api.py @@ -696,6 +696,10 @@ class Api(Iface): def findFiles(self, pattern): pass + @RequirePerm(Permission.All) + def findPackages(self, tags): + pass + ############################# # Modify Downloads ############################# @@ -751,8 +755,22 @@ class Api(Iface): ############################# @RequirePerm(Permission.Modify) - def setPackagePaused(self, pid, paused): - pass + def updatePackage(self, pack): + """Allows to modify several package attributes. + + :param pid: package id + :param data: :class:`PackageInfo` + """ + pid = pack.pid + p = self.core.files.getPackage(pid) + if not p: raise PackageDoesNotExists(pid) + + for key, value in data.iteritems(): + if key == "id": continue + setattr(p, key, value) + + p.sync() + self.core.files.save() @RequirePerm(Permission.Modify) def setPackageFolder(self, pid, path): @@ -803,23 +821,6 @@ class Api(Iface): """ self.core.files.orderFiles(fids, pid, position) - @RequirePerm(Permission.Modify) - def setPackageData(self, pid, data): - """Allows to modify several package attributes. - - :param pid: package id - :param data: dict that maps attribute to desired value - """ - p = self.core.files.getPackage(pid) - if not p: raise PackageDoesNotExists(pid) - - for key, value in data.iteritems(): - if key == "id": continue - setattr(p, key, value) - - p.sync() - self.core.files.save() - ############################# # User Interaction ############################# diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index f3992465c..b303b9b0f 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -29,6 +29,12 @@ class FileMethods(DatabaseMethods): self.c.execute("SELECT COUNT(*) FROM files") return self.c.fetchone()[0] + @queue + def downloadcount(self, user=None): + """ number of downloads """ + self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus != 0") + return self.c.fetchone()[0] + @queue def queuecount(self, user=None): """ number of files in queue not finished yet""" @@ -156,12 +162,14 @@ class FileMethods(DatabaseMethods): return data @queue - def getAllPackages(self, root=None, owner=None): + def getAllPackages(self, root=None, owner=None, tags=None): """ Return dict with package information :param root: optional root to filter + :param owner: optional user id + :param tags: optional tag list """ - qry = ('SELECT pid, name, folder, root, owner, site, comment, password, added, status, packageorder ' + qry = ('SELECT pid, name, folder, root, owner, site, comment, password, added, tags, status, packageorder ' 'FROM packages%s ORDER BY root, packageorder') if root is None: @@ -180,9 +188,8 @@ class FileMethods(DatabaseMethods): data = OrderedDict() for r in self.c: data[r[0]] = PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], None, r[9], r[10], stats.get(r[0], zero_stats) + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9].split(","), r[10], r[11], stats.get(r[0], zero_stats) ) - # TODO: tags return data @@ -242,7 +249,7 @@ class FileMethods(DatabaseMethods): if stats: stats = self.getPackageStats(pid=pid) - self.c.execute('SELECT pid, name, folder, root, owner, site, comment, password, added, status, packageorder ' + self.c.execute('SELECT pid, name, folder, root, owner, site, comment, password, added, tags, status, packageorder ' 'FROM packages WHERE pid=?', (pid,)) r = self.c.fetchone() @@ -250,7 +257,7 @@ class FileMethods(DatabaseMethods): return None else: return PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], None, r[9], r[10], stats.get(r[0], zero_stats) if stats else None + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9].split(","), r[10], r[11], stats.get(r[0], zero_stats) if stats else None ) @async @@ -272,8 +279,8 @@ class FileMethods(DatabaseMethods): @async def updatePackage(self, p): - self.c.execute('UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, status=? WHERE pid=?', - (p.name, p.folder, p.site, p.comment, p.password, p.status, p.pid)) + self.c.execute('UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, tags=?, status=? WHERE pid=?', + (p.name, p.folder, p.site, p.comment, p.password, ",".join(p.tags), p.status, p.pid)) # TODO: most modifying methods needs owner argument to avoid checking beforehand @async diff --git a/module/remote/create_ttypes.py b/module/remote/create_ttypes.py index fc4d75d32..c50142322 100644 --- a/module/remote/create_ttypes.py +++ b/module/remote/create_ttypes.py @@ -83,6 +83,9 @@ def main(): class BaseObject(object): \t__slots__ = [] +\tdef __str__(self): +\t\treturn "<%s %s>" % (self.__class__.__name__, ", ".join("%s=%s" % (k,getattr(self,k)) for k in self.__slots__)) + """) dev = open(join(path, "ttypes_debug.py"), "wb") diff --git a/module/remote/pyload.thrift b/module/remote/pyload.thrift index 50164c292..183fd3af8 100644 --- a/module/remote/pyload.thrift +++ b/module/remote/pyload.thrift @@ -427,7 +427,9 @@ service Pyload { PackageInfo getPackageInfo(1: PackageID pid) throws (1: PackageDoesNotExists e), FileInfo getFileInfo(1: FileID fid) throws (1: FileDoesNotExists e), + TreeCollection findFiles(1: string pattern), + TreeCollection findPackages(1: list tags), /////////////////////// // Modify Downloads @@ -445,9 +447,8 @@ service Pyload { ///////////////////////// // moving package while downloading is not possible, so they will return bool to indicate success - void setPackagePaused(1: PackageID pid, 2: bool paused) throws (1: PackageDoesNotExists e), + void updatePackage(1: PackageInfo pack) throws (1: PackageDoesNotExists e), bool setPackageFolder(1: PackageID pid, 2: string path) throws (1: PackageDoesNotExists e), - void setPackageData(1: PackageID pid, 2: map data) throws (1: PackageDoesNotExists e), // as above, this will move files on disk bool movePackage(1: PackageID pid, 2: PackageID root) throws (1: PackageDoesNotExists e), diff --git a/module/remote/ttypes.py b/module/remote/ttypes.py index 4368479fd..eb990f2e8 100644 --- a/module/remote/ttypes.py +++ b/module/remote/ttypes.py @@ -6,6 +6,9 @@ class BaseObject(object): __slots__ = [] + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, ", ".join("%s=%s" % (k,getattr(self,k)) for k in self.__slots__)) + class DownloadState: All = 0 Finished = 1 @@ -271,14 +274,13 @@ class PackageStats(BaseObject): self.sizedone = sizedone class ProgressInfo(BaseObject): - __slots__ = ['plugin', 'name', 'statusmsg', 'eta', 'format_eta', 'done', 'total', 'download'] + __slots__ = ['plugin', 'name', 'statusmsg', 'eta', 'done', 'total', 'download'] - def __init__(self, plugin=None, name=None, statusmsg=None, eta=None, format_eta=None, done=None, total=None, download=None): + def __init__(self, plugin=None, name=None, statusmsg=None, eta=None, done=None, total=None, download=None): self.plugin = plugin self.name = name self.statusmsg = statusmsg self.eta = eta - self.format_eta = format_eta self.done = done self.total = total self.download = download @@ -384,6 +386,8 @@ class Iface(object): pass def findFiles(self, pattern): pass + def findPackages(self, tags): + pass def freeSpace(self): pass def generateAndAddPackages(self, links, paused): @@ -486,12 +490,8 @@ class Iface(object): pass def setInteractionResult(self, iid, result): pass - def setPackageData(self, pid, data): - pass def setPackageFolder(self, pid, path): pass - def setPackagePaused(self, pid, paused): - pass def setPassword(self, username, old_password, new_password): pass def stop(self): @@ -510,6 +510,8 @@ class Iface(object): pass def updateAccountInfo(self, account): pass + def updatePackage(self, pack): + pass def updateUserData(self, data): pass def uploadContainer(self, filename, data): diff --git a/module/remote/ttypes_debug.py b/module/remote/ttypes_debug.py index 6ca9e892a..807a8451b 100644 --- a/module/remote/ttypes_debug.py +++ b/module/remote/ttypes_debug.py @@ -23,7 +23,7 @@ classes = { 'PackageDoesNotExists' : [int], 'PackageInfo' : [int, basestring, basestring, int, int, basestring, basestring, basestring, int, (list, basestring), int, int, PackageStats, (list, int), (list, int)], 'PackageStats' : [int, int, int, int], - 'ProgressInfo' : [basestring, basestring, basestring, int, basestring, int, int, (None, DownloadProgress)], + 'ProgressInfo' : [basestring, basestring, basestring, int, int, int, (None, DownloadProgress)], 'ServerStatus' : [int, int, int, bool, bool, bool], 'ServiceDoesNotExists' : [basestring, basestring], 'ServiceException' : [basestring], @@ -54,6 +54,7 @@ methods = { 'deleteFiles': None, 'deletePackages': None, 'findFiles': TreeCollection, + 'findPackages': TreeCollection, 'freeSpace': int, 'generateAndAddPackages': (list, int), 'generateDownloadLink': basestring, @@ -105,9 +106,7 @@ methods = { 'saveConfig': None, 'setConfigHandler': None, 'setInteractionResult': None, - 'setPackageData': None, 'setPackageFolder': bool, - 'setPackagePaused': None, 'setPassword': bool, 'stop': None, 'stopAllDownloads': None, @@ -117,6 +116,7 @@ methods = { 'unpauseServer': None, 'updateAccount': None, 'updateAccountInfo': None, + 'updatePackage': None, 'updateUserData': None, 'uploadContainer': int, } diff --git a/module/web/static/js/models/Package.js b/module/web/static/js/models/Package.js index 5a2940c66..64b066dbc 100644 --- a/module/web/static/js/models/Package.js +++ b/module/web/static/js/models/Package.js @@ -15,6 +15,7 @@ define(['jquery', 'backbone', 'underscore', 'collections/FileList', 'require'], comment: "", password: "", added: -1, + tags: null, status: -1, packageorder: -1, stats: null, diff --git a/tests/other/test_database.py b/tests/other/test_database.py deleted file mode 100644 index dd733b4ff..000000000 --- a/tests/other/test_database.py +++ /dev/null @@ -1,192 +0,0 @@ -# -*- coding: utf-8 -*- - -from tests.helper.Stubs import Core -from tests.helper.BenchmarkTest import BenchmarkTest - -from module.Api import DownloadState -from module.database import DatabaseBackend - -# disable asyncronous queries -DatabaseBackend.async = DatabaseBackend.queue - -from random import choice - -class TestDatabase(BenchmarkTest): - bench = ["insert", "insert_links", "insert_many", "get_packages", - "get_files", "get_files_queued", "get_package_childs", "get_package_files", - "get_package_data", "get_file_data", "find_files", "collector", "purge"] - pids = None - fids = None - owner = 123 - pstatus = 0 - - @classmethod - def setUpClass(cls): - cls.pids = [-1] - cls.fids = [] - - cls.db = DatabaseBackend(Core()) - cls.db.manager = cls.db.core - - cls.db.setup() - cls.db.purgeAll() - - @classmethod - def tearDownClass(cls): - cls.db.purgeAll() - cls.db.shutdown() - - # benchmarker ignore setup - def setUp(self): - self.db.purgeAll() - self.pids = [-1] - self.fids = [] - - self.test_insert(20) - self.test_insert_many() - self.fids = self.db.getAllFiles().keys() - - - def test_insert(self, n=200): - for i in range(n): - pid = self.db.addPackage("name", "folder", choice(self.pids), "password", "site", "comment", self.pstatus, - self.owner) - self.pids.append(pid) - - def test_insert_links(self): - for i in range(10000): - fid = self.db.addLink("url %s" % i, "name", "plugin", choice(self.pids), self.owner) - self.fids.append(fid) - - def test_insert_many(self): - for pid in self.pids: - self.db.addLinks([("url %s" % i, "plugin") for i in range(50)], pid, self.owner) - - def test_get_packages(self): - packs = self.db.getAllPackages() - n = len(packs) - assert n == len(self.pids) - 1 - - print "Fetched %d packages" % n - self.assert_pack(choice(packs.values())) - - def test_get_files(self): - files = self.db.getAllFiles() - n = len(files) - assert n >= len(self.pids) - - print "Fetched %d files" % n - self.assert_file(choice(files.values())) - - def test_get_files_queued(self): - files = self.db.getAllFiles(state=DownloadState.Unfinished) - print "Fetched %d files queued" % len(files) - - def test_delete(self): - pid = choice(self.pids) - self.db.deletePackage(pid) - self.pids.remove(pid) - - def test_get_package_childs(self): - pid = choice(self.pids) - packs = self.db.getAllPackages(root=pid) - - print "Package %d has %d packages" % (pid, len(packs)) - self.assert_pack(choice(packs.values())) - - def test_get_package_files(self): - pid = choice(self.pids) - files = self.db.getAllFiles(package=pid) - - print "Package %d has %d files" % (pid, len(files)) - self.assert_file(choice(files.values())) - - def test_get_package_data(self, stats=False): - pid = choice(self.pids) - p = self.db.getPackageInfo(pid, stats) - self.assert_pack(p) - # test again with stat - if not stats: - self.test_get_package_data(True) - - def test_get_file_data(self): - fid = choice(self.fids) - f = self.db.getFileInfo(fid) - self.assert_file(f) - - def test_find_files(self): - files = self.db.getAllFiles(search="1") - print "Found %s files" % len(files) - f = choice(files.values()) - - assert "1" in f.name - - def test_collector(self): - self.db.saveCollector(0, "data") - assert self.db.retrieveCollector(0) == "data" - self.db.deleteCollector(0) - - def test_purge(self): - self.db.purgeLinks() - - - def test_user_context(self): - self.db.purgeAll() - - p1 = self.db.addPackage("name", "folder", 0, "password", "site", "comment", self.pstatus, 0) - self.db.addLink("url", "name", "plugin", p1, 0) - p2 = self.db.addPackage("name", "folder", 0, "password", "site", "comment", self.pstatus, 1) - self.db.addLink("url", "name", "plugin", p2, 1) - - assert len(self.db.getAllPackages(owner=0)) == 1 == len(self.db.getAllFiles(owner=0)) - assert len(self.db.getAllPackages(root=0, owner=0)) == 1 == len(self.db.getAllFiles(package=p1, owner=0)) - assert len(self.db.getAllPackages(owner=1)) == 1 == len(self.db.getAllFiles(owner=1)) - assert len(self.db.getAllPackages(root=0, owner=1)) == 1 == len(self.db.getAllFiles(package=p2, owner=1)) - assert len(self.db.getAllPackages()) == 2 == len(self.db.getAllFiles()) - - self.db.deletePackage(p1, 1) - assert len(self.db.getAllPackages(owner=0)) == 1 == len(self.db.getAllFiles(owner=0)) - self.db.deletePackage(p1, 0) - assert len(self.db.getAllPackages(owner=1)) == 1 == len(self.db.getAllFiles(owner=1)) - self.db.deletePackage(p2) - - assert len(self.db.getAllPackages()) == 0 - - def test_count(self): - self.db.purgeAll() - - assert self.db.filecount() == 0 - assert self.db.queuecount() == 0 - assert self.db.processcount() == 0 - - def assert_file(self, f): - try: - assert f is not None - self.assert_int(f, ("fid", "status", "size", "media", "fileorder", "added", "package", "owner")) - assert f.status in range(5) - assert f.owner == self.owner - assert f.media in range(1024) - assert f.package in self.pids - assert f.added > 10 ** 6 # date is usually big integer - except: - print f - raise - - def assert_pack(self, p): - try: - assert p is not None - self.assert_int(p, ("pid", "root", "added", "status", "packageorder", "owner")) - assert p.pid in self.pids - assert p.owner == self.owner - assert p.status in range(5) - assert p.root in self.pids - assert p.added > 10 ** 6 - except: - print p - raise - - def assert_int(self, obj, list): - for attr in list: assert type(getattr(obj, attr)) == int - -if __name__ == "__main__": - TestDatabase.benchmark() \ No newline at end of file diff --git a/tests/other/test_filedatabase.py b/tests/other/test_filedatabase.py new file mode 100644 index 000000000..2450e5423 --- /dev/null +++ b/tests/other/test_filedatabase.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- + +from tests.helper.Stubs import Core +from tests.helper.BenchmarkTest import BenchmarkTest + +from module.Api import DownloadState, PackageInfo, FileInfo +from module.database import DatabaseBackend + +# disable asyncronous queries +DatabaseBackend.async = DatabaseBackend.queue + +from random import choice + +class TestDatabase(BenchmarkTest): + bench = ["insert", "insert_links", "insert_many", "get_packages", + "get_files", "get_files_queued", "get_package_childs", "get_package_files", + "get_package_data", "get_file_data", "find_files", "collector", "purge"] + pids = None + fids = None + owner = 123 + pstatus = 0 + + @classmethod + def setUpClass(cls): + cls.pids = [-1] + cls.fids = [] + + cls.db = DatabaseBackend(Core()) + cls.db.manager = cls.db.core + + cls.db.setup() + cls.db.purgeAll() + + @classmethod + def tearDownClass(cls): + cls.db.purgeAll() + cls.db.shutdown() + + # benchmarker ignore setup + def setUp(self): + self.db.purgeAll() + self.pids = [-1] + self.fids = [] + + self.test_insert(20) + self.test_insert_many() + self.fids = self.db.getAllFiles().keys() + + + def test_insert(self, n=200): + for i in range(n): + pid = self.db.addPackage("name", "folder", choice(self.pids), "password", "site", "comment", self.pstatus, + self.owner) + self.pids.append(pid) + + def test_insert_links(self): + for i in range(10000): + fid = self.db.addLink("url %s" % i, "name", "plugin", choice(self.pids), self.owner) + self.fids.append(fid) + + def test_insert_many(self): + for pid in self.pids: + self.db.addLinks([("url %s" % i, "plugin") for i in range(50)], pid, self.owner) + + def test_get_packages(self): + packs = self.db.getAllPackages() + n = len(packs) + assert n == len(self.pids) - 1 + + print "Fetched %d packages" % n + self.assert_pack(choice(packs.values())) + + def test_get_files(self): + files = self.db.getAllFiles() + n = len(files) + assert n >= len(self.pids) + + print "Fetched %d files" % n + self.assert_file(choice(files.values())) + + def test_get_files_queued(self): + files = self.db.getAllFiles(state=DownloadState.Unfinished) + print "Fetched %d files queued" % len(files) + + def test_delete(self): + pid = choice(self.pids) + self.db.deletePackage(pid) + self.pids.remove(pid) + + def test_get_package_childs(self): + pid = choice(self.pids) + packs = self.db.getAllPackages(root=pid) + + print "Package %d has %d packages" % (pid, len(packs)) + self.assert_pack(choice(packs.values())) + + def test_get_package_files(self): + pid = choice(self.pids) + files = self.db.getAllFiles(package=pid) + + print "Package %d has %d files" % (pid, len(files)) + self.assert_file(choice(files.values())) + + def test_get_package_data(self, stats=False): + pid = choice(self.pids) + p = self.db.getPackageInfo(pid, stats) + self.assert_pack(p) + # test again with stat + if not stats: + self.test_get_package_data(True) + + def test_get_file_data(self): + fid = choice(self.fids) + f = self.db.getFileInfo(fid) + self.assert_file(f) + + def test_find_files(self): + files = self.db.getAllFiles(search="1") + print "Found %s files" % len(files) + f = choice(files.values()) + + assert "1" in f.name + + def test_collector(self): + self.db.saveCollector(0, "data") + assert self.db.retrieveCollector(0) == "data" + self.db.deleteCollector(0) + + def test_purge(self): + self.db.purgeLinks() + + + def test_user_context(self): + self.db.purgeAll() + + p1 = self.db.addPackage("name", "folder", 0, "password", "site", "comment", self.pstatus, 0) + self.db.addLink("url", "name", "plugin", p1, 0) + p2 = self.db.addPackage("name", "folder", 0, "password", "site", "comment", self.pstatus, 1) + self.db.addLink("url", "name", "plugin", p2, 1) + + assert len(self.db.getAllPackages(owner=0)) == 1 == len(self.db.getAllFiles(owner=0)) + assert len(self.db.getAllPackages(root=0, owner=0)) == 1 == len(self.db.getAllFiles(package=p1, owner=0)) + assert len(self.db.getAllPackages(owner=1)) == 1 == len(self.db.getAllFiles(owner=1)) + assert len(self.db.getAllPackages(root=0, owner=1)) == 1 == len(self.db.getAllFiles(package=p2, owner=1)) + assert len(self.db.getAllPackages()) == 2 == len(self.db.getAllFiles()) + + self.db.deletePackage(p1, 1) + assert len(self.db.getAllPackages(owner=0)) == 1 == len(self.db.getAllFiles(owner=0)) + self.db.deletePackage(p1, 0) + assert len(self.db.getAllPackages(owner=1)) == 1 == len(self.db.getAllFiles(owner=1)) + self.db.deletePackage(p2) + + assert len(self.db.getAllPackages()) == 0 + + def test_count(self): + self.db.purgeAll() + + assert self.db.filecount() == 0 + assert self.db.downloadcount() == 0 + assert self.db.queuecount() == 0 + assert self.db.processcount() == 0 + + def test_update(self): + p1 = self.db.addPackage("name", "folder", 0, "password", "site", "comment", self.pstatus, 0) + pack = self.db.getPackageInfo(p1) + assert isinstance(pack, PackageInfo) + + pack.folder = "new folder" + pack.comment = "lol" + pack.tags.append("video") + + self.db.updatePackage(pack) + + pack = self.db.getPackageInfo(p1) + assert pack.folder == "new folder" + assert pack.comment == "lol" + assert "video" in pack.tags + + def assert_file(self, f): + try: + assert f is not None + assert isinstance(f, FileInfo) + self.assert_int(f, ("fid", "status", "size", "media", "fileorder", "added", "package", "owner")) + assert f.status in range(5) + assert f.owner == self.owner + assert f.media in range(1024) + assert f.package in self.pids + assert f.added > 10 ** 6 # date is usually big integer + except: + print f + raise + + def assert_pack(self, p): + try: + assert p is not None + assert isinstance(p, PackageInfo) + self.assert_int(p, ("pid", "root", "added", "status", "packageorder", "owner")) + assert p.pid in self.pids + assert p.owner == self.owner + assert p.status in range(5) + assert p.root in self.pids + assert p.added > 10 ** 6 + assert isinstance(p.tags, list) + except: + print p + raise + + def assert_int(self, obj, list): + for attr in list: assert type(getattr(obj, attr)) == int + +if __name__ == "__main__": + TestDatabase.benchmark() \ No newline at end of file -- cgit v1.2.3