diff options
| -rw-r--r-- | module/Api.py | 29 | ||||
| -rw-r--r-- | module/FileManager.py | 4 | ||||
| -rw-r--r-- | module/database/FileDatabase.py | 10 | ||||
| -rw-r--r-- | module/remote/create_ttypes.py | 2 | ||||
| -rw-r--r-- | module/remote/pyload.thrift | 8 | ||||
| -rw-r--r-- | module/remote/ttypes.py | 67 | ||||
| -rw-r--r-- | pavement.py | 4 | ||||
| -rw-r--r-- | tests/test_database.py | 5 | ||||
| -rw-r--r-- | tests/test_filemanager.py | 27 |
9 files changed, 91 insertions, 65 deletions
diff --git a/module/Api.py b/module/Api.py index cf28fda5f..d8e7c537e 100644 --- a/module/Api.py +++ b/module/Api.py @@ -25,7 +25,6 @@ from dis import opmap from remote.ttypes import * -from datatypes.PyFile import PyFile from utils import compare_time, to_string, bits_set, get_index from utils.fs import free_space from common.packagetools import parseNames @@ -88,6 +87,19 @@ class UserContext(object): urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&]*)", re.IGNORECASE) +stateMap = { + DownloadState.All: frozenset(getattr(DownloadStatus, x) for x in dir(DownloadStatus) if not x.startswith("_")), + DownloadState.Finished : frozenset((DownloadStatus.Finished, DownloadStatus.Skipped)), + DownloadState.Unfinished : None, + DownloadState.Failed : frozenset((DownloadStatus.Failed, DownloadStatus.TempOffline, DownloadStatus.Aborted)), + DownloadState.Unmanaged: None, #TODO +} + +stateMap[DownloadState.Unfinished] = frozenset(stateMap[DownloadState.All].difference(stateMap[DownloadState.Finished])) + +def state_string(state): + return ",".join(str(x) for x in stateMap[state]) + def has_permission(userPermission, Permission): return bits_set(Permission, userPermission) @@ -618,9 +630,9 @@ class Api(Iface): return self.getFileTree(-1, True) @RequirePerm(Permission.All) - def getAllUnfinishedFiles(self): - """ same as `getUnfinishedFileTree for toplevel root and full tree""" - return self.getUnfinishedFileTree(-1, True) + def getFilteredFiles(self, state): + """ same as `getFilteredFileTree` for toplevel root and full tree""" + return self.getFilteredFileTree(-1, state, True) @RequirePerm(Permission.All) def getFileTree(self, pid, full): @@ -631,17 +643,18 @@ class Api(Iface): :param full: go down the complete tree or only the first layer :return: :class:`TreeCollection` """ - return self.core.files.getTree(pid, full, False) + return self.core.files.getTree(pid, full, DownloadState.All) @RequirePerm(Permission.All) - def getUnfinishedFileTree(self, pid, full): - """ Same as `getFileTree` but only contains unfinished files. + def getFilteredFileTree(self, pid, full, state): + """ Same as `getFileTree` but only contains files with specific download state. :param pid: package id :param full: go down the complete tree or only the first layer + :param state: :class:`DownloadState`, the attributes used for filtering :return: :class:`TreeCollection` """ - return self.core.files.getTree(pid, full, False) + return self.core.files.getTree(pid, full, state) @RequirePerm(Permission.All) def getPackageContent(self, pid): diff --git a/module/FileManager.py b/module/FileManager.py index 60fa9dbb3..cf4989f99 100644 --- a/module/FileManager.py +++ b/module/FileManager.py @@ -175,7 +175,7 @@ class FileManager: return self.db.getFileInfo(fid) @lock - def getTree(self, pid, full, unfinished): + def getTree(self, pid, full, state): """ return a TreeCollection and fill the info data of containing packages. optional filter only unfnished files """ @@ -185,7 +185,7 @@ class FileManager: root = pid if not full else None packs = self.db.getAllPackages(root) - files = self.db.getAllFiles(package=root, unfinished=unfinished) + files = self.db.getAllFiles(package=root, state=state) # updating from cache for fid, f in self.files.iteritems(): diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index e9ef99cbe..f3992465c 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -17,7 +17,7 @@ from new_collections import OrderedDict -from module.Api import DownloadInfo, FileInfo, PackageInfo, PackageStats +from module.Api import DownloadInfo, FileInfo, PackageInfo, PackageStats, DownloadState as DS, state_string from module.database import DatabaseMethods, queue, async, inner zero_stats = PackageStats(0, 0, 0, 0) @@ -37,7 +37,7 @@ class FileMethods(DatabaseMethods): return self.c.fetchone()[0] @queue - def processcount(self, fid, user=None): + def processcount(self, fid=-1, user=None): """ number of files which have to be processed """ # status in online, queued, starting, waiting, downloading self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (fid, )) @@ -112,7 +112,7 @@ class FileMethods(DatabaseMethods): self.c.execute('DELETE FROM collector WHERE owner=?', (owner,)) @queue - def getAllFiles(self, package=None, search=None, unfinished=False, owner=None): + def getAllFiles(self, package=None, search=None, state=None, owner=None): """ Return dict with file information :param package: optional package to filter out @@ -125,8 +125,8 @@ class FileMethods(DatabaseMethods): arg = [] - if unfinished: - qry += 'dlstatus NOT IN (0, 5, 6) AND ' + if state is not None and state != DS.All: + qry += 'dlstatus IN (%s) AND ' % state_string(state) if owner is not None: qry += 'owner=? AND ' arg.append(owner) diff --git a/module/remote/create_ttypes.py b/module/remote/create_ttypes.py index ec4f07a7d..a9a93bde7 100644 --- a/module/remote/create_ttypes.py +++ b/module/remote/create_ttypes.py @@ -52,7 +52,7 @@ class BaseObject(object): name = enum.__name__ f.write("class %s:\n" % name) - for attr in dir(enum): + for attr in sorted(dir(enum), key=lambda x: getattr(enum, x)): if attr.startswith("_") or attr in ("read", "write"): continue f.write("\t%s = %s\n" % (attr, getattr(enum, attr))) diff --git a/module/remote/pyload.thrift b/module/remote/pyload.thrift index 3b0f74cbc..ea2aa7347 100644 --- a/module/remote/pyload.thrift +++ b/module/remote/pyload.thrift @@ -34,7 +34,7 @@ enum DownloadStatus { // Download states, combination of several downloadstatuses // defined in Filedatabase -enum DownloadStates { +enum DownloadState { All, Finished, Unfinished, @@ -406,15 +406,15 @@ service Pyload { void deleteCollLink(1: string url), //////////////////////////// - // File Information retrival + // File Information retrieval //////////////////////////// TreeCollection getAllFiles(), - TreeCollection getAllUnfinishedFiles(), + TreeCollection getFilteredFiles(1: DownloadState state), // pid -1 for root, full=False only delivers first level in tree TreeCollection getFileTree(1: PackageID pid, 2: bool full), - TreeCollection getUnfinishedFileTree(1: PackageID pid, 2: bool full), + TreeCollection getFilteredFileTree(1: PackageID pid, 2: bool full, 3: DownloadState state), // same as above with full=False TreeCollection getPackageContent(1: PackageID pid), diff --git a/module/remote/ttypes.py b/module/remote/ttypes.py index 70105275d..ecff45170 100644 --- a/module/remote/ttypes.py +++ b/module/remote/ttypes.py @@ -6,77 +6,78 @@ class BaseObject(object): __slots__ = [] -class DownloadStates: - Failed = 2 - Finished = 0 - Unfinished = 1 - Unmanaged = 3 +class DownloadState: + All = 0 + Finished = 1 + Unfinished = 2 + Failed = 3 + Unmanaged = 4 class DownloadStatus: - Aborted = 12 - Custom = 15 - Decrypting = 13 - Downloading = 10 - Failed = 7 - Finished = 5 NA = 0 Offline = 1 Online = 2 - Paused = 4 - Processing = 14 Queued = 3 + Paused = 4 + Finished = 5 Skipped = 6 + Failed = 7 Starting = 8 + Waiting = 9 + Downloading = 10 TempOffline = 11 + Aborted = 12 + Decrypting = 13 + Processing = 14 + Custom = 15 Unknown = 16 - Waiting = 9 class FileStatus: - Missing = 1 Ok = 0 + Missing = 1 Remote = 2 class Input: - Bool = 4 - Click = 5 - List = 8 - Multiple = 7 NA = 0 + Text = 1 + Textbox = 2 Password = 3 + Bool = 4 + Click = 5 Select = 6 + Multiple = 7 + List = 8 Table = 9 - Text = 1 - Textbox = 2 class MediaType: All = 0 - Archive = 32 + Other = 1 Audio = 2 - Document = 16 Image = 4 - Other = 1 Video = 8 + Document = 16 + Archive = 32 class Output: All = 0 - Captcha = 2 Notification = 1 + Captcha = 2 Query = 4 class PackageStatus: - Folder = 2 Ok = 0 Paused = 1 + Folder = 2 Remote = 3 class Permission: - Accounts = 16 - Add = 1 All = 0 + Add = 1 Delete = 2 + Modify = 4 Download = 8 + Accounts = 16 Interaction = 32 - Modify = 4 Plugins = 64 class Role: @@ -393,8 +394,6 @@ class Iface(object): pass def getAllInfo(self): pass - def getAllUnfinishedFiles(self): - pass def getAllUserData(self): pass def getCollector(self): @@ -407,6 +406,10 @@ class Iface(object): pass def getFileTree(self, pid, full): pass + def getFilteredFileTree(self, pid, full, state): + pass + def getFilteredFiles(self, state): + pass def getGlobalPlugins(self): pass def getInfoByPlugin(self, plugin): @@ -425,8 +428,6 @@ class Iface(object): pass def getServerVersion(self): pass - def getUnfinishedFileTree(self, pid, full): - pass def getUserData(self): pass def getUserPlugins(self): diff --git a/pavement.py b/pavement.py index 6ed9b8689..6c4925acf 100644 --- a/pavement.py +++ b/pavement.py @@ -182,7 +182,9 @@ def ttypes(options): outdir = PROJECT_DIR / "module" / "remote" - (outdir / "gen-py").rmtree() + + if (outdir / "gen-py").exists(): + (outdir / "gen-py").rmtree() cmd = [options.ttypes.path, "-strict", "-o", outdir, "--gen", "py:slots,dynamic", outdir / "pyload.thrift"] diff --git a/tests/test_database.py b/tests/test_database.py index b7408e213..fb134ff41 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -5,6 +5,7 @@ from collections import defaultdict from helper.Stubs import Core from helper.BenchmarkTest import BenchmarkTest +from module.Api import DownloadState from module.database import DatabaseBackend # disable asyncronous queries @@ -81,7 +82,7 @@ class TestDatabase(BenchmarkTest): self.assert_file(choice(files.values())) def test_get_files_queued(self): - files = self.db.getAllFiles(unfinished=True) + files = self.db.getAllFiles(state=DownloadState.Unfinished) print "Fetched %d files queued" % len(files) def test_delete(self): @@ -159,7 +160,7 @@ class TestDatabase(BenchmarkTest): assert self.db.filecount() == 0 assert self.db.queuecount() == 0 - assert self.db.proccesscount() == 0 + assert self.db.processcount() == 0 def assert_file(self, f): try: diff --git a/tests/test_filemanager.py b/tests/test_filemanager.py index f5bdd9df3..7b82840b1 100644 --- a/tests/test_filemanager.py +++ b/tests/test_filemanager.py @@ -9,8 +9,10 @@ from module.database import DatabaseBackend # disable asyncronous queries DatabaseBackend.async = DatabaseBackend.queue +from module.Api import DownloadState from module.FileManager import FileManager + class TestFileManager(BenchmarkTest): bench = ["add_packages", "add_files", "get_files_root", "get", "get_package_content", "get_package_tree", @@ -82,11 +84,18 @@ class TestFileManager(BenchmarkTest): p.delete() - self.m.getTree(-1, True, False) + self.m.getTree(-1, True, None) + + def test_get_filtered(self): + all = self.m.getTree(-1, True, None) + finished = self.m.getTree(-1, True, DownloadState.Finished) + unfinished = self.m.getTree(-1, True, DownloadState.Unfinished) + + assert len(finished.files) + len(unfinished.files) == len(all.files) == self.m.getFileCount() def test_get_files_root(self): - view = self.m.getTree(-1, True, False) + view = self.m.getTree(-1, True, None) for pid in self.pids: assert pid in view.packages @@ -99,14 +108,14 @@ class TestFileManager(BenchmarkTest): def test_get_package_content(self): - view = self.m.getTree(choice(self.pids), False, False) + view = self.m.getTree(choice(self.pids), False, None) p = view.root assert len(view.packages) == len(p.pids) for pid in p.pids: assert pid in view.packages def test_get_package_tree(self): - view = self.m.getTree(choice(self.pids), True, False) + view = self.m.getTree(choice(self.pids), True, None) for pid in view.root.pids: assert pid in view.packages for fid in view.root.fids: assert fid in view.files @@ -119,7 +128,7 @@ class TestFileManager(BenchmarkTest): self.m.addLinks([("url", "plugin") for i in range(100)], parent) pids = [self.m.addPackage("c", "", parent, "", "", "", False) for i in range(5)] - v = self.m.getTree(parent, False, False) + v = self.m.getTree(parent, False, None) self.assert_ordered(pids, 0, 5, v.root.pids, v.packages, True) pid = v.packages.keys()[0] @@ -136,7 +145,7 @@ class TestFileManager(BenchmarkTest): def test_order_files(self): parent = self.m.addPackage("order", "", -1, "", "", "", False) self.m.addLinks([("url", "plugin") for i in range(100)], parent) - v = self.m.getTree(parent, False, False) + v = self.m.getTree(parent, False, None) fids = v.root.fids[10:20] v = self.assert_files_ordered(parent, fids, 0) @@ -144,7 +153,7 @@ class TestFileManager(BenchmarkTest): fids = v.root.fids[20:30] self.m.orderFiles(fids, parent, 99) - v = self.m.getTree(parent, False, False) + v = self.m.getTree(parent, False, None) assert fids[-1] == v.root.fids[-1] assert fids[0] == v.root.fids[90] self.assert_ordered(fids, 90, 100, v.root.fids, v.files) @@ -153,12 +162,12 @@ class TestFileManager(BenchmarkTest): v = self.assert_files_ordered(parent, fids, 20) self.m.orderFiles(fids, parent, 80) - v = self.m.getTree(parent, False, False) + v = self.m.getTree(parent, False, None) self.assert_ordered(fids, 61, 81, v.root.fids, v.files) fids = v.root.fids[50:51] self.m.orderFiles(fids, parent, 99) - v = self.m.getTree(parent, False, False) + v = self.m.getTree(parent, False, None) self.assert_ordered(fids, 99, 100, v.root.fids, v.files) fids = v.root.fids[50:51] |