diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/helper/BenchmarkTest.py | 66 | ||||
| -rw-r--r-- | tests/helper/Stubs.py | 11 | ||||
| -rw-r--r-- | tests/test_database.py | 171 | ||||
| -rw-r--r-- | tests/test_filemanager.py | 214 |
4 files changed, 458 insertions, 4 deletions
diff --git a/tests/helper/BenchmarkTest.py b/tests/helper/BenchmarkTest.py new file mode 100644 index 000000000..d28c52959 --- /dev/null +++ b/tests/helper/BenchmarkTest.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +from time import time + + +class BenchmarkTest: + + bench = [] + results = {} + + @classmethod + def timestamp(cls, name, a): + t = time() + r = cls.results.get(name, []) + r.append((t-a) * 1000) + cls.results[name] = r + + @classmethod + def benchmark(cls, n=1): + + print "Benchmarking %s" % cls.__name__ + print + + for i in range(n): + cls.collect_results() + + if "setUpClass" in cls.results: + cls.bench.insert(0, "setUpClass") + + if "tearDownClass" in cls.results: + cls.bench.append("tearDownClass") + + length = str(max([len(k) for k in cls.bench]) + 1) + total = 0 + + for k in cls.bench: + v = cls.results[k] + + if len(v) > 1: + print ("%" + length +"s: %s | average: %.2f ms") % (k, ", ".join(["%.2f" % x for x in v]), sum(v)/len(v)) + total += sum(v)/len(v) + else: + print ("%" + length +"s: %.2f ms") % (k, v[0]) + total += v[0] + + print "\ntotal: %.2f ms" % total + + + @classmethod + def collect_results(cls): + if hasattr(cls, "setUpClass"): + a = time() + cls.setUpClass() + cls.timestamp("setUpClass", a) + + obj = cls() + + for f in cls.bench: + a = time() + getattr(obj, "test_" + f)() + cls.timestamp(f, a) + + if hasattr(cls, "tearDownClass"): + a = time() + cls.tearDownClass() + cls.timestamp("tearDownClass", a)
\ No newline at end of file diff --git a/tests/helper/Stubs.py b/tests/helper/Stubs.py index ade15f0c4..be2f5052b 100644 --- a/tests/helper/Stubs.py +++ b/tests/helper/Stubs.py @@ -58,8 +58,8 @@ class Core: def __init__(self): self.log = NoLog() - self.api = self - self.core = self + self.api = self.core = self + self.threadManager = self self.debug = True self.captcha = True self.config = ConfigParser() @@ -67,7 +67,7 @@ class Core: self.requestFactory = RequestFactory(self) __builtin__.pyreq = self.requestFactory self.accountManager = AccountManager() - self.hookManager = self.eventManager = self.interActionManager = NoopClass() + self.addonManager = self.eventManager = self.interActionManager = NoopClass() self.js = JsEngine() self.cache = {} self.packageCache = {} @@ -86,6 +86,9 @@ class Core: def updatePackage(self, *args): pass + def processingIds(self, *args): + return [] + def getPackage(self, id): return PyPackage(self, 0, "tmp", "tmp", "", "", 0, 0) @@ -118,5 +121,5 @@ class Thread(BaseThread): __builtin__._ = lambda x: x __builtin__.pypath = abspath(join(dirname(__file__), "..", "..")) -__builtin__.hookManager = NoopClass() +__builtin__.addonManager = NoopClass() __builtin__.pyreq = None diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 000000000..9fe7796a4 --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- + +from collections import defaultdict + +from helper.Stubs import Core +from helper.BenchmarkTest import BenchmarkTest + +from module.database import DatabaseBackend + +# disable asyncronous queries +DatabaseBackend.async = DatabaseBackend.queue + +from random import choice + +class TestDatabase(BenchmarkTest): + bench = ["insert", "insert_links", "insert_many", "get_packages", + "get_files", "get_files_queued", "get_package_childs", "get_package_files", + "get_package_data", "get_file_data", "find_files", "collector", "purge"] + pids = None + fids = None + + @classmethod + def setUpClass(cls): + cls.pids = [-1] + cls.fids = [] + + cls.db = DatabaseBackend(Core()) + cls.db.manager = cls.db.core + cls.db.manager.statusMsg = defaultdict(lambda: "statusmsg") + + cls.db.setup() + cls.db.purgeAll() + + @classmethod + def tearDownClass(cls): + cls.db.purgeAll() + cls.db.shutdown() + + # benchmarker ignore setup + def setUp(self): + self.db.purgeAll() + self.pids = [-1] + self.fids = [] + + self.test_insert(20) + self.test_insert_many() + self.fids = self.db.getAllFiles().keys() + + + def test_insert(self, n=200): + for i in range(n): + pid = self.db.addPackage("name", "folder", choice(self.pids), "password", "site", "comment", 0) + self.pids.append(pid) + + def test_insert_links(self): + for i in range(10000): + fid = self.db.addLink("url %s" % i, "name", "plugin", choice(self.pids)) + self.fids.append(fid) + + def test_insert_many(self): + for pid in self.pids: + self.db.addLinks([("url %s" % i, "plugin") for i in range(50)], pid) + + def test_get_packages(self): + packs = self.db.getAllPackages() + n = len(packs) + assert n == len(self.pids) -1 + + print "Fetched %d packages" % n + self.assert_pack(choice(packs.values())) + + def test_get_files(self): + files = self.db.getAllFiles() + n = len(files) + assert n >= len(self.pids) + + print "Fetched %d files" % n + self.assert_file(choice(files.values())) + + def test_get_files_queued(self): + files = self.db.getAllFiles(unfinished=True) + print "Fetched %d files queued" % len(files) + + def test_delete(self): + pid = choice(self.pids) + self.db.deletePackage(pid) + self.pids.remove(pid) + + def test_get_package_childs(self): + pid = choice(self.pids) + packs = self.db.getAllPackages(root=pid) + + print "Package %d has %d packages" % (pid, len(packs)) + self.assert_pack(choice(packs.values())) + + def test_get_package_files(self): + pid = choice(self.pids) + files = self.db.getAllFiles(package=pid) + + print "Package %d has %d files" % (pid, len(files)) + self.assert_file(choice(files.values())) + + def test_get_package_data(self, stats=False): + pid = choice(self.pids) + p = self.db.getPackageInfo(pid, stats) + self.assert_pack(p) + # test again with stat + if not stats: + self.test_get_package_data(True) + + def test_get_file_data(self): + fid = choice(self.fids) + f = self.db.getFileInfo(fid) + self.assert_file(f) + + def test_find_files(self): + files = self.db.getAllFiles(search="1") + print "Found %s files" % len(files) + f = choice(files.values()) + + assert "1" in f.name + + def test_collector(self): + self.db.deleteCollector() + assert not self.db.getCollector() + + self.db.addCollector("plugin", "package", [("name", 0, 0, "url %d" % i) for i in range(10)]) + coll = self.db.getCollector() + assert len(coll) == 10 + assert coll[0].plugin == "plugin" + assert coll[0].packagename == "package" + assert coll[0].name == "name" + assert "url" in coll[0].url + + self.db.deleteCollector(url="url 1") + assert len(self.db.getCollector()) == 9 + self.db.deleteCollector(package="package") + assert not self.db.getCollector() + + def test_purge(self): + self.db.purgeLinks() + + def assert_file(self, f): + try: + assert f is not None + self.assert_int(f, ("fid", "status", "size", "media", "fileorder", "added", "package")) + assert f.status in range(5) + assert f.media in range(1024) + assert f.package in self.pids + assert f.added > 10 ** 6 # date is usually big integer + except: + print f + raise + + def assert_pack(self, p): + try: + assert p is not None + self.assert_int(p, ("pid", "root", "added", "status", "packageorder")) + assert p.pid in self.pids + assert p.status in range(5) + assert p.root in self.pids + assert p.added > 10 ** 6 + except: + print p + raise + + def assert_int(self, obj, list): + for attr in list: assert type(getattr(obj, attr)) == int + +if __name__ == "__main__": + TestDatabase.benchmark()
\ No newline at end of file diff --git a/tests/test_filemanager.py b/tests/test_filemanager.py new file mode 100644 index 000000000..a9c6d3375 --- /dev/null +++ b/tests/test_filemanager.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- + +from random import choice + +from helper.Stubs import Core +from helper.BenchmarkTest import BenchmarkTest + +from module.database import DatabaseBackend +# disable asyncronous queries +DatabaseBackend.async = DatabaseBackend.queue + +from module.FileManager import FileManager + +class TestFileManager(BenchmarkTest): + bench = ["add_packages", "add_files", "get_files_root", "get", + "get_package_content", "get_package_tree", + "order_package", "order_files", "move"] + + pids = [-1] + count = 100 + + @classmethod + def setUpClass(cls): + c = Core() + # db needs seperate initialisation + cls.db = c.db = DatabaseBackend(c) + cls.db.setup() + cls.db.purgeAll() + + cls.m = cls.db.manager = FileManager(c) + + @classmethod + def tearDownClass(cls): + cls.db.purgeAll() + cls.db.shutdown() + + + # benchmarker ignore setup + def setUp(self): + self.db.purgeAll() + self.pids = [-1] + + self.count = 20 + self.test_add_packages() + self.test_add_files() + + def test_add_packages(self): + for i in range(100): + pid = self.m.addPackage("name", "folder", choice(self.pids), "", "", "", False) + self.pids.append(pid) + + if -1 in self.pids: + self.pids.remove(-1) + + def test_add_files(self): + for pid in self.pids: + self.m.addLinks([("plugin %d" % i, "url %s" % i) for i in range(self.count)], pid) + + count = self.m.getQueueCount() + files = self.count * len(self.pids) + # in test runner files get added twice + assert count == files or count == files * 2 + + def test_get(self): + info = self.m.getPackageInfo(choice(self.pids)) + assert info.stats.linkstotal == self.count + + fid = choice(info.fids) + f = self.m.getFile(fid) + assert f.fid in self.m.files + + f.name = "new name" + f.sync() + finfo = self.m.getFileInfo(fid) + assert finfo is not None + assert finfo.name == "new name" + + p = self.m.getPackage(choice(self.pids)) + assert p is not None + assert p.pid in self.m.packages + p.sync() + + p.delete() + + self.m.getView(-1, True, False) + + + def test_get_files_root(self): + view = self.m.getView(-1, True, False) + + for pid in self.pids: + assert pid in view.packages + + assert len(view.packages) == len(self.pids) + + p = choice(view.packages.values()) + assert len(p.fids) == self.count + assert p.stats.linkstotal == self.count + + + def test_get_package_content(self): + view = self.m.getView(choice(self.pids), False, False) + p = view.root + + assert len(view.packages) == len(p.pids) + for pid in p.pids: assert pid in view.packages + + def test_get_package_tree(self): + view = self.m.getView(choice(self.pids), True, False) + for pid in view.root.pids: assert pid in view.packages + for fid in view.root.fids: assert fid in view.files + + def test_delete(self): + self.m.deleteFile(self.count * 5) + self.m.deletePackage(choice(self.pids)) + + def test_order_package(self): + parent = self.m.addPackage("order", "", -1, "", "", "", False) + self.m.addLinks([("url", "plugin") for i in range(100)], parent) + + pids = [self.m.addPackage("c", "", parent, "", "", "", False) for i in range(5)] + v = self.m.getView(parent, False, False) + self.assert_ordered(pids, 0, 5, v.root.pids, v.packages, True) + + pid = v.packages.keys()[0] + self.assert_pack_ordered(parent, pid, 3) + self.assert_pack_ordered(parent, pid, 0) + self.assert_pack_ordered(parent, pid, 0) + self.assert_pack_ordered(parent, pid, 4) + pid = v.packages.keys()[2] + self.assert_pack_ordered(parent, pid, 4) + self.assert_pack_ordered(parent, pid, 3) + self.assert_pack_ordered(parent, pid, 2) + + + def test_order_files(self): + parent = self.m.addPackage("order", "", -1, "", "", "", False) + self.m.addLinks([("url", "plugin") for i in range(100)], parent) + v = self.m.getView(parent, False, False) + + fids = v.root.fids[10:20] + v = self.assert_files_ordered(parent, fids, 0) + + fids = v.root.fids[20:30] + + self.m.orderFiles(fids, parent, 99) + v = self.m.getView(parent, False, False) + assert fids[-1] == v.root.fids[-1] + assert fids[0] == v.root.fids[90] + self.assert_ordered(fids, 90, 100, v.root.fids, v.files) + + fids = v.root.fids[80:] + v = self.assert_files_ordered(parent, fids, 20) + + self.m.orderFiles(fids, parent, 80) + v = self.m.getView(parent, False, False) + self.assert_ordered(fids, 61, 81, v.root.fids, v.files) + + fids = v.root.fids[50:51] + self.m.orderFiles(fids, parent, 99) + v = self.m.getView(parent, False, False) + self.assert_ordered(fids, 99, 100, v.root.fids, v.files) + + fids = v.root.fids[50:51] + v = self.assert_files_ordered(parent, fids, 0) + + + def assert_files_ordered(self, parent, fids, pos): + fs = [self.m.getFile(choice(fids)) for i in range(5)] + self.m.orderFiles(fids, parent, pos) + v = self.m.getView(parent, False, False) + self.assert_ordered(fids, pos, pos+len(fids), v.root.fids, v.files) + + return v + + def assert_pack_ordered(self, parent, pid, pos): + self.m.orderPackage(pid, pos) + v = self.m.getView(parent, False, False) + self.assert_ordered([pid], pos, pos+1, v.root.pids, v.packages, True) + + # assert that ordering is total, complete with no gaps + def assert_ordered(self, part, start, end, data, dict, pack=False): + assert data[start:end] == part + if pack: + assert sorted([p.packageorder for p in dict.values()]) == range(len(dict)) + assert [dict[pid].packageorder for pid in part] == range(start, end) + else: + assert sorted([f.fileorder for f in dict.values()]) == range(len(dict)) + assert [dict[fid].fileorder for fid in part] == range(start, end) + + + def test_move(self): + + pid = self.pids[-1] + pid2 = self.pids[1] + + self.m.movePackage(pid, -1) + v = self.m.getView(-1, False, False) + + assert v.root.pids[-1] == pid + assert sorted([p.packageorder for p in v.packages.values()]) == range(len(v.packages)) + + v = self.m.getView(pid, False, False) + fids = v.root.fids[10:20] + self.m.moveFiles(fids, pid2) + v = self.m.getView(pid2, False, False) + + assert sorted([f.fileorder for f in v.files.values()]) == range(len(v.files)) + assert len(v.files) == self.count + len(fids) + + + +if __name__ == "__main__": + TestFileManager.benchmark()
\ No newline at end of file |