From b4f11103dc031df77117ee4cbb08f8e87305d20a Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 5 Jun 2011 21:26:30 +0200 Subject: api tester --- module/JsEngine.py | 146 -------------------------- module/common/APIExerciser.py | 197 ++++++++++++++++++++++++++++++++++++ module/common/JsEngine.py | 146 ++++++++++++++++++++++++++ module/common/__init__.py | 2 + module/plugins/Plugin.py | 2 +- module/plugins/hoster/BasePlugin.py | 5 + module/plugins/hoster/YoutubeCom.py | 31 +++--- module/web/webinterface.py | 2 +- 8 files changed, 367 insertions(+), 164 deletions(-) delete mode 100644 module/JsEngine.py create mode 100644 module/common/APIExerciser.py create mode 100644 module/common/JsEngine.py create mode 100644 module/common/__init__.py (limited to 'module') diff --git a/module/JsEngine.py b/module/JsEngine.py deleted file mode 100644 index 0b451c9dd..000000000 --- a/module/JsEngine.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from imp import find_module -from os.path import join, exists - -ENGINE = "" - -try: - find_module("spidermonkey") - ENGINE = "spidermonkey" -except: - pass - -if not ENGINE: - try: - find_module("PyV8") - ENGINE = "pyv8" - except: - pass - -if not ENGINE: - try: - import subprocess - - subprocess.Popen(["js", "-v"], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - p = subprocess.Popen(["js", "-e", "print(23+19)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - #integrity check - if out.strip() == "42": - ENGINE = "js" - except: - pass - - -if not ENGINE or ENGINE: - try: - path = "" #path where to find rhino - - if exists("/usr/share/java/js.jar"): - path = "/usr/share/java/js.jar" - elif exists("js.jar"): - path = "js.jar" - elif exists(join(pypath, "js.jar")): #may raises an exception, but js.jar wasnt found anyway - path = join(pypath, "js.jar") - - if not path: - raise Exception - - import subprocess - - p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", "print(23+19)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - #integrity check - if out.strip() == "42": - ENGINE = "rhino" - except: - pass - -class JsEngine(): - def __init__(self): - self.engine = ENGINE - self.init = False - - def __nonzero__(self): - return False if not ENGINE else True - - def eval(self, script): - if not self.init: - if ENGINE == "spidermonkey": - import spidermonkey - global spidermonkey - - elif ENGINE == "pyv8": - import PyV8 - global PyV8 - - self.init = True - - if not ENGINE: - raise Exception("No JS Engine") - elif ENGINE == "spidermonkey": - return self.eval_spidermonkey(script) - elif ENGINE == "pyv8": - return self.eval_pyv8(script) - elif ENGINE == "js": - return self.eval_js(script) - elif ENGINE == "rhino": - return self.eval_rhino(script) - - - def eval_spidermonkey(self, script): - rt = spidermonkey.Runtime() - cx = rt.new_context() - return cx.execute(script) - - def eval_pyv8(self, script): - rt = PyV8.JSContext() - rt.enter() - return rt.eval(script) - - def eval_js(self, script): - script = "print(eval('%s'))" % script.replace("'", '"') - p = subprocess.Popen(["js", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) - out, err = p.communicate() - res = out.strip() - return res - - def eval_rhino(self, script): - script = "print(eval('%s'))" % script.replace("'", '"') - p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) - out, err = p.communicate() - res = out.strip() - return res - - def error(self): - return _("No js engine detected, please install either Spidermonkey, ossp-js, pyv8 or rhino") - -if __name__ == "__main__": - js = JsEngine() - import subprocess - import spidermonkey - #import PyV8 - - test = '"a"+"b"' - - print js.eval_js(test) - print js.eval_spidermonkey(test) - print js.eval_rhino(test) - print js.eval_pyv8(test) \ No newline at end of file diff --git a/module/common/APIExerciser.py b/module/common/APIExerciser.py new file mode 100644 index 000000000..f645f12df --- /dev/null +++ b/module/common/APIExerciser.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- + +import string +from threading import Thread +from random import choice, random, sample, randint +from time import time, sleep + +from traceback import print_exc, format_exc + +from module.remote.thriftbackend.ThriftClient import ThriftClient + +def createURLs(): + """ create some urls, some may fail """ + urls = [] + for x in range(0, randint(20, 100)): + name = "DEBUG_API" + if randint(0, 5) == 5: + name = "" #this link will fail + + urls.append(name + "".join(sample(string.ascii_letters, randint(10, 20)))) + + return urls + +AVOID = (0,3,8) + +class APIExerciser(Thread): + """ tests api randomly """ + + def __init__(self, core): + Thread.__init__(self) + self.setDaemon(True) + self.core = core + self.methods = core.server_methods + self.count = 0 #number of methods + self.time = time() + + self.start() + + def run(self): + out = open("error.log", "ab") + #core errors are not logged of course + out.write("\n" + "Starting\n") + out.flush() + + while True: + try: + self.testAPI() + except Exception: + print_exc() + out.write(format_exc() + 2 * "\n") + out.flush() + + if not self.count % 100: + print "Tested %s api calls" % self.count + if not self.count % 1000: + out.write("Tested %s api calls\n" % self.count) + out.flush() + + + #sleep(random() / 500) + + def testAPI(self): + m = ["status_downloads", "status_server", "add_package", "get_package_data", "get_file_data", "del_links", + "del_packages", + "get_queue", "get_collector", "get_queue_info", "get_collector_info", "is_captcha_waiting"] + + method = choice(m) + #print "Testing:", method + + if hasattr(self, method): + res = getattr(self, method)() + else: + res = getattr(self.methods, method)() + + self.count += 1 + + #print res + + def add_package(self): + name = "".join(sample(string.ascii_letters, 10)) + urls = createURLs() + + self.methods.add_package(name, urls, 1) + + + def del_links(self): + info = self.methods.get_queue() + if not info: return + + pid = choice(info.keys()) + pack = info[pid] + links = pack["links"] + #filter links which are not queued, finished or failed + fids = filter(lambda x: links[x]["status"] not in AVOID, links.keys()) + + if len(fids): + fids = sample(fids, randint(1, max(len(fids) / 2, 1))) + self.methods.del_links(fids) + + + def del_packages(self): + info = self.methods.get_queue_info() + if not info: return + + pids = info.keys() + if len(pids): + pids = sample(pids, randint(1, max(len(pids) / 2, 1))) + filtered = [] + + for p in pids: + info = self.methods.get_package_data(p) + append = True + for link in info["links"].itervalues(): + if link["status"] not in AVOID: + append = False + break + + if append: filtered.append(p) + + self.methods.del_packages(filtered) + + def get_file_data(self): + info = self.methods.get_queue() + if info: + p = info[choice(info.keys())] + if p["links"]: + self.methods.get_file_data(choice(p["links"].keys())) + + def get_package_data(self): + info = self.methods.get_queue_info() + if info: + self.methods.get_package_data(choice(info.keys())) + + +class ThriftExerciser(APIExerciser): + def __init__(self, core): + self.thrift = ThriftClient() + self.thrift.login("user", "pw") + + APIExerciser.__init__(self, core) + + def testAPI(self): + m = ["statusDownloads", "statusServer", "addPackage", "getPackageData", "getFileData", "deleteFiles", + "deletePackages", + "getQueue", "getCollector", "getQueueData", "getCollectorData", "isCaptchaWaiting"] + + method = choice(m) + #print "Testing:", method + + if hasattr(self, method): + res = getattr(self, method)() + else: + res = getattr(self.thrift, method)() + + self.count += 1 + + #print res + + def addPackage(self): + name = "".join(sample(string.ascii_letters, 10)) + urls = createURLs() + + self.thrift.addPackage(name, urls, 0) + + + def deleteFiles(self): + info = self.thrift.getQueueData() + if not info: return + + pack = choice(info) + fids = pack.links + + if len(fids): + fids = [f.fid for f in sample(fids, randint(1, max(len(fids) / 2, 1)))] + self.thrift.deleteFiles(fids) + + + def deletePackages(self): + info = self.thrift.getQueue() + if not info: return + + pids = [p.pid for p in info] + if len(pids): + pids = sample(pids, randint(1, max(len(pids) / 2, 1))) + self.thrift.deletePackages(pids) + + def getFileData(self): + info = self.thrift.getQueueData() + if info: + p = choice(info) + if p.links: + self.thrift.getFileData(choice(p.links).fid) + + def getPackageData(self): + info = self.thrift.getQueue() + if info: + self.thrift.getPackageData(choice(info).pid) \ No newline at end of file diff --git a/module/common/JsEngine.py b/module/common/JsEngine.py new file mode 100644 index 000000000..0b451c9dd --- /dev/null +++ b/module/common/JsEngine.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from imp import find_module +from os.path import join, exists + +ENGINE = "" + +try: + find_module("spidermonkey") + ENGINE = "spidermonkey" +except: + pass + +if not ENGINE: + try: + find_module("PyV8") + ENGINE = "pyv8" + except: + pass + +if not ENGINE: + try: + import subprocess + + subprocess.Popen(["js", "-v"], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + p = subprocess.Popen(["js", "-e", "print(23+19)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + #integrity check + if out.strip() == "42": + ENGINE = "js" + except: + pass + + +if not ENGINE or ENGINE: + try: + path = "" #path where to find rhino + + if exists("/usr/share/java/js.jar"): + path = "/usr/share/java/js.jar" + elif exists("js.jar"): + path = "js.jar" + elif exists(join(pypath, "js.jar")): #may raises an exception, but js.jar wasnt found anyway + path = join(pypath, "js.jar") + + if not path: + raise Exception + + import subprocess + + p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", "print(23+19)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + #integrity check + if out.strip() == "42": + ENGINE = "rhino" + except: + pass + +class JsEngine(): + def __init__(self): + self.engine = ENGINE + self.init = False + + def __nonzero__(self): + return False if not ENGINE else True + + def eval(self, script): + if not self.init: + if ENGINE == "spidermonkey": + import spidermonkey + global spidermonkey + + elif ENGINE == "pyv8": + import PyV8 + global PyV8 + + self.init = True + + if not ENGINE: + raise Exception("No JS Engine") + elif ENGINE == "spidermonkey": + return self.eval_spidermonkey(script) + elif ENGINE == "pyv8": + return self.eval_pyv8(script) + elif ENGINE == "js": + return self.eval_js(script) + elif ENGINE == "rhino": + return self.eval_rhino(script) + + + def eval_spidermonkey(self, script): + rt = spidermonkey.Runtime() + cx = rt.new_context() + return cx.execute(script) + + def eval_pyv8(self, script): + rt = PyV8.JSContext() + rt.enter() + return rt.eval(script) + + def eval_js(self, script): + script = "print(eval('%s'))" % script.replace("'", '"') + p = subprocess.Popen(["js", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) + out, err = p.communicate() + res = out.strip() + return res + + def eval_rhino(self, script): + script = "print(eval('%s'))" % script.replace("'", '"') + p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) + out, err = p.communicate() + res = out.strip() + return res + + def error(self): + return _("No js engine detected, please install either Spidermonkey, ossp-js, pyv8 or rhino") + +if __name__ == "__main__": + js = JsEngine() + import subprocess + import spidermonkey + #import PyV8 + + test = '"a"+"b"' + + print js.eval_js(test) + print js.eval_spidermonkey(test) + print js.eval_rhino(test) + print js.eval_pyv8(test) \ No newline at end of file diff --git a/module/common/__init__.py b/module/common/__init__.py new file mode 100644 index 000000000..de6d13128 --- /dev/null +++ b/module/common/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'christian' + \ No newline at end of file diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 449b60a59..396069adb 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -37,7 +37,7 @@ if os.name != "nt": from itertools import islice -from module.utils import save_join, decode, removeChars +from module.utils import save_join, removeChars def chunks(iterable, size): it = iter(iterable) diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 7b3f56c4d..49bab50f1 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -20,6 +20,11 @@ class BasePlugin(Hoster): def process(self, pyfile): """main function""" + #debug part, for api exerciser + if pyfile.url.startswith("DEBUG_API"): + self.multiDL = False + return + # self.__name__ = "NetloadIn" # pyfile.name = "test" # self.html = self.load("http://localhost:9000/short") diff --git a/module/plugins/hoster/YoutubeCom.py b/module/plugins/hoster/YoutubeCom.py index ccd98606e..0e0fc2d9e 100644 --- a/module/plugins/hoster/YoutubeCom.py +++ b/module/plugins/hoster/YoutubeCom.py @@ -11,23 +11,23 @@ class YoutubeCom(Hoster): __pattern__ = r"http://(www\.)?(de\.)?\youtube\.com/watch\?v=.*" __version__ = "0.2" __config__ = [("quality", "sd;hd;fullhd", "Quality Setting", "hd"), - ("fmt", "int", "FMT Number 0-38", 0)] + ("fmt", "int", "FMT Number 0-38", 0)] __description__ = """Youtube.com Video Download Hoster""" __author_name__ = ("spoob") __author_mail__ = ("spoob@pyload.org") - formats = { 5 : (".flv", 400, 240), - 34 : (".flv", 640, 360), - 35 : (".flv", 854, 480), - 18 : (".mp4", 480, 360), - 22 : (".mp4", 1280, 720), - 37 : (".mp4", 1920, 1080), - 38 : (".mp4", 4096, 3072), - 43 : (".webm", 640, 360), - 45 : (".webm", 1280, 720), - 17 : (".3gp", 176, 144) - } - + formats = {5: (".flv", 400, 240), + 34: (".flv", 640, 360), + 35: (".flv", 854, 480), + 18: (".mp4", 480, 360), + 22: (".mp4", 1280, 720), + 37: (".mp4", 1920, 1080), + 38: (".mp4", 4096, 3072), + 43: (".webm", 640, 360), + 45: (".webm", 1280, 720), + 17: (".3gp", 176, 144) + } + def process(self, pyfile): html = self.load(pyfile.url, decode=True) @@ -60,7 +60,6 @@ class YoutubeCom(Hoster): fmt_url_map = re.search(fmt_pattern, html).group(1) links = urllib.unquote(fmt_url_map).split(",") - fmt_dict = {} for link in links: fmt = link.split("|")[0] @@ -73,13 +72,13 @@ class YoutubeCom(Hoster): self.logDebug("Found links: %s" % fmt_dict) - fmt = reduce(lambda x,y: x if abs(x-desired_fmt) <= abs(y-desired_fmt) else y, fmt_dict.keys()) + fmt = reduce(lambda x, y: x if abs(x - desired_fmt) <= abs(y - desired_fmt) else y, fmt_dict.keys()) self.logDebug("Choose fmt: %s" % fmt) file_suffix = ".flv" if fmt in self.formats: - file_suffix = self.formats[fmt][0] + file_suffix = self.formats[fmt][0] name = re.search(file_name_pattern, html).group(1).replace("/", "") + file_suffix pyfile.name = name #.replace("&", "&").replace("ö", "oe").replace("ä", "ae").replace("ü", "ue") diff --git a/module/web/webinterface.py b/module/web/webinterface.py index 1c28861ae..387a2cbbf 100644 --- a/module/web/webinterface.py +++ b/module/web/webinterface.py @@ -52,7 +52,7 @@ else: PYLOAD = ServerThread.core.server_methods config = ServerThread.core.config -from module.JsEngine import JsEngine +from module.common.JsEngine import JsEngine JS = JsEngine() -- cgit v1.2.3