diff options
| -rw-r--r-- | module/common/APIExerciser.py | 197 | ||||
| -rw-r--r-- | module/common/JsEngine.py (renamed from module/JsEngine.py) | 0 | ||||
| -rw-r--r-- | module/common/__init__.py | 2 | ||||
| -rw-r--r-- | module/plugins/Plugin.py | 2 | ||||
| -rw-r--r-- | module/plugins/hoster/BasePlugin.py | 5 | ||||
| -rw-r--r-- | module/plugins/hoster/YoutubeCom.py | 31 | ||||
| -rw-r--r-- | module/web/webinterface.py | 2 | ||||
| -rwxr-xr-x | pyLoadCore.py | 8 |
8 files changed, 228 insertions, 19 deletions
diff --git a/module/common/APIExerciser.py b/module/common/APIExerciser.py new file mode 100644 index 000000000..f645f12df --- /dev/null +++ b/module/common/APIExerciser.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- + +import string +from threading import Thread +from random import choice, random, sample, randint +from time import time, sleep + +from traceback import print_exc, format_exc + +from module.remote.thriftbackend.ThriftClient import ThriftClient + +def createURLs(): + """ create some urls, some may fail """ + urls = [] + for x in range(0, randint(20, 100)): + name = "DEBUG_API" + if randint(0, 5) == 5: + name = "" #this link will fail + + urls.append(name + "".join(sample(string.ascii_letters, randint(10, 20)))) + + return urls + +AVOID = (0,3,8) + +class APIExerciser(Thread): + """ tests api randomly """ + + def __init__(self, core): + Thread.__init__(self) + self.setDaemon(True) + self.core = core + self.methods = core.server_methods + self.count = 0 #number of methods + self.time = time() + + self.start() + + def run(self): + out = open("error.log", "ab") + #core errors are not logged of course + out.write("\n" + "Starting\n") + out.flush() + + while True: + try: + self.testAPI() + except Exception: + print_exc() + out.write(format_exc() + 2 * "\n") + out.flush() + + if not self.count % 100: + print "Tested %s api calls" % self.count + if not self.count % 1000: + out.write("Tested %s api calls\n" % self.count) + out.flush() + + + #sleep(random() / 500) + + def testAPI(self): + m = ["status_downloads", "status_server", "add_package", "get_package_data", "get_file_data", "del_links", + "del_packages", + "get_queue", "get_collector", "get_queue_info", "get_collector_info", "is_captcha_waiting"] + + method = choice(m) + #print "Testing:", method + + if hasattr(self, method): + res = getattr(self, method)() + else: + res = getattr(self.methods, method)() + + self.count += 1 + + #print res + + def add_package(self): + name = "".join(sample(string.ascii_letters, 10)) + urls = createURLs() + + self.methods.add_package(name, urls, 1) + + + def del_links(self): + info = self.methods.get_queue() + if not info: return + + pid = choice(info.keys()) + pack = info[pid] + links = pack["links"] + #filter links which are not queued, finished or failed + fids = filter(lambda x: links[x]["status"] not in AVOID, links.keys()) + + if len(fids): + fids = sample(fids, randint(1, max(len(fids) / 2, 1))) + self.methods.del_links(fids) + + + def del_packages(self): + info = self.methods.get_queue_info() + if not info: return + + pids = info.keys() + if len(pids): + pids = sample(pids, randint(1, max(len(pids) / 2, 1))) + filtered = [] + + for p in pids: + info = self.methods.get_package_data(p) + append = True + for link in info["links"].itervalues(): + if link["status"] not in AVOID: + append = False + break + + if append: filtered.append(p) + + self.methods.del_packages(filtered) + + def get_file_data(self): + info = self.methods.get_queue() + if info: + p = info[choice(info.keys())] + if p["links"]: + self.methods.get_file_data(choice(p["links"].keys())) + + def get_package_data(self): + info = self.methods.get_queue_info() + if info: + self.methods.get_package_data(choice(info.keys())) + + +class ThriftExerciser(APIExerciser): + def __init__(self, core): + self.thrift = ThriftClient() + self.thrift.login("user", "pw") + + APIExerciser.__init__(self, core) + + def testAPI(self): + m = ["statusDownloads", "statusServer", "addPackage", "getPackageData", "getFileData", "deleteFiles", + "deletePackages", + "getQueue", "getCollector", "getQueueData", "getCollectorData", "isCaptchaWaiting"] + + method = choice(m) + #print "Testing:", method + + if hasattr(self, method): + res = getattr(self, method)() + else: + res = getattr(self.thrift, method)() + + self.count += 1 + + #print res + + def addPackage(self): + name = "".join(sample(string.ascii_letters, 10)) + urls = createURLs() + + self.thrift.addPackage(name, urls, 0) + + + def deleteFiles(self): + info = self.thrift.getQueueData() + if not info: return + + pack = choice(info) + fids = pack.links + + if len(fids): + fids = [f.fid for f in sample(fids, randint(1, max(len(fids) / 2, 1)))] + self.thrift.deleteFiles(fids) + + + def deletePackages(self): + info = self.thrift.getQueue() + if not info: return + + pids = [p.pid for p in info] + if len(pids): + pids = sample(pids, randint(1, max(len(pids) / 2, 1))) + self.thrift.deletePackages(pids) + + def getFileData(self): + info = self.thrift.getQueueData() + if info: + p = choice(info) + if p.links: + self.thrift.getFileData(choice(p.links).fid) + + def getPackageData(self): + info = self.thrift.getQueue() + if info: + self.thrift.getPackageData(choice(info).pid)
\ No newline at end of file diff --git a/module/JsEngine.py b/module/common/JsEngine.py index 0b451c9dd..0b451c9dd 100644 --- a/module/JsEngine.py +++ b/module/common/JsEngine.py diff --git a/module/common/__init__.py b/module/common/__init__.py new file mode 100644 index 000000000..de6d13128 --- /dev/null +++ b/module/common/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'christian' +
\ No newline at end of file diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 449b60a59..396069adb 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -37,7 +37,7 @@ if os.name != "nt": from itertools import islice -from module.utils import save_join, decode, removeChars +from module.utils import save_join, removeChars def chunks(iterable, size): it = iter(iterable) diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 7b3f56c4d..49bab50f1 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -20,6 +20,11 @@ class BasePlugin(Hoster): def process(self, pyfile): """main function""" + #debug part, for api exerciser + if pyfile.url.startswith("DEBUG_API"): + self.multiDL = False + return + # self.__name__ = "NetloadIn" # pyfile.name = "test" # self.html = self.load("http://localhost:9000/short") diff --git a/module/plugins/hoster/YoutubeCom.py b/module/plugins/hoster/YoutubeCom.py index ccd98606e..0e0fc2d9e 100644 --- a/module/plugins/hoster/YoutubeCom.py +++ b/module/plugins/hoster/YoutubeCom.py @@ -11,23 +11,23 @@ class YoutubeCom(Hoster): __pattern__ = r"http://(www\.)?(de\.)?\youtube\.com/watch\?v=.*" __version__ = "0.2" __config__ = [("quality", "sd;hd;fullhd", "Quality Setting", "hd"), - ("fmt", "int", "FMT Number 0-38", 0)] + ("fmt", "int", "FMT Number 0-38", 0)] __description__ = """Youtube.com Video Download Hoster""" __author_name__ = ("spoob") __author_mail__ = ("spoob@pyload.org") - formats = { 5 : (".flv", 400, 240), - 34 : (".flv", 640, 360), - 35 : (".flv", 854, 480), - 18 : (".mp4", 480, 360), - 22 : (".mp4", 1280, 720), - 37 : (".mp4", 1920, 1080), - 38 : (".mp4", 4096, 3072), - 43 : (".webm", 640, 360), - 45 : (".webm", 1280, 720), - 17 : (".3gp", 176, 144) - } - + formats = {5: (".flv", 400, 240), + 34: (".flv", 640, 360), + 35: (".flv", 854, 480), + 18: (".mp4", 480, 360), + 22: (".mp4", 1280, 720), + 37: (".mp4", 1920, 1080), + 38: (".mp4", 4096, 3072), + 43: (".webm", 640, 360), + 45: (".webm", 1280, 720), + 17: (".3gp", 176, 144) + } + def process(self, pyfile): html = self.load(pyfile.url, decode=True) @@ -60,7 +60,6 @@ class YoutubeCom(Hoster): fmt_url_map = re.search(fmt_pattern, html).group(1) links = urllib.unquote(fmt_url_map).split(",") - fmt_dict = {} for link in links: fmt = link.split("|")[0] @@ -73,13 +72,13 @@ class YoutubeCom(Hoster): self.logDebug("Found links: %s" % fmt_dict) - fmt = reduce(lambda x,y: x if abs(x-desired_fmt) <= abs(y-desired_fmt) else y, fmt_dict.keys()) + fmt = reduce(lambda x, y: x if abs(x - desired_fmt) <= abs(y - desired_fmt) else y, fmt_dict.keys()) self.logDebug("Choose fmt: %s" % fmt) file_suffix = ".flv" if fmt in self.formats: - file_suffix = self.formats[fmt][0] + file_suffix = self.formats[fmt][0] name = re.search(file_name_pattern, html).group(1).replace("/", "") + file_suffix pyfile.name = name #.replace("&", "&").replace("ö", "oe").replace("ä", "ae").replace("ü", "ue") diff --git a/module/web/webinterface.py b/module/web/webinterface.py index 1c28861ae..387a2cbbf 100644 --- a/module/web/webinterface.py +++ b/module/web/webinterface.py @@ -52,7 +52,7 @@ else: PYLOAD = ServerThread.core.server_methods config = ServerThread.core.config -from module.JsEngine import JsEngine +from module.common.JsEngine import JsEngine JS = JsEngine() diff --git a/pyLoadCore.py b/pyLoadCore.py index 1bec0f7cc..08ae526d2 100755 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -51,7 +51,7 @@ from module.ThreadManager import ThreadManager from module.web.ServerThread import WebServer from module.PyFile import PyFile from module.Scheduler import Scheduler -from module.JsEngine import JsEngine +from module.common.JsEngine import JsEngine from module.remote.RemoteManager import RemoteManager from module.database import DatabaseBackend from module.database import FileHandler @@ -389,6 +389,12 @@ class Core(object): self.log.info(_("pyLoad is up and running")) + #test api +# from module.common.APIExerciser import APIExerciser +# APIExerciser(self) +# APIExerciser(self) +# APIExerciser(self) + while True: sleep(2) if self.do_restart: |