api tester

8 files changed, 228 insertions, 19 deletions

diff --git a/module/common/APIExerciser.py b/module/common/APIExerciser.py
new file mode 100644
index 000000000..f645f12df
--- /dev/null
+++ b/module/common/APIExerciser.py
@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+
+import string
+from threading import Thread
+from random import choice, random, sample, randint
+from time import time, sleep
+
+from traceback import print_exc, format_exc
+
+from module.remote.thriftbackend.ThriftClient import ThriftClient
+
+def createURLs():
+    """ create some urls, some may fail """
+    urls = []
+    for x in range(0, randint(20, 100)):
+        name = "DEBUG_API"
+        if randint(0, 5) == 5:
+            name = "" #this link will fail
+
+        urls.append(name + "".join(sample(string.ascii_letters, randint(10, 20))))
+
+    return urls
+
+AVOID = (0,3,8)
+
+class APIExerciser(Thread):
+    """ tests api randomly """
+
+    def __init__(self, core):
+        Thread.__init__(self)
+        self.setDaemon(True)
+        self.core = core
+        self.methods = core.server_methods
+        self.count = 0 #number of methods
+        self.time = time()
+
+        self.start()
+
+    def run(self):
+        out = open("error.log", "ab")
+        #core errors are not logged of course
+        out.write("\n" + "Starting\n")
+        out.flush()
+
+        while True:
+            try:
+                self.testAPI()
+            except Exception:
+                print_exc()
+                out.write(format_exc() + 2 * "\n")
+                out.flush()
+
+            if not self.count % 100:
+                print "Tested %s api calls" % self.count
+            if not self.count % 1000:
+                out.write("Tested %s api calls\n" % self.count)
+                out.flush()
+
+
+                #sleep(random() / 500)
+
+    def testAPI(self):
+        m = ["status_downloads", "status_server", "add_package", "get_package_data", "get_file_data", "del_links",
+             "del_packages",
+             "get_queue", "get_collector", "get_queue_info", "get_collector_info", "is_captcha_waiting"]
+
+        method = choice(m)
+        #print "Testing:", method
+
+        if hasattr(self, method):
+            res = getattr(self, method)()
+        else:
+            res = getattr(self.methods, method)()
+
+        self.count += 1
+
+        #print res
+
+    def add_package(self):
+        name = "".join(sample(string.ascii_letters, 10))
+        urls = createURLs()
+
+        self.methods.add_package(name, urls, 1)
+
+
+    def del_links(self):
+        info = self.methods.get_queue()
+        if not info: return
+
+        pid = choice(info.keys())
+        pack = info[pid]
+        links = pack["links"]
+        #filter links which are not queued, finished or failed
+        fids = filter(lambda x: links[x]["status"] not in AVOID, links.keys())
+
+        if len(fids):
+            fids = sample(fids, randint(1, max(len(fids) / 2, 1)))
+            self.methods.del_links(fids)
+
+
+    def del_packages(self):
+        info = self.methods.get_queue_info()
+        if not info: return
+
+        pids = info.keys()
+        if len(pids):
+            pids = sample(pids, randint(1, max(len(pids) / 2, 1)))
+            filtered = []
+
+            for p in pids:
+                info = self.methods.get_package_data(p)
+                append = True
+                for link in info["links"].itervalues():
+                    if link["status"] not in AVOID:
+                        append = False
+                        break
+
+                if append: filtered.append(p)
+
+            self.methods.del_packages(filtered)
+
+    def get_file_data(self):
+        info = self.methods.get_queue()
+        if info:
+            p = info[choice(info.keys())]
+            if p["links"]:
+                self.methods.get_file_data(choice(p["links"].keys()))
+
+    def get_package_data(self):
+        info = self.methods.get_queue_info()
+        if info:
+            self.methods.get_package_data(choice(info.keys()))
+
+
+class ThriftExerciser(APIExerciser):
+    def __init__(self, core):
+        self.thrift = ThriftClient()
+        self.thrift.login("user", "pw")
+
+        APIExerciser.__init__(self, core)
+
+    def testAPI(self):
+        m = ["statusDownloads", "statusServer", "addPackage", "getPackageData", "getFileData", "deleteFiles",
+             "deletePackages",
+             "getQueue", "getCollector", "getQueueData", "getCollectorData", "isCaptchaWaiting"]
+
+        method = choice(m)
+        #print "Testing:", method
+
+        if hasattr(self, method):
+            res = getattr(self, method)()
+        else:
+            res = getattr(self.thrift, method)()
+
+        self.count += 1
+
+        #print res
+
+    def addPackage(self):
+        name = "".join(sample(string.ascii_letters, 10))
+        urls = createURLs()
+
+        self.thrift.addPackage(name, urls, 0)
+
+
+    def deleteFiles(self):
+        info = self.thrift.getQueueData()
+        if not info: return
+
+        pack = choice(info)
+        fids = pack.links
+
+        if len(fids):
+            fids = [f.fid for f in sample(fids, randint(1, max(len(fids) / 2, 1)))]
+            self.thrift.deleteFiles(fids)
+
+
+    def deletePackages(self):
+        info = self.thrift.getQueue()
+        if not info: return
+
+        pids = [p.pid for p in info]
+        if len(pids):
+            pids = sample(pids, randint(1, max(len(pids) / 2, 1)))
+            self.thrift.deletePackages(pids)
+
+    def getFileData(self):
+        info = self.thrift.getQueueData()
+        if info:
+            p = choice(info)
+            if p.links:
+                self.thrift.getFileData(choice(p.links).fid)
+
+    def getPackageData(self):
+        info = self.thrift.getQueue()
+        if info:
+            self.thrift.getPackageData(choice(info).pid)
\ No newline at end of file
diff --git a/module/JsEngine.py b/module/common/JsEngine.py
index 0b451c9dd..0b451c9dd 100644
--- a/module/JsEngine.py
+++ b/module/common/JsEngine.py
diff --git a/module/common/__init__.py b/module/common/__init__.py
new file mode 100644
index 000000000..de6d13128
--- /dev/null
+++ b/module/common/__init__.py
@@ -0,0 +1,2 @@
+__author__ = 'christian'
+  
\ No newline at end of file
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index 449b60a59..396069adb 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -37,7 +37,7 @@ if os.name != "nt":
 
 from itertools import islice
 
-from module.utils import save_join, decode, removeChars
+from module.utils import save_join, removeChars
 
 def chunks(iterable, size):
     it = iter(iterable)
diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py
index 7b3f56c4d..49bab50f1 100644
--- a/module/plugins/hoster/BasePlugin.py
+++ b/module/plugins/hoster/BasePlugin.py
@@ -20,6 +20,11 @@ class BasePlugin(Hoster):
     def process(self, pyfile):
         """main function"""
 
+        #debug part, for api exerciser
+        if pyfile.url.startswith("DEBUG_API"):
+            self.multiDL = False
+            return
+
 #        self.__name__ = "NetloadIn"
 #        pyfile.name = "test"
 #        self.html = self.load("http://localhost:9000/short")
diff --git a/module/plugins/hoster/YoutubeCom.py b/module/plugins/hoster/YoutubeCom.py
index ccd98606e..0e0fc2d9e 100644
--- a/module/plugins/hoster/YoutubeCom.py
+++ b/module/plugins/hoster/YoutubeCom.py
@@ -11,23 +11,23 @@ class YoutubeCom(Hoster):
     __pattern__ = r"http://(www\.)?(de\.)?\youtube\.com/watch\?v=.*"
     __version__ = "0.2"
     __config__ = [("quality", "sd;hd;fullhd", "Quality Setting", "hd"),
-                   ("fmt", "int", "FMT Number 0-38", 0)]
+            ("fmt", "int", "FMT Number 0-38", 0)]
     __description__ = """Youtube.com Video Download Hoster"""
     __author_name__ = ("spoob")
     __author_mail__ = ("spoob@pyload.org")
 
-    formats = {  5 : (".flv", 400, 240),
-                34 : (".flv", 640, 360),
-                35 : (".flv", 854, 480),
-                18 : (".mp4", 480, 360),
-                22 : (".mp4", 1280, 720),
-                37 : (".mp4", 1920, 1080),
-                38 : (".mp4", 4096, 3072),
-                43 : (".webm", 640, 360),
-                45 : (".webm", 1280, 720),
-                17 : (".3gp", 176, 144)
-              }
-                        
+    formats = {5: (".flv", 400, 240),
+               34: (".flv", 640, 360),
+               35: (".flv", 854, 480),
+               18: (".mp4", 480, 360),
+               22: (".mp4", 1280, 720),
+               37: (".mp4", 1920, 1080),
+               38: (".mp4", 4096, 3072),
+               43: (".webm", 640, 360),
+               45: (".webm", 1280, 720),
+               17: (".3gp", 176, 144)
+    }
+
 
     def process(self, pyfile):
         html = self.load(pyfile.url, decode=True)
@@ -60,7 +60,6 @@ class YoutubeCom(Hoster):
         fmt_url_map = re.search(fmt_pattern, html).group(1)
         links = urllib.unquote(fmt_url_map).split(",")
 
-
         fmt_dict = {}
         for link in links:
             fmt = link.split("|")[0]
@@ -73,13 +72,13 @@ class YoutubeCom(Hoster):
 
         self.logDebug("Found links: %s" % fmt_dict)
 
-        fmt = reduce(lambda x,y: x if abs(x-desired_fmt) <= abs(y-desired_fmt) else y, fmt_dict.keys())
+        fmt = reduce(lambda x, y: x if abs(x - desired_fmt) <= abs(y - desired_fmt) else y, fmt_dict.keys())
 
         self.logDebug("Choose fmt: %s" % fmt)
 
         file_suffix = ".flv"
         if fmt in self.formats:
-	    file_suffix = self.formats[fmt][0]
+            file_suffix = self.formats[fmt][0]
         name = re.search(file_name_pattern, html).group(1).replace("/", "") + file_suffix
         pyfile.name = name #.replace("&amp;", "&").replace("ö", "oe").replace("ä", "ae").replace("ü", "ue")       
 
diff --git a/module/web/webinterface.py b/module/web/webinterface.py
index 1c28861ae..387a2cbbf 100644
--- a/module/web/webinterface.py
+++ b/module/web/webinterface.py
@@ -52,7 +52,7 @@ else:
     PYLOAD = ServerThread.core.server_methods
     config = ServerThread.core.config
 
-from module.JsEngine import JsEngine
+from module.common.JsEngine import JsEngine
 
 JS = JsEngine()
 
diff --git a/pyLoadCore.py b/pyLoadCore.py
index 1bec0f7cc..08ae526d2 100755
--- a/pyLoadCore.py
+++ b/pyLoadCore.py
@@ -51,7 +51,7 @@ from module.ThreadManager import ThreadManager
 from module.web.ServerThread import WebServer
 from module.PyFile import PyFile
 from module.Scheduler import Scheduler
-from module.JsEngine import JsEngine
+from module.common.JsEngine import JsEngine
 from module.remote.RemoteManager import RemoteManager
 from module.database import DatabaseBackend
 from module.database import FileHandler
@@ -389,6 +389,12 @@ class Core(object):
 
         self.log.info(_("pyLoad is up and running"))
 
+        #test api
+#        from module.common.APIExerciser import APIExerciser
+#        APIExerciser(self)
+#        APIExerciser(self)
+#        APIExerciser(self)
+
         while True:
             sleep(2)
             if self.do_restart: