evaluate different webservers

8 files changed, 144 insertions, 19 deletions

diff --git a/docs/api/datatypes.rst b/docs/api/datatypes.rst
index d4df56905..5516189f3 100644
--- a/docs/api/datatypes.rst
+++ b/docs/api/datatypes.rst
@@ -258,11 +258,12 @@ for various languages. It is also a good overview of avaible methods and return
      }
 
      struct ConfigInfo {
-       1: string name,
+       1: string name
        2: string label,
        3: string description,
-       4: bool saved,
-       5: bool activated,
+       4: bool addon,
+       5: bool user_context,
+       6: optional bool activated,
      }
 
      struct EventInfo {
@@ -365,10 +366,16 @@ for various languages. It is also a good overview of avaible methods and return
        ///////////////////////
 
        map<string, ConfigHolder> getConfig(),
-       list<ConfigInfo> getGlobalPlugins(),
-       list<ConfigInfo> getUserPlugins(),
+       string getConfigValue(1: string section, 2: string option),
+
+       // two methods with ambigous classification, could be configuration or addon related
+       list<ConfigInfo> getCoreConfig(),
+       list<ConfigInfo> getPluginConfig(),
+       list<ConfigInfo> getAvailablePlugins(),
 
        ConfigHolder configurePlugin(1: PluginName plugin),
+
+       void setConfigValue(1: string section, 2: string option, 3: string value),
        void saveConfig(1: ConfigHolder config),
        void deleteConfig(1: PluginName plugin),
        void setConfigHandler(1: PluginName plugin, 2: InteractionID iid, 3: JSONString value),
@@ -523,8 +530,8 @@ for various languages. It is also a good overview of avaible methods and return
        // Addon Methods
        ///////////////////////
 
-       map<PluginName, list<AddonInfo>> getAllInfo(),
-       list<AddonInfo> getInfoByPlugin(1: PluginName plugin),
+       //map<PluginName, list<AddonInfo>> getAllInfo(),
+       //list<AddonInfo> getInfoByPlugin(1: PluginName plugin),
 
        map<PluginName, list<AddonService>> getAddonHandler(),
        bool hasAddonHandler(1: PluginName plugin, 2: string func),
diff --git a/docs/system/overview.rst b/docs/system/overview.rst
index 00e439f45..09e3bc857 100755
--- a/docs/system/overview.rst
+++ b/docs/system/overview.rst
@@ -20,6 +20,7 @@ In this section you will find information and diagrams to better understand the
 
     plugin_hierarchy.rst
     hoster_diagrams.rst
+    webserver_evaluation.rst
 
 
 .. rubric:: Footnotes
\ No newline at end of file
diff --git a/docs/system/webserver_evaluation.rst b/docs/system/webserver_evaluation.rst
new file mode 100644
index 000000000..2dc8d3265
--- /dev/null
+++ b/docs/system/webserver_evaluation.rst
@@ -0,0 +1,85 @@
+.. _webserver_evaluation:
+
+====================
+Webserver Evaluation
+====================
+
+pyLoad supports all kind of webserver that are usable with bottle.py [1]_.
+For this reason we evaluted each of them to find the ones that are worth to be supported in pyLoad. The results of this
+evaluation make sure pyload can select the most suited webserver with its auto selecting algorithm.
+
+First selection
+---------------
+
+The first step was to take a short look at every webserver. For some it was not needed to further inspect them,
+since they don't meet our requirements.
+
+================== ===============================================================
+Disregarded server Reason
+================== ===============================================================
+paste              threaded server, no improvement to bundled one
+twisted            Too heavy (30 MB RAM min), far more complex as what we need
+diesel             Problems with setup, no default packages, Not working in tests
+gunicorn           Preforking server, messes many things up in our use-case
+gevent             Not usuable with several threads
+gae                Google App Engine, not for personal maschines
+rocket             threaded server, seems not better than bundled one
+================== ===============================================================
+
+pyLoad has an threaded server bundled itself. All threaded server that were tested seems not better than this
+implementation and thus were not further benchmarked. "flup", known as "fastcgi" in pyload, serves a different
+use-case and is not taken into consideration here.
+
+Comparision
+-----------
+
+The remaining servers, were evaluated for different criteria. We ran the following benchmark with different options:
+
+    ab -n 15000 -c 1 http://127.0.0.1:8001/login
+
+This benchmark was ran with -c 1, -c 5, as well as -k option to test performance with more concurrency and keep-alive
+feature, we use time per request (mean, across all concurrent requests) for comparision.
+
+Additionally we collected RAM usage statistic before (b.) (only 2-3 pages retrieved) and after (a.) the benchmarks were run.
+The comparision also includes some notes and available packages or features, especially SSL is of interest.
+
+========== ======== ======== ====== ====== ======= ======= === ============= ================================
+Server     RAM (b.) RAM (a.) -c 1   -c 5   -c 1 -k -c 5 -k SSL Packages      Notes
+========== ======== ======== ====== ====== ======= ======= === ============= ================================
+wsgiref    21.7     22.6     1.240  1.179  1.312   1.513   No  Included
+threaded   25.5     28.0     0.912  1.139  0.656   0.784   Yes Included
+tornado    23.8     25.9     0.874  0.935  -       -       Yes mac,deb,arch
+fapws3     22.3     23.8     0.740  0.733  0.786   0.594   No  pip           Very reliable under load,
+                                                                             problem with shutdown, will
+                                                                             need patches for integration
+meinheld   22.3     23.7     0.622  1.001  1.076   1.388   Yes pip           Segfaults when shutdown
+eventlet   25.0     26.0     1.021  1.031  0.755   0.740   Yes mac,deb       Struggles a bit under load
+                                                                             More ram with -k (27.6)
+
+bjoern     21.7     23       0.623  1.062  -       -       No  git           memory-leak with faulty -k
+========== ======== ======== ====== ====== ======= ======= === ============= ================================
+
+"wsgiref" is a standard implementation shipped with python and within pyLoad known as builtin.
+"threaded" is taken from cherryPy and also included with pyLoad.
+The keep-alive implementation of "ab" is not 100% compliant, some server struggle with it.
+
+Conclusion
+----------
+
+The wsgiref server is known to show strange performance on some system and is therefore not selected by default anymore.
+The included threaded server has all needed functions, including SSL, and is usable without any other packages.
+Threaded will be selected in case none of the other server is installed.
+
+Our auto-select will favor RAM usage over performance too choose the most lightweight server as possible.
+Activating SSL will decrease the options, many lightweight servers don't include SSL by choice.
+They suggest tools like pound [2]_, stunnel [3]_, or any other reverse proxy capable server. Also these that are capable
+of SSL suggest using other tools, their SSL performance was not tested here.
+
+pyLoad will select a server in following order:
+fapws3 -> meinheld -> bjoern -> tornado -> eventlet
+
+.. rubric:: Footnotes
+
+.. [1] https://bitbucket.org/spoob/pyload/src/127adb41465712548949ea872a5453e4b0b0fbb8/module/lib/bottle.py?at=default#cl-2555
+.. [2] http://www.apsis.ch/pound/
+.. [3] https://www.stunnel.org/index.html
\ No newline at end of file
diff --git a/module/config/default.py b/module/config/default.py
index e55ba6593..902d4a6ad 100644
--- a/module/config/default.py
+++ b/module/config/default.py
@@ -62,7 +62,8 @@ def make_config(config):
             ("template", "str", _("Template"), _("Tooltip"), "default"),
             ("activated", "bool", _("Activated"), _("Tooltip"), True),
             ("prefix", "str", _("Path Prefix"), _("Tooltip"), ""),
-            ("server", "threaded;fastcgi;fallback;lightweight", _("Server"), _("Tooltip"), "threaded"),
+            ("server", "auto;threaded;fallback;fastcgi", _("Server"), _("Tooltip"), "auto"),
+            ("force_server", "str", _("Favor specific server"), _("Tooltip"), ""),
             ("host", "ip", _("IP"), _("Tooltip"), "0.0.0.0"),
             ("https", "bool", _("Use HTTPS"), _("Tooltip"), False),
             ("port", "int", _("Port"), _("Tooltip"), 8001),
diff --git a/module/web/ServerThread.py b/module/web/ServerThread.py
index dc30f4bc5..ffe5ae380 100644
--- a/module/web/ServerThread.py
+++ b/module/web/ServerThread.py
@@ -38,6 +38,8 @@ class WebServer(threading.Thread):
     def run(self):
         self.running = True
 
+        # TODO: clean this up
+
         import webinterface
         global webinterface
 
@@ -63,22 +65,51 @@ class WebServer(threading.Thread):
                 log.warning(_("Of course you need to be familiar with linux and know how to compile software"))
                 self.server = "builtin"
 
-
         try:
             if self.server == "fastcgi":
                 self.start_fcgi()
-            elif self.server in ("threaded", "builtin"):
+            elif self.server == "threaded":
                 self.start_threaded()
-            elif self.server == "lightweight":
-                self.start_lightweight()
-            else:
+            elif self.server == "fallback":
                 self.start_fallback()
+            else:
+                self.start_auto()
+
         except Exception, e:
             log.error(_("Failed starting webserver: " + e.message))
             self.error = e
             if core:
                 core.print_exc()
 
+    def start_auto(self):
+        # TODO: select server
+
+#        server = "wsgiref"
+#        server = "tornado"
+#        server = "fapws3"
+#        server = "meinheld"
+#        server = "eventlet"
+#        server = "bjoern"
+        server = "threaded"
+
+        if server == "threaded":
+            return self.start_threaded()
+        if server == "wsgiref":
+            return self.start_fallback()
+        if server == "bjoern":
+            return self.start_lightweight()
+        if server == "meinheld":
+            def noop(*args, **kwargs):
+                pass
+            from meinheld import server as sv
+            sv.set_access_logger(None)
+            sv.set_error_logger(None)
+
+            sv.kill_server = noop
+
+        log.info("AUTO server %s" % server)
+        webinterface.run_server(host=self.host, port=self.port, server=server)
+
     def start_fallback(self):
         if self.https:
             log.warning(_("This server offers no SSL, please consider using threaded instead"))
diff --git a/module/web/static/js/views/packageView.js b/module/web/static/js/views/packageView.js
index 365f3a69f..5d8aa7738 100644
--- a/module/web/static/js/views/packageView.js
+++ b/module/web/static/js/views/packageView.js
@@ -53,13 +53,12 @@ define(['jquery', 'views/abstract/itemView', 'underscore', 'views/fileView'],
                         ul.append(new fileView({model: file}).render().el);
                     });
 
-                    this.$el.append(ul);
-
                     // TODO: additionally it could be placed out of viewport first
                     // The real height can only be retrieved when element is on DOM and display:true
                     ul.css('visibility', 'hidden');
+                    this.$el.append(ul);
+
                     var height = ul.height();
-                    ul.css('visibility', '');
 
                     // Hide the element when not expanded
                     if (!this.expanded) {
@@ -67,6 +66,7 @@ define(['jquery', 'views/abstract/itemView', 'underscore', 'views/fileView'],
                         ul.height(0);
                     }
 
+                    ul.css('visibility', '');
                     ul.data('height', height);
                     console.log(ul.data("height"));
 
diff --git a/module/web/static/js/views/settingsView.js b/module/web/static/js/views/settingsView.js
index aba16eb66..a322cdae7 100644
--- a/module/web/static/js/views/settingsView.js
+++ b/module/web/static/js/views/settingsView.js
@@ -1,7 +1,7 @@
 define(['jquery', 'underscore', 'backbone'],
     function($, _, Backbone) {
 
-        // Renders a single package item
+        // Renders settings over view page
         return Backbone.View.extend({
 
             el: "#content",
diff --git a/pyload-cli.py b/pyload-cli.py
index cf8fabd1a..684958f1e 100755
--- a/pyload-cli.py
+++ b/pyload-cli.py
@@ -333,10 +333,10 @@ class Cli:
 
 
         elif command == "pause":
-            self.client.pause()
+            self.client.pauseServer()
 
         elif command == "unpause":
-            self.client.unpause()
+            self.client.unpauseServer()
 
         elif command == "toggle":
             self.client.togglePause()