diff options
| -rw-r--r-- | docs/api/datatypes.rst | 21 | ||||
| -rwxr-xr-x | docs/system/overview.rst | 1 | ||||
| -rw-r--r-- | docs/system/webserver_evaluation.rst | 88 | ||||
| -rw-r--r-- | module/config/default.py | 3 | ||||
| -rw-r--r-- | module/web/ServerThread.py | 115 | ||||
| -rw-r--r-- | module/web/servers.py | 162 | ||||
| -rw-r--r-- | module/web/static/js/views/packageView.js | 6 | ||||
| -rw-r--r-- | module/web/static/js/views/settingsView.js | 2 | ||||
| -rw-r--r-- | module/web/utils.py | 28 | ||||
| -rw-r--r-- | module/web/webinterface.py | 19 | ||||
| -rwxr-xr-x | pyload-cli.py | 4 | ||||
| -rwxr-xr-x | pyload.py | 3 |
12 files changed, 340 insertions, 112 deletions
diff --git a/docs/api/datatypes.rst b/docs/api/datatypes.rst index d4df56905..5516189f3 100644 --- a/docs/api/datatypes.rst +++ b/docs/api/datatypes.rst @@ -258,11 +258,12 @@ for various languages. It is also a good overview of avaible methods and return } struct ConfigInfo { - 1: string name, + 1: string name 2: string label, 3: string description, - 4: bool saved, - 5: bool activated, + 4: bool addon, + 5: bool user_context, + 6: optional bool activated, } struct EventInfo { @@ -365,10 +366,16 @@ for various languages. It is also a good overview of avaible methods and return /////////////////////// map<string, ConfigHolder> getConfig(), - list<ConfigInfo> getGlobalPlugins(), - list<ConfigInfo> getUserPlugins(), + string getConfigValue(1: string section, 2: string option), + + // two methods with ambigous classification, could be configuration or addon related + list<ConfigInfo> getCoreConfig(), + list<ConfigInfo> getPluginConfig(), + list<ConfigInfo> getAvailablePlugins(), ConfigHolder configurePlugin(1: PluginName plugin), + + void setConfigValue(1: string section, 2: string option, 3: string value), void saveConfig(1: ConfigHolder config), void deleteConfig(1: PluginName plugin), void setConfigHandler(1: PluginName plugin, 2: InteractionID iid, 3: JSONString value), @@ -523,8 +530,8 @@ for various languages. It is also a good overview of avaible methods and return // Addon Methods /////////////////////// - map<PluginName, list<AddonInfo>> getAllInfo(), - list<AddonInfo> getInfoByPlugin(1: PluginName plugin), + //map<PluginName, list<AddonInfo>> getAllInfo(), + //list<AddonInfo> getInfoByPlugin(1: PluginName plugin), map<PluginName, list<AddonService>> getAddonHandler(), bool hasAddonHandler(1: PluginName plugin, 2: string func), diff --git a/docs/system/overview.rst b/docs/system/overview.rst index 00e439f45..09e3bc857 100755 --- a/docs/system/overview.rst +++ b/docs/system/overview.rst @@ -20,6 +20,7 @@ In this section you will find information and diagrams to better understand the plugin_hierarchy.rst hoster_diagrams.rst + webserver_evaluation.rst .. rubric:: Footnotes
\ No newline at end of file diff --git a/docs/system/webserver_evaluation.rst b/docs/system/webserver_evaluation.rst new file mode 100644 index 000000000..607ef1dad --- /dev/null +++ b/docs/system/webserver_evaluation.rst @@ -0,0 +1,88 @@ +.. _webserver_evaluation: + +==================== +Webserver Evaluation +==================== + +pyLoad supports all kind of webserver that are usable with bottle.py [1]_. +For this reason we evaluted each of them to find the ones that are worth to be supported in pyLoad. The results of this +evaluation make sure pyload can select the most suited webserver with its auto selecting algorithm. + +First selection +--------------- + +The first step was to take a short look at every webserver. For some it was not needed to further inspect them, +since they don't meet our requirements. + +================== ================================================================== +Disregarded server Reason +================== ================================================================== +paste threaded server, no improvement to bundled one +twisted Too heavy (30 MB RAM min), far more complex as what we need +diesel Problems with setup, no default packages, Not working in tests +gunicorn Preforking server, messes many things up in our use-case +gevent Not usuable with several threads +gae Google App Engine, not for personal maschines +rocket threaded server, seems not better than bundled one +waitress no large improvement to bundled threaded +================== ================================================================== + +pyLoad has an threaded server bundled itself. All threaded server that were tested seems not better than this +implementation and thus were not further benchmarked. "flup", known as "fastcgi" in pyload, serves a different +use-case and is not taken into consideration here. + +Comparision +----------- + +The remaining servers, were evaluated for different criteria. We ran the following benchmark with different options: + + ab -n 15000 -c 1 http://127.0.0.1:8001/login + +This benchmark was ran with -c 1, -c 5, as well as -k option to test performance with more concurrency and keep-alive +feature, we use time per request (mean, across all concurrent requests) for comparision. + +Additionally we collected RAM usage statistic before (b.) (only 2-3 pages retrieved) and after (a.) the benchmarks were run. +The comparision also includes some notes and available packages or features, especially SSL is of interest. + +========== ======== ======== ====== ====== ======= ======= === ============= ================================ +Server RAM (b.) RAM (a.) -c 1 -c 5 -c 1 -k -c 5 -k SSL Packages Notes +========== ======== ======== ====== ====== ======= ======= === ============= ================================ +wsgiref 21.7 22.6 1.240 1.179 1.312 1.513 No Included +threaded 25.5 28.0 0.912 1.139 0.656 0.784 Yes Included +tornado 23.8 25.9 0.874 0.935 - - Yes mac,deb,arch + freebsd +fapws3 22.3 23.8 0.740 0.733 0.786 0.594 No pip Very reliable under load, + problem with shutdown, will + need patches for integration +meinheld 22.3 23.7 0.622 1.001 1.076 1.388 Yes pip Segfaults when shutdown +eventlet 25.0 26.0 1.021 1.031 0.755 0.740 Yes mac,deb Struggles a bit under load + freebsd More ram with -k (27.6) + +bjoern 21.7 23.2 0.623 0.513 - - No git, freebsd memory-leak with faulty -k + packages out of date +========== ======== ======== ====== ====== ======= ======= === ============= ================================ + +"wsgiref" is a standard implementation shipped with python and within pyLoad known as builtin. +"threaded" is taken from cherryPy and also included with pyLoad. +The keep-alive implementation of "ab" is not 100% compliant, some server struggle with it. + +Conclusion +---------- + +The wsgiref server is known to show strange performance on some system and is therefore not selected by default anymore. +The included threaded server has all needed functions, including SSL, and is usable without any other packages. +Threaded will be selected in case none of the other server is installed. + +Our auto-select will favor RAM usage over performance too choose the most lightweight server as possible. +Activating SSL will decrease the options, many lightweight servers don't include SSL by choice. +They suggest tools like pound [2]_, stunnel [3]_, or any other reverse proxy capable server. Also these that are capable +of SSL suggest using other tools, their SSL performance was not tested here. + +pyLoad will select a server in following order: +fapws3 -> meinheld -> bjoern -> tornado -> eventlet + +.. rubric:: Footnotes + +.. [1] https://bitbucket.org/spoob/pyload/src/127adb41465712548949ea872a5453e4b0b0fbb8/module/lib/bottle.py?at=default#cl-2555 +.. [2] http://www.apsis.ch/pound/ +.. [3] https://www.stunnel.org/index.html
\ No newline at end of file diff --git a/module/config/default.py b/module/config/default.py index e55ba6593..902d4a6ad 100644 --- a/module/config/default.py +++ b/module/config/default.py @@ -62,7 +62,8 @@ def make_config(config): ("template", "str", _("Template"), _("Tooltip"), "default"), ("activated", "bool", _("Activated"), _("Tooltip"), True), ("prefix", "str", _("Path Prefix"), _("Tooltip"), ""), - ("server", "threaded;fastcgi;fallback;lightweight", _("Server"), _("Tooltip"), "threaded"), + ("server", "auto;threaded;fallback;fastcgi", _("Server"), _("Tooltip"), "auto"), + ("force_server", "str", _("Favor specific server"), _("Tooltip"), ""), ("host", "ip", _("IP"), _("Tooltip"), "0.0.0.0"), ("https", "bool", _("Use HTTPS"), _("Tooltip"), False), ("port", "int", _("Port"), _("Tooltip"), 8001), diff --git a/module/web/ServerThread.py b/module/web/ServerThread.py index dc30f4bc5..99137b591 100644 --- a/module/web/ServerThread.py +++ b/module/web/ServerThread.py @@ -31,13 +31,14 @@ class WebServer(threading.Thread): self.key = config["ssl"]["key"] self.host = config['webinterface']['host'] self.port = config['webinterface']['port'] + self.debug = config['general']['debug_mode'] + self.force_server = config['webinterface']['force_server'] self.error = None self.setDaemon(True) def run(self): self.running = True - import webinterface global webinterface @@ -46,76 +47,86 @@ class WebServer(threading.Thread): log.warning(_("SSL certificates not found.")) self.https = False - if self.server == "fastcgi": - try: - import flup - except: - log.warning(_("Can't use %(server)s, python-flup is not installed!") % { - "server": self.server}) - self.server = "builtin" - elif self.server == "lightweight": - try: - import bjoern - except Exception, e: - log.error(_("Error importing lightweight server: %s") % e) - log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) - log.warning(_("Copy the boern.so to module/lib folder or use setup.py install")) - log.warning(_("Of course you need to be familiar with linux and know how to compile software")) - self.server = "builtin" + prefer = None + + # These cases covers all settings + if self.server == "threaded": + prefer = "threaded" + elif self.server == "fastcgi": + prefer = "flup" + elif self.server == "fallback": + prefer = "wsgiref" + server = self.select_server(prefer) try: - if self.server == "fastcgi": - self.start_fcgi() - elif self.server in ("threaded", "builtin"): - self.start_threaded() - elif self.server == "lightweight": - self.start_lightweight() - else: - self.start_fallback() + self.start_server(server) + except Exception, e: log.error(_("Failed starting webserver: " + e.message)) self.error = e if core: core.print_exc() - def start_fallback(self): - if self.https: - log.warning(_("This server offers no SSL, please consider using threaded instead")) + def select_server(self, prefer=None): + """ find a working server """ + from servers import all_server - log.info(_("Starting fallback webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_server(host=self.host, port=self.port) + unavailable = [] + server = None + for server in all_server: - def start_threaded(self): - if self.https: - log.info(_("Starting threaded SSL webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - else: - self.cert = "" - self.key = "" - log.info(_("Starting threaded webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + if self.force_server and self.force_server == server.NAME: + break # Found server + # When force_server is set, no further checks have to be made + elif self.force_server: + continue - webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) + if prefer and prefer == server.NAME: + break # found prefered server + elif prefer: # prefer is similar to force, but force has precedence + continue - def start_fcgi(self): - from flup.server.threadedserver import ThreadedServer + # Filter for server that offer ssl if needed + if self.https and not server.SSL: + continue - def noop(*args, **kwargs): - pass + try: + if server.find(): + break # Found a server + else: + unavailable.append(server.NAME) + except Exception, e: + log.error(_("Failed importing webserver: " + e.message)) - # Monkey patch signal handler, it does not work from threads - ThreadedServer._installSignalHandlers = noop + if unavailable: # Just log whats not available to have some debug information + log.debug("Unavailable webserver: " + ",".join(unavailable)) - log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_server(host=self.host, port=self.port, server="flup") + if not server and self.force_server: + server = self.force_server # just return the name + return server - def start_lightweight(self): - if self.https: - log.warning(_("This server offers no SSL, please consider using threaded instead")) - log.info( - _("Starting lightweight webserver (bjoern): %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_server(host=self.host, port=self.port, server="bjoern") + def start_server(self, server): + + from servers import ServerAdapter + + if issubclass(server, ServerAdapter): + + if self.https and not server.SSL: + log.warning(_("This server offers no SSL, please consider using threaded instead")) + + # Now instantiate the serverAdapter + server = server(self.host, self.port, self.key, self.cert, 6, self.debug) # todo, num_connections + name = server.NAME + + else: # server is just a string + name = server + + log.info(_("Starting %(name)s webserver: %(host)s:%(port)d") % {"name": name, "host": self.host, "port": self.port}) + webinterface.run_server(host=self.host, port=self.port, server=server) + # check if an error was raised for n seconds diff --git a/module/web/servers.py b/module/web/servers.py new file mode 100644 index 000000000..a3c51e36b --- /dev/null +++ b/module/web/servers.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from bottle import ServerAdapter as BaseAdapter + +class ServerAdapter(BaseAdapter): + SSL = False + NAME = "" + + def __init__(self, host, port, key, cert, connections, debug, **kwargs): + BaseAdapter.__init__(self, host, port, **kwargs) + self.key = key + self.cert = cert + self.connection = connections + self.debug = debug + + @classmethod + def find(cls): + """ Check if server is available by trying to import it + + :raises Exception: importing C dependant library could also fail with other reasons + :return: True on success + """ + try: + __import__(cls.NAME) + return True + except ImportError: + return False + + def run(self, handler): + raise NotImplementedError + + +class CherryPyWSGI(ServerAdapter): + SSL = True + NAME = "threaded" + + @classmethod + def find(cls): + return True + + def run(self, handler): + from wsgiserver import CherryPyWSGIServer + + if self.cert and self.key: + CherryPyWSGIServer.ssl_certificate = self.cert + CherryPyWSGIServer.ssl_private_key = self.key + + server = CherryPyWSGIServer((self.host, self.port), handler, numthreads=self.connection) + server.start() + + +class FapwsServer(ServerAdapter): + """ Does not work very good currently """ + + NAME = "fapws" + + def run(self, handler): # pragma: no cover + import fapws._evwsgi as evwsgi + from fapws import base, config + + port = self.port + if float(config.SERVER_IDENT[-2:]) > 0.4: + # fapws3 silently changed its API in 0.5 + port = str(port) + evwsgi.start(self.host, port) + evwsgi.set_base_module(base) + + def app(environ, start_response): + environ['wsgi.multiprocess'] = False + return handler(environ, start_response) + + evwsgi.wsgi_cb(('', app)) + evwsgi.run() + + +# TODO: ssl +class MeinheldServer(ServerAdapter): + SSL = True + NAME = "meinheld" + + def run(self, handler): + from meinheld import server + + if self.quiet: + server.set_access_logger(None) + server.set_error_logger(None) + + server.listen((self.host, self.port)) + server.run(handler) + +# todo:ssl +class TornadoServer(ServerAdapter): + """ The super hyped asynchronous server by facebook. Untested. """ + + SSL = True + NAME = "tornado" + + def run(self, handler): # pragma: no cover + import tornado.wsgi, tornado.httpserver, tornado.ioloop + + container = tornado.wsgi.WSGIContainer(handler) + server = tornado.httpserver.HTTPServer(container) + server.listen(port=self.port) + tornado.ioloop.IOLoop.instance().start() + + +class BjoernServer(ServerAdapter): + """ Fast server written in C: https://github.com/jonashaag/bjoern """ + + NAME = "bjoern" + + def run(self, handler): + from bjoern import run + + run(handler, self.host, self.port) + + +# todo: ssl +class EventletServer(ServerAdapter): + + SSL = True + NAME = "eventlet" + + def run(self, handler): + from eventlet import wsgi, listen + + try: + wsgi.server(listen((self.host, self.port)), handler, + log_output=(not self.quiet)) + except TypeError: + # Needed to ignore the log + class NoopLog: + def write(self, *args): + pass + + # Fallback, if we have old version of eventlet + wsgi.server(listen((self.host, self.port)), handler, log=NoopLog()) + + +class FlupFCGIServer(ServerAdapter): + + SSL = False + NAME = "flup" + + def run(self, handler): # pragma: no cover + import flup.server.fcgi + from flup.server.threadedserver import ThreadedServer + + def noop(*args, **kwargs): + pass + + # Monkey patch signal handler, it does not work from threads + ThreadedServer._installSignalHandlers = noop + + self.options.setdefault('bindAddress', (self.host, self.port)) + flup.server.fcgi.WSGIServer(handler, **self.options).run() + +# Order is important and gives every server precedence over others! +all_server = [BjoernServer, TornadoServer, EventletServer, CherryPyWSGI] +# Some are deactivated because they have some flaws +##all_server = [FapwsServer, MeinheldServer, BjoernServer, TornadoServer, EventletServer, CherryPyWSGI]
\ No newline at end of file diff --git a/module/web/static/js/views/packageView.js b/module/web/static/js/views/packageView.js index 365f3a69f..5d8aa7738 100644 --- a/module/web/static/js/views/packageView.js +++ b/module/web/static/js/views/packageView.js @@ -53,13 +53,12 @@ define(['jquery', 'views/abstract/itemView', 'underscore', 'views/fileView'], ul.append(new fileView({model: file}).render().el); }); - this.$el.append(ul); - // TODO: additionally it could be placed out of viewport first // The real height can only be retrieved when element is on DOM and display:true ul.css('visibility', 'hidden'); + this.$el.append(ul); + var height = ul.height(); - ul.css('visibility', ''); // Hide the element when not expanded if (!this.expanded) { @@ -67,6 +66,7 @@ define(['jquery', 'views/abstract/itemView', 'underscore', 'views/fileView'], ul.height(0); } + ul.css('visibility', ''); ul.data('height', height); console.log(ul.data("height")); diff --git a/module/web/static/js/views/settingsView.js b/module/web/static/js/views/settingsView.js index aba16eb66..a322cdae7 100644 --- a/module/web/static/js/views/settingsView.js +++ b/module/web/static/js/views/settingsView.js @@ -1,7 +1,7 @@ define(['jquery', 'underscore', 'backbone'], function($, _, Backbone) { - // Renders a single package item + // Renders settings over view page return Backbone.View.extend({ el: "#content", diff --git a/module/web/utils.py b/module/web/utils.py index 0b8c8f9df..cbe8f3071 100644 --- a/module/web/utils.py +++ b/module/web/utils.py @@ -1,23 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" import re -from bottle import request, HTTPError, redirect, ServerAdapter +from bottle import request, HTTPError, redirect from webinterface import env, TEMPLATE, PYLOAD, SETUP @@ -101,14 +86,3 @@ def login_required(perm=None): return _view return _dec - - -class CherryPyWSGI(ServerAdapter): - - numthreads = 6 - - def run(self, handler): - from wsgiserver import CherryPyWSGIServer - - server = CherryPyWSGIServer((self.host, self.port), handler, numthreads=self.numthreads) - server.start() diff --git a/module/web/webinterface.py b/module/web/webinterface.py index 0fdf3c2ab..cec0f24a4 100644 --- a/module/web/webinterface.py +++ b/module/web/webinterface.py @@ -124,26 +124,9 @@ import setup_app import cnl_app
import api_app
-
# Server Adapter
-
-def run_server(host="0.0.0.0", port="8000", server="wsgiref"):
+def run_server(host, port, server):
run(app=web, host=host, port=port, quiet=True, server=server)
-
-def run_threaded(host="0.0.0.0", port="8000", threads=6, cert="", key=""):
- from wsgiserver import CherryPyWSGIServer
-
- if cert and key:
- CherryPyWSGIServer.ssl_certificate = cert
- CherryPyWSGIServer.ssl_private_key = key
-
- # todo: threads configurable
- from utils import CherryPyWSGI
- CherryPyWSGI.numthreads = threads
-
- run(app=web, host=host, port=port, server=CherryPyWSGI, quiet=True)
-
-
if __name__ == "__main__":
run(app=web, port=8001)
diff --git a/pyload-cli.py b/pyload-cli.py index cf8fabd1a..684958f1e 100755 --- a/pyload-cli.py +++ b/pyload-cli.py @@ -333,10 +333,10 @@ class Cli: elif command == "pause": - self.client.pause() + self.client.pauseServer() elif command == "unpause": - self.client.unpause() + self.client.unpauseServer() elif command == "toggle": self.client.togglePause() @@ -475,7 +475,8 @@ class Core(object): self.shutdown() self.log.info(_("pyLoad quits")) self.removeLogger() - _exit(0) #@TODO thrift blocks shutdown + _exit(0) + # TODO check exits codes, clean exit is still blocked self.threadManager.work() self.interactionManager.work() |