summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--docs/system/webserver_evaluation.rst13
-rw-r--r--module/web/ServerThread.py144
-rw-r--r--module/web/servers.py162
-rw-r--r--module/web/utils.py28
-rw-r--r--module/web/webinterface.py19
-rwxr-xr-xpyload.py3
6 files changed, 236 insertions, 133 deletions
diff --git a/docs/system/webserver_evaluation.rst b/docs/system/webserver_evaluation.rst
index 2dc8d3265..607ef1dad 100644
--- a/docs/system/webserver_evaluation.rst
+++ b/docs/system/webserver_evaluation.rst
@@ -14,9 +14,9 @@ First selection
The first step was to take a short look at every webserver. For some it was not needed to further inspect them,
since they don't meet our requirements.
-================== ===============================================================
+================== ==================================================================
Disregarded server Reason
-================== ===============================================================
+================== ==================================================================
paste threaded server, no improvement to bundled one
twisted Too heavy (30 MB RAM min), far more complex as what we need
diesel Problems with setup, no default packages, Not working in tests
@@ -24,7 +24,8 @@ gunicorn Preforking server, messes many things up in our use-case
gevent Not usuable with several threads
gae Google App Engine, not for personal maschines
rocket threaded server, seems not better than bundled one
-================== ===============================================================
+waitress no large improvement to bundled threaded
+================== ==================================================================
pyLoad has an threaded server bundled itself. All threaded server that were tested seems not better than this
implementation and thus were not further benchmarked. "flup", known as "fastcgi" in pyload, serves a different
@@ -49,14 +50,16 @@ Server RAM (b.) RAM (a.) -c 1 -c 5 -c 1 -k -c 5 -k SSL Packages Not
wsgiref 21.7 22.6 1.240 1.179 1.312 1.513 No Included
threaded 25.5 28.0 0.912 1.139 0.656 0.784 Yes Included
tornado 23.8 25.9 0.874 0.935 - - Yes mac,deb,arch
+ freebsd
fapws3 22.3 23.8 0.740 0.733 0.786 0.594 No pip Very reliable under load,
problem with shutdown, will
need patches for integration
meinheld 22.3 23.7 0.622 1.001 1.076 1.388 Yes pip Segfaults when shutdown
eventlet 25.0 26.0 1.021 1.031 0.755 0.740 Yes mac,deb Struggles a bit under load
- More ram with -k (27.6)
+ freebsd More ram with -k (27.6)
-bjoern 21.7 23 0.623 1.062 - - No git memory-leak with faulty -k
+bjoern 21.7 23.2 0.623 0.513 - - No git, freebsd memory-leak with faulty -k
+ packages out of date
========== ======== ======== ====== ====== ======= ======= === ============= ================================
"wsgiref" is a standard implementation shipped with python and within pyLoad known as builtin.
diff --git a/module/web/ServerThread.py b/module/web/ServerThread.py
index ffe5ae380..99137b591 100644
--- a/module/web/ServerThread.py
+++ b/module/web/ServerThread.py
@@ -31,15 +31,14 @@ class WebServer(threading.Thread):
self.key = config["ssl"]["key"]
self.host = config['webinterface']['host']
self.port = config['webinterface']['port']
+ self.debug = config['general']['debug_mode']
+ self.force_server = config['webinterface']['force_server']
self.error = None
self.setDaemon(True)
def run(self):
self.running = True
-
- # TODO: clean this up
-
import webinterface
global webinterface
@@ -48,32 +47,20 @@ class WebServer(threading.Thread):
log.warning(_("SSL certificates not found."))
self.https = False
- if self.server == "fastcgi":
- try:
- import flup
- except:
- log.warning(_("Can't use %(server)s, python-flup is not installed!") % {
- "server": self.server})
- self.server = "builtin"
- elif self.server == "lightweight":
- try:
- import bjoern
- except Exception, e:
- log.error(_("Error importing lightweight server: %s") % e)
- log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern"))
- log.warning(_("Copy the boern.so to module/lib folder or use setup.py install"))
- log.warning(_("Of course you need to be familiar with linux and know how to compile software"))
- self.server = "builtin"
+ prefer = None
+
+ # These cases covers all settings
+ if self.server == "threaded":
+ prefer = "threaded"
+ elif self.server == "fastcgi":
+ prefer = "flup"
+ elif self.server == "fallback":
+ prefer = "wsgiref"
+
+ server = self.select_server(prefer)
try:
- if self.server == "fastcgi":
- self.start_fcgi()
- elif self.server == "threaded":
- self.start_threaded()
- elif self.server == "fallback":
- self.start_fallback()
- else:
- self.start_auto()
+ self.start_server(server)
except Exception, e:
log.error(_("Failed starting webserver: " + e.message))
@@ -81,72 +68,65 @@ class WebServer(threading.Thread):
if core:
core.print_exc()
- def start_auto(self):
- # TODO: select server
-
-# server = "wsgiref"
-# server = "tornado"
-# server = "fapws3"
-# server = "meinheld"
-# server = "eventlet"
-# server = "bjoern"
- server = "threaded"
-
- if server == "threaded":
- return self.start_threaded()
- if server == "wsgiref":
- return self.start_fallback()
- if server == "bjoern":
- return self.start_lightweight()
- if server == "meinheld":
- def noop(*args, **kwargs):
- pass
- from meinheld import server as sv
- sv.set_access_logger(None)
- sv.set_error_logger(None)
-
- sv.kill_server = noop
-
- log.info("AUTO server %s" % server)
- webinterface.run_server(host=self.host, port=self.port, server=server)
+ def select_server(self, prefer=None):
+ """ find a working server """
+ from servers import all_server
- def start_fallback(self):
- if self.https:
- log.warning(_("This server offers no SSL, please consider using threaded instead"))
+ unavailable = []
+ server = None
+ for server in all_server:
- log.info(_("Starting fallback webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port})
- webinterface.run_server(host=self.host, port=self.port)
+ if self.force_server and self.force_server == server.NAME:
+ break # Found server
+ # When force_server is set, no further checks have to be made
+ elif self.force_server:
+ continue
- def start_threaded(self):
- if self.https:
- log.info(_("Starting threaded SSL webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port})
- else:
- self.cert = ""
- self.key = ""
- log.info(_("Starting threaded webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port})
+ if prefer and prefer == server.NAME:
+ break # found prefered server
+ elif prefer: # prefer is similar to force, but force has precedence
+ continue
- webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key)
+ # Filter for server that offer ssl if needed
+ if self.https and not server.SSL:
+ continue
- def start_fcgi(self):
- from flup.server.threadedserver import ThreadedServer
+ try:
+ if server.find():
+ break # Found a server
+ else:
+ unavailable.append(server.NAME)
+ except Exception, e:
+ log.error(_("Failed importing webserver: " + e.message))
- def noop(*args, **kwargs):
- pass
+ if unavailable: # Just log whats not available to have some debug information
+ log.debug("Unavailable webserver: " + ",".join(unavailable))
- # Monkey patch signal handler, it does not work from threads
- ThreadedServer._installSignalHandlers = noop
+ if not server and self.force_server:
+ server = self.force_server # just return the name
- log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port})
- webinterface.run_server(host=self.host, port=self.port, server="flup")
+ return server
- def start_lightweight(self):
- if self.https:
- log.warning(_("This server offers no SSL, please consider using threaded instead"))
+ def start_server(self, server):
+
+ from servers import ServerAdapter
+
+ if issubclass(server, ServerAdapter):
+
+ if self.https and not server.SSL:
+ log.warning(_("This server offers no SSL, please consider using threaded instead"))
+
+ # Now instantiate the serverAdapter
+ server = server(self.host, self.port, self.key, self.cert, 6, self.debug) # todo, num_connections
+ name = server.NAME
+
+ else: # server is just a string
+ name = server
+
+ log.info(_("Starting %(name)s webserver: %(host)s:%(port)d") % {"name": name, "host": self.host, "port": self.port})
+ webinterface.run_server(host=self.host, port=self.port, server=server)
- log.info(
- _("Starting lightweight webserver (bjoern): %(host)s:%(port)d") % {"host": self.host, "port": self.port})
- webinterface.run_server(host=self.host, port=self.port, server="bjoern")
# check if an error was raised for n seconds
diff --git a/module/web/servers.py b/module/web/servers.py
new file mode 100644
index 000000000..a3c51e36b
--- /dev/null
+++ b/module/web/servers.py
@@ -0,0 +1,162 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from bottle import ServerAdapter as BaseAdapter
+
+class ServerAdapter(BaseAdapter):
+ SSL = False
+ NAME = ""
+
+ def __init__(self, host, port, key, cert, connections, debug, **kwargs):
+ BaseAdapter.__init__(self, host, port, **kwargs)
+ self.key = key
+ self.cert = cert
+ self.connection = connections
+ self.debug = debug
+
+ @classmethod
+ def find(cls):
+ """ Check if server is available by trying to import it
+
+ :raises Exception: importing C dependant library could also fail with other reasons
+ :return: True on success
+ """
+ try:
+ __import__(cls.NAME)
+ return True
+ except ImportError:
+ return False
+
+ def run(self, handler):
+ raise NotImplementedError
+
+
+class CherryPyWSGI(ServerAdapter):
+ SSL = True
+ NAME = "threaded"
+
+ @classmethod
+ def find(cls):
+ return True
+
+ def run(self, handler):
+ from wsgiserver import CherryPyWSGIServer
+
+ if self.cert and self.key:
+ CherryPyWSGIServer.ssl_certificate = self.cert
+ CherryPyWSGIServer.ssl_private_key = self.key
+
+ server = CherryPyWSGIServer((self.host, self.port), handler, numthreads=self.connection)
+ server.start()
+
+
+class FapwsServer(ServerAdapter):
+ """ Does not work very good currently """
+
+ NAME = "fapws"
+
+ def run(self, handler): # pragma: no cover
+ import fapws._evwsgi as evwsgi
+ from fapws import base, config
+
+ port = self.port
+ if float(config.SERVER_IDENT[-2:]) > 0.4:
+ # fapws3 silently changed its API in 0.5
+ port = str(port)
+ evwsgi.start(self.host, port)
+ evwsgi.set_base_module(base)
+
+ def app(environ, start_response):
+ environ['wsgi.multiprocess'] = False
+ return handler(environ, start_response)
+
+ evwsgi.wsgi_cb(('', app))
+ evwsgi.run()
+
+
+# TODO: ssl
+class MeinheldServer(ServerAdapter):
+ SSL = True
+ NAME = "meinheld"
+
+ def run(self, handler):
+ from meinheld import server
+
+ if self.quiet:
+ server.set_access_logger(None)
+ server.set_error_logger(None)
+
+ server.listen((self.host, self.port))
+ server.run(handler)
+
+# todo:ssl
+class TornadoServer(ServerAdapter):
+ """ The super hyped asynchronous server by facebook. Untested. """
+
+ SSL = True
+ NAME = "tornado"
+
+ def run(self, handler): # pragma: no cover
+ import tornado.wsgi, tornado.httpserver, tornado.ioloop
+
+ container = tornado.wsgi.WSGIContainer(handler)
+ server = tornado.httpserver.HTTPServer(container)
+ server.listen(port=self.port)
+ tornado.ioloop.IOLoop.instance().start()
+
+
+class BjoernServer(ServerAdapter):
+ """ Fast server written in C: https://github.com/jonashaag/bjoern """
+
+ NAME = "bjoern"
+
+ def run(self, handler):
+ from bjoern import run
+
+ run(handler, self.host, self.port)
+
+
+# todo: ssl
+class EventletServer(ServerAdapter):
+
+ SSL = True
+ NAME = "eventlet"
+
+ def run(self, handler):
+ from eventlet import wsgi, listen
+
+ try:
+ wsgi.server(listen((self.host, self.port)), handler,
+ log_output=(not self.quiet))
+ except TypeError:
+ # Needed to ignore the log
+ class NoopLog:
+ def write(self, *args):
+ pass
+
+ # Fallback, if we have old version of eventlet
+ wsgi.server(listen((self.host, self.port)), handler, log=NoopLog())
+
+
+class FlupFCGIServer(ServerAdapter):
+
+ SSL = False
+ NAME = "flup"
+
+ def run(self, handler): # pragma: no cover
+ import flup.server.fcgi
+ from flup.server.threadedserver import ThreadedServer
+
+ def noop(*args, **kwargs):
+ pass
+
+ # Monkey patch signal handler, it does not work from threads
+ ThreadedServer._installSignalHandlers = noop
+
+ self.options.setdefault('bindAddress', (self.host, self.port))
+ flup.server.fcgi.WSGIServer(handler, **self.options).run()
+
+# Order is important and gives every server precedence over others!
+all_server = [BjoernServer, TornadoServer, EventletServer, CherryPyWSGI]
+# Some are deactivated because they have some flaws
+##all_server = [FapwsServer, MeinheldServer, BjoernServer, TornadoServer, EventletServer, CherryPyWSGI] \ No newline at end of file
diff --git a/module/web/utils.py b/module/web/utils.py
index 0b8c8f9df..cbe8f3071 100644
--- a/module/web/utils.py
+++ b/module/web/utils.py
@@ -1,23 +1,8 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-"""
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, see <http://www.gnu.org/licenses/>.
-
- @author: RaNaN
-"""
import re
-from bottle import request, HTTPError, redirect, ServerAdapter
+from bottle import request, HTTPError, redirect
from webinterface import env, TEMPLATE, PYLOAD, SETUP
@@ -101,14 +86,3 @@ def login_required(perm=None):
return _view
return _dec
-
-
-class CherryPyWSGI(ServerAdapter):
-
- numthreads = 6
-
- def run(self, handler):
- from wsgiserver import CherryPyWSGIServer
-
- server = CherryPyWSGIServer((self.host, self.port), handler, numthreads=self.numthreads)
- server.start()
diff --git a/module/web/webinterface.py b/module/web/webinterface.py
index 0fdf3c2ab..cec0f24a4 100644
--- a/module/web/webinterface.py
+++ b/module/web/webinterface.py
@@ -124,26 +124,9 @@ import setup_app
import cnl_app
import api_app
-
# Server Adapter
-
-def run_server(host="0.0.0.0", port="8000", server="wsgiref"):
+def run_server(host, port, server):
run(app=web, host=host, port=port, quiet=True, server=server)
-
-def run_threaded(host="0.0.0.0", port="8000", threads=6, cert="", key=""):
- from wsgiserver import CherryPyWSGIServer
-
- if cert and key:
- CherryPyWSGIServer.ssl_certificate = cert
- CherryPyWSGIServer.ssl_private_key = key
-
- # todo: threads configurable
- from utils import CherryPyWSGI
- CherryPyWSGI.numthreads = threads
-
- run(app=web, host=host, port=port, server=CherryPyWSGI, quiet=True)
-
-
if __name__ == "__main__":
run(app=web, port=8001)
diff --git a/pyload.py b/pyload.py
index 389e1648c..43d091a36 100755
--- a/pyload.py
+++ b/pyload.py
@@ -475,7 +475,8 @@ class Core(object):
self.shutdown()
self.log.info(_("pyLoad quits"))
self.removeLogger()
- _exit(0) #@TODO thrift blocks shutdown
+ _exit(0)
+ # TODO check exits codes, clean exit is still blocked
self.threadManager.work()
self.interactionManager.work()
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-05 08:58:05 +0100