From fd11206fe002e46119dd58f31e526a8e85a9a1a1 Mon Sep 17 00:00:00 2001 From: mkaay Date: Mon, 16 Nov 2009 17:43:33 +0100 Subject: added lastEffectiveURL --- module/network/Request.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'module') diff --git a/module/network/Request.py b/module/network/Request.py index eb67da95d..9f27af4ec 100755 --- a/module/network/Request.py +++ b/module/network/Request.py @@ -47,6 +47,7 @@ class Request: self.abort = False + self.lastEffectiveURL = None self.lastURL = None self.auth = False @@ -138,6 +139,7 @@ class Request: self.pycurl.perform() + self.lastEffectiveURL = self.pycurl.getinfo(pycurl.EFFECTIVE_URL) self.lastURL = url header = self.get_header() @@ -165,6 +167,7 @@ class Request: if rep.headers["content-encoding"] == "gzip": output = GzipFile('', 'r', 0, StringIO(output)).read() + self.lastEffectiveURL = rep.geturl() self.lastURL = url return output -- cgit v1.2.3 From 900964358d3c5328292e8af73730c3bc25e75406 Mon Sep 17 00:00:00 2001 From: mkaay Date: Mon, 16 Nov 2009 18:22:29 +0100 Subject: relink.us fix --- module/plugins/RelinkUs.py | 50 +++++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 14 deletions(-) (limited to 'module') diff --git a/module/plugins/RelinkUs.py b/module/plugins/RelinkUs.py index 4bfa5a18b..840497413 100644 --- a/module/plugins/RelinkUs.py +++ b/module/plugins/RelinkUs.py @@ -2,8 +2,9 @@ # -*- coding: utf-8 -*- import re +import time -from module.Plugin import Plugin +from Plugin import Plugin class RelinkUs(Plugin): @@ -12,14 +13,15 @@ class RelinkUs(Plugin): props = {} props['name'] = "RelinkUs" props['type'] = "container" - props['pattern'] = r"http://(www\.)?relink.us/go.php" - props['version'] = "0.1" + props['pattern'] = r"http://(www\.)?relink.us/(f|((view|go).php))" + props['version'] = "0.2" props['description'] = """Relink.us Container Plugin""" - props['author_name'] = ("spoob") - props['author_mail'] = ("spoob@pyload.org") + props['author_name'] = ("Sleeper-") + props['author_mail'] = ("@nonymous") self.props = props self.parent = parent self.html = None + self.multi_dl = False def file_exists(self): """ returns True or False @@ -27,13 +29,33 @@ class RelinkUs(Plugin): return True def proceed(self, url, location): - url = self.parent.url - self.html = self.req.load(url) - container_id = url.split("id=")[-1] + container_id = self.parent.url.split("/")[-1].split("id=")[-1] + url = "http://relink.us/view.php?id="+container_id + self.html = self.req.load(url, cookies=True) temp_links = [] - link_number = len(re.findall(r"test_\d+", self.html)) - for number in range(0, link_number): - new_link = re.search("src='(.*)'>", self.req.load("http://relink.us/f/%s/1/%i" % (container_id, number))).group(1) - temp_links.append(new_link) - print temp_links - self.links = temp_links + + # Download Ad-Frames, otherwise we aren't enabled for download + iframes = re.findall("src=['\"]([^'\"]*)['\"](.*)>", self.html) + for iframe in iframes: + self.req.load("http://relink.us/"+iframe[0], cookies=True) + + link_strings = re.findall(r"onclick=\"getFile\(\'([^)]*)\'\);changeBackgroundColor", self.html) + + for link_string in link_strings: + self.req.lastURL = url + + # Set Download File + framereq = self.req.load("http://relink.us/frame.php?"+link_string, cookies=True) + + new_link = self.req.lastEffectiveURL + + if re.match(r"http://(www\.)?relink.us/",new_link): + # Find iframe + new_link = re.search("src=['\"]([^'\"]*)['\"](.*)>", framereq).group(1) + # Wait some secs for relink.us server... + time.sleep(5) + + print new_link + temp_links.append(new_link) + + self.links = temp_links -- cgit v1.2.3 From 737345718371b5faae703b7ed3f4e9725abf5158 Mon Sep 17 00:00:00 2001 From: mkaay Date: Mon, 16 Nov 2009 18:32:37 +0100 Subject: indentation fix --- module/plugins/RelinkUs.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'module') diff --git a/module/plugins/RelinkUs.py b/module/plugins/RelinkUs.py index 840497413..9ca209be3 100644 --- a/module/plugins/RelinkUs.py +++ b/module/plugins/RelinkUs.py @@ -34,28 +34,28 @@ class RelinkUs(Plugin): self.html = self.req.load(url, cookies=True) temp_links = [] - # Download Ad-Frames, otherwise we aren't enabled for download - iframes = re.findall("src=['\"]([^'\"]*)['\"](.*)>", self.html) - for iframe in iframes: - self.req.load("http://relink.us/"+iframe[0], cookies=True) + # Download Ad-Frames, otherwise we aren't enabled for download + iframes = re.findall("src=['\"]([^'\"]*)['\"](.*)>", self.html) + for iframe in iframes: + self.req.load("http://relink.us/"+iframe[0], cookies=True) - link_strings = re.findall(r"onclick=\"getFile\(\'([^)]*)\'\);changeBackgroundColor", self.html) + link_strings = re.findall(r"onclick=\"getFile\(\'([^)]*)\'\);changeBackgroundColor", self.html) - for link_string in link_strings: - self.req.lastURL = url + for link_string in link_strings: + self.req.lastURL = url - # Set Download File - framereq = self.req.load("http://relink.us/frame.php?"+link_string, cookies=True) + # Set Download File + framereq = self.req.load("http://relink.us/frame.php?"+link_string, cookies=True) - new_link = self.req.lastEffectiveURL + new_link = self.req.lastEffectiveURL - if re.match(r"http://(www\.)?relink.us/",new_link): - # Find iframe - new_link = re.search("src=['\"]([^'\"]*)['\"](.*)>", framereq).group(1) - # Wait some secs for relink.us server... - time.sleep(5) + if re.match(r"http://(www\.)?relink.us/",new_link): + # Find iframe + new_link = re.search("src=['\"]([^'\"]*)['\"](.*)>", framereq).group(1) + # Wait some secs for relink.us server... + time.sleep(5) - print new_link - temp_links.append(new_link) + print new_link + temp_links.append(new_link) - self.links = temp_links + self.links = temp_links -- cgit v1.2.3 From 2599ae0c23fd9f0766ba22a1741d458a08985823 Mon Sep 17 00:00:00 2001 From: mkaay Date: Mon, 16 Nov 2009 18:38:16 +0100 Subject: indentation fix, 2nd try -.- --- module/plugins/RelinkUs.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'module') diff --git a/module/plugins/RelinkUs.py b/module/plugins/RelinkUs.py index 9ca209be3..1700beb2b 100644 --- a/module/plugins/RelinkUs.py +++ b/module/plugins/RelinkUs.py @@ -34,28 +34,28 @@ class RelinkUs(Plugin): self.html = self.req.load(url, cookies=True) temp_links = [] - # Download Ad-Frames, otherwise we aren't enabled for download - iframes = re.findall("src=['\"]([^'\"]*)['\"](.*)>", self.html) - for iframe in iframes: - self.req.load("http://relink.us/"+iframe[0], cookies=True) + # Download Ad-Frames, otherwise we aren't enabled for download + iframes = re.findall("src=['\"]([^'\"]*)['\"](.*)>", self.html) + for iframe in iframes: + self.req.load("http://relink.us/"+iframe[0], cookies=True) - link_strings = re.findall(r"onclick=\"getFile\(\'([^)]*)\'\);changeBackgroundColor", self.html) + link_strings = re.findall(r"onclick=\"getFile\(\'([^)]*)\'\);changeBackgroundColor", self.html) - for link_string in link_strings: - self.req.lastURL = url + for link_string in link_strings: + self.req.lastURL = url - # Set Download File - framereq = self.req.load("http://relink.us/frame.php?"+link_string, cookies=True) + # Set Download File + framereq = self.req.load("http://relink.us/frame.php?"+link_string, cookies=True) - new_link = self.req.lastEffectiveURL + new_link = self.req.lastEffectiveURL - if re.match(r"http://(www\.)?relink.us/",new_link): - # Find iframe - new_link = re.search("src=['\"]([^'\"]*)['\"](.*)>", framereq).group(1) - # Wait some secs for relink.us server... - time.sleep(5) + if re.match(r"http://(www\.)?relink.us/",new_link): + # Find iframe + new_link = re.search("src=['\"]([^'\"]*)['\"](.*)>", framereq).group(1) + # Wait some secs for relink.us server... + time.sleep(5) - print new_link - temp_links.append(new_link) + print new_link + temp_links.append(new_link) - self.links = temp_links + self.links = temp_links -- cgit v1.2.3 From ec73cb69f724da8594acad320e0ff1cf9543d131 Mon Sep 17 00:00:00 2001 From: mkaay Date: Mon, 16 Nov 2009 18:40:28 +0100 Subject: indentation fix, 3nd try.. --- module/plugins/RelinkUs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module') diff --git a/module/plugins/RelinkUs.py b/module/plugins/RelinkUs.py index 1700beb2b..786733558 100644 --- a/module/plugins/RelinkUs.py +++ b/module/plugins/RelinkUs.py @@ -50,10 +50,10 @@ class RelinkUs(Plugin): new_link = self.req.lastEffectiveURL if re.match(r"http://(www\.)?relink.us/",new_link): - # Find iframe - new_link = re.search("src=['\"]([^'\"]*)['\"](.*)>", framereq).group(1) - # Wait some secs for relink.us server... - time.sleep(5) + # Find iframe + new_link = re.search("src=['\"]([^'\"]*)['\"](.*)>", framereq).group(1) + # Wait some secs for relink.us server... + time.sleep(5) print new_link temp_links.append(new_link) -- cgit v1.2.3 From 7579e0750c7bd81c916012e2842c8a96ad8fa414 Mon Sep 17 00:00:00 2001 From: mkaay Date: Sun, 22 Nov 2009 15:04:30 +0100 Subject: secure xmlrpc --- module/file_list.py | 2 +- module/remote/SecureXMLRPCServer.py | 103 ++++++++++++++++++++++++++++++++++++ module/remote/SocketServer.py | 84 ----------------------------- module/thread_list.py | 2 +- 4 files changed, 105 insertions(+), 86 deletions(-) create mode 100644 module/remote/SecureXMLRPCServer.py delete mode 100644 module/remote/SocketServer.py (limited to 'module') diff --git a/module/file_list.py b/module/file_list.py index c55ce5bc6..1701b801b 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -113,7 +113,7 @@ class File_List(object): output = open('links.pkl', 'wb') cPickle.dump(self.data, output, -1) - self.inform_client() + #self.inform_client() self.lock.release() diff --git a/module/remote/SecureXMLRPCServer.py b/module/remote/SecureXMLRPCServer.py new file mode 100644 index 000000000..fb666b4ab --- /dev/null +++ b/module/remote/SecureXMLRPCServer.py @@ -0,0 +1,103 @@ +# Source: http://sources.gentoo.org/viewcvs.py/gimli/server/SecureXMLRPCServer.py?view=markup +# which seems to be based on http://www.sabren.net/code/python/SecureXMLRPCServer.py +# +# Changes: +# 2007-01-06 Christian Hoffmann +# * Bugfix: replaced getattr by hasattr in the conditional +# (lead to an error otherwise) +# * SecureXMLRPCServer: added self.instance = None, otherwise a "wrong" +# exception is raised when calling unknown methods via xmlrpc +# * Added HTTP Basic authentication support +# +# Modified for the Sceradon project +# +# This code is in the public domain +# and is provided AS-IS WITH NO WARRANTY WHATSOEVER. +# $Id: SecureXMLRPCServer.py 5 2007-01-06 17:54:13Z hoffie $ + +from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler +from OpenSSL import SSL +import SocketServer +import socket +import base64 + +class SecureSocketServer(SocketServer.TCPServer, SocketServer.ThreadingMixIn): + def __init__(self, addr, cert, key, requestHandler, verify_cert_func=None): + SocketServer.TCPServer.__init__(self, addr, requestHandler) + ctx = SSL.Context(SSL.SSLv23_METHOD) + if not verify_cert_func and hasattr(self, 'verify_client_cert'): + verify_cert_func = getattr(self, 'verify_client_cert') + if verify_cert_func: + ctx.set_verify(SSL.VERIFY_PEER|SSL.VERIFY_FAIL_IF_NO_PEER_CERT, verify_cert_func) + ctx.use_privatekey_file(key) + ctx.use_certificate_file(cert) + + tmpConnection = SSL.Connection(ctx, socket.socket(socket.AF_INET, socket.SOCK_STREAM)) + self.socket = SecureSocketConnection(tmpConnection) + + self.server_bind() + self.server_activate() + + def finish_request(self, request, client_address): + """Finish one request by instantiating RequestHandlerClass.""" + self.RequestHandlerClass(request, client_address, self) + + +class SecureXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): + def __init__(self, request, client_address, server, client_digest=None): + self.authMap = server.getAuthenticationMap() + SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server) + self.client_digest = client_digest + + def setup(self): + self.connection = self.request + self.rfile = socket._fileobject(self.request, "rb", self.rbufsize) + self.wfile = socket._fileobject(self.request, "wb", self.wbufsize) + + def do_POST(self): + # authentication + if self.authMap != None: # explicit None! + if self.headers.has_key('authorization') and self.headers['authorization'].startswith('Basic '): + authenticationString = base64.b64decode(self.headers['authorization'].split(' ')[1]) + if authenticationString.find(':') != -1: + username, password = authenticationString.split(':', 1) + if self.authMap.has_key(username) and self.verifyPassword(username, password): + return SimpleXMLRPCRequestHandler.do_POST(self) + self.send_response(401) + self.end_headers() + return False + return SimpleXMLRPCRequestHandler.do_POST(self) + + def verifyPassword(self, username, givenPassword): + return self.authMap[username] == givenPassword + + +class SecureXMLRPCServer(SecureSocketServer, SimpleXMLRPCServer): + def __init__(self, address, cert, key, authenticationMap = None, handler=SecureXMLRPCRequestHandler, verify_cert_func=None): + self.logRequests = False + SecureSocketServer.__init__(self, address, cert, key, handler, verify_cert_func) + # This comes from SimpleXMLRPCServer.__init__()->SimpleXMLRPCDispatcher.__init__() + self.funcs = {} + self.instance = None + self.authenticationMap = authenticationMap + + def getAuthenticationMap(self): + return self.authenticationMap + + +class SecureSocketConnection: + def __init__(self, connection): + self.__dict__["connection"] = connection + + def __getattr__(self, name): + return getattr(self.__dict__["connection"], name) + + def __setattr__(self, name, value): + setattr(self.__dict__["connection"], name, value) + + def shutdown(self, how=1): + self.__dict__["connection"].shutdown() + + def accept(self): + connection, address = self.__dict__["connection"].accept() + return (SecureSocketConnection(connection), address) diff --git a/module/remote/SocketServer.py b/module/remote/SocketServer.py deleted file mode 100644 index 2edd1d9f3..000000000 --- a/module/remote/SocketServer.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -authored by: RaNaN - -This modul class handels all incoming and outgoing data between server and gui - -""" -import asynchat -import asyncore -import socket -import threading -import time - -from RequestHandler import RequestHandler - - -class ServerThread(threading.Thread): - def __init__(self, pycore): - threading.Thread.__init__(self) - self.setDaemon(True) - self.server = MainServerSocket(int(pycore.config['remote']['port']), pycore) - - def run(self): - asyncore.loop() - print "loop closed" - - def sockets(self): - """returns all connected sockets in a list""" - sockets = [] - for value in asyncore.socket_map.values(): - if SecondaryServerSocket == value.__class__: - sockets.append(value) - - return sockets - - def push_all(self, obj): - """push obj to all sockets""" - for socket in self.sockets(): - socket.push_obj(obj) - - -class MainServerSocket(asyncore.dispatcher): - def __init__(self, port, pycore): - asyncore.dispatcher.__init__(self) - pycore.logger.info('initing Remote-Server') - self.pycore = pycore - try: - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.bind(('', port)) - self.listen(5) - except: - raise Exception("Can't create socket") - def handle_accept(self): - newSocket, address = self.accept() - self.pycore.logger.info("Connected from " + str(address)) - SecondaryServerSocket(newSocket, self.pycore) - def handle_close(self): - print "going to close" - self.close() - - -class SecondaryServerSocket(asynchat.async_chat): - def __init__(self, socket, pycore): - asynchat.async_chat.__init__(self, socket) - self.pycore = pycore - self.handler = RequestHandler(pycore) - self.set_terminator('\n') - self.data = "" - def collect_incoming_data(self, data): - self.data += data - def found_terminator(self): - time.sleep(0.2) # if response is received to fast client gets an error O_o ?!?!? - rep = self.handler.proceed(self.data) - self.push(rep) - self.data = "" - #having fun with the data - def handle_close(self): - self.pycore.logger.info("Disconnected from "+ str(self.getpeername())) - self.close() - def push_obj(self, obj): - obj = self.handler.encrypt(obj) - self.push(obj) diff --git a/module/thread_list.py b/module/thread_list.py index d4f2dac97..da13afba6 100644 --- a/module/thread_list.py +++ b/module/thread_list.py @@ -60,7 +60,7 @@ class Thread_List(object): def get_job(self): """return job if suitable, otherwise send thread idle""" - if not self.parent.is_dltime() or self.pause or self.reconnecting or not self.list.files: #conditions when threads dont download + if not self.parent.is_time_download() or self.pause or self.reconnecting or not self.list.files: #conditions when threads dont download return None self.init_reconnect() -- cgit v1.2.3 From 293b957ce40bf3318f1c605887c2755ebd9b3e09 Mon Sep 17 00:00:00 2001 From: sitacuisses Date: Wed, 18 Nov 2009 00:47:28 +0100 Subject: user: sitacuisses branch 'default' added module/plugins/FilesmonsterCom.py --- module/plugins/FilesmonsterCom.py | 94 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 module/plugins/FilesmonsterCom.py (limited to 'module') diff --git a/module/plugins/FilesmonsterCom.py b/module/plugins/FilesmonsterCom.py new file mode 100644 index 000000000..27c687363 --- /dev/null +++ b/module/plugins/FilesmonsterCom.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Plugin for www.filesmonster.com +# this plugin isn't fully implemented yet,but it does download +# todo: +# detect, if reconnect is necessary +# download-error handling +# postpone download, if speed is below a set limit +# implement premium-access +# optional replace blanks in filename with underscores + +import re +import urllib +import time +from Plugin import Plugin + +class FilesmonsterCom(Plugin): + + def __init__(self, parent): + Plugin.__init__(self, parent) + props = {} + props['name'] = "FilesmonsterCom" + props['type'] = "hoster" + props['pattern'] = r"http://(www.)??filesmonster.com/download.php" + props['version'] = "0.1" + props['description'] = """Filesmonster.com Download Plugin""" + props['author_name'] = ("sitacuisses","spoob") + props['author_mail'] = ("sitacuisses@yahoo.de","spoob@pyload.org") + self.props = props + self.parent = parent + self.html = None + self.want_reconnect = False + self.multi_dl = False + self.htmlwithlink = None + self.url = None + self.filerequest = None + + def download_html(self): + self.url = self.parent.url + self.html = self.req.load(self.url) # get the start page + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + if self.html == None: + self.download_html() + if not self.want_reconnect: + self.get_download_page() # the complex work is done here + file_url = self.htmlwithlink + return file_url + else: + return False + + def get_file_name(self): + if self.html == None: + self.download_html() + if not self.want_reconnect: + file_name = re.search(r"File\sname:\s(.*?)", self.html).group(1) + return file_name + else: + return self.parent.url + + def file_exists(self): + """ returns True or False + """ + if self.html == None: + self.download_html() + if re.search(r"Such file does not exist or it has been removed for infringement of copyrights.", self.html) != None: + return False + else: + return True + + def get_download_page(self): + herewego = re.findall(r"\s*\n\s*", self.html) + the_download_page = self.req.load("http://filesmonster.com/get/free/", None, herewego) + temporary_filtered = re.search(r"\s*\n(\s*\s*\n)*?\s*", the_download_page).group(0) + all_the_tuples = re.findall(r" Date: Mon, 16 Nov 2009 18:56:44 +0100 Subject: Fixed DLC Plugin for new Plugin System --- module/plugins/DLC.pyc | Bin 3692 -> 2228 bytes module/plugins/UploadedTo.py | 3 +-- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'module') diff --git a/module/plugins/DLC.pyc b/module/plugins/DLC.pyc index 5d4809178..3adddec00 100644 Binary files a/module/plugins/DLC.pyc and b/module/plugins/DLC.pyc differ diff --git a/module/plugins/UploadedTo.py b/module/plugins/UploadedTo.py index bcb0a7008..4bddbd6f1 100644 --- a/module/plugins/UploadedTo.py +++ b/module/plugins/UploadedTo.py @@ -69,8 +69,7 @@ class UploadedTo(Plugin): tries += 1 if tries > 5: - raise Exception, "Error while preparing DL, HTML dump: %s" % self.html - + raise Exception, "Error while preparing DL" return True def download_api_data(self): -- cgit v1.2.3 From 8afd72d4a918108a081a6a94ccc72f4c2d275897 Mon Sep 17 00:00:00 2001 From: spoob Date: Mon, 16 Nov 2009 20:28:33 +0100 Subject: Fixed DLC Plugin --- module/plugins/DLC.pyc | Bin 2228 -> 2231 bytes 1 file changed, 0 insertions(+), 0 deletions(-) (limited to 'module') diff --git a/module/plugins/DLC.pyc b/module/plugins/DLC.pyc index 3adddec00..980dd8ace 100644 Binary files a/module/plugins/DLC.pyc and b/module/plugins/DLC.pyc differ -- cgit v1.2.3 From b391a5b9061d7d41bf1ae7df71f5a5e64b8491e8 Mon Sep 17 00:00:00 2001 From: mkaay Date: Sun, 22 Nov 2009 17:44:03 +0100 Subject: fixed SecureXMLRPCServer --- module/remote/SecureXMLRPCServer.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'module') diff --git a/module/remote/SecureXMLRPCServer.py b/module/remote/SecureXMLRPCServer.py index fb666b4ab..158e44863 100644 --- a/module/remote/SecureXMLRPCServer.py +++ b/module/remote/SecureXMLRPCServer.py @@ -75,6 +75,9 @@ class SecureXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): class SecureXMLRPCServer(SecureSocketServer, SimpleXMLRPCServer): def __init__(self, address, cert, key, authenticationMap = None, handler=SecureXMLRPCRequestHandler, verify_cert_func=None): self.logRequests = False + self._send_traceback_header = False + self.encoding = "utf-8" + self.allow_none = True SecureSocketServer.__init__(self, address, cert, key, handler, verify_cert_func) # This comes from SimpleXMLRPCServer.__init__()->SimpleXMLRPCDispatcher.__init__() self.funcs = {} -- cgit v1.2.3 From 419a93755b6180bbab69aaf973cd99f49b4a2268 Mon Sep 17 00:00:00 2001 From: spoob Date: Sun, 22 Nov 2009 18:51:14 +0100 Subject: Webinterface with XMLRPC --- module/web/WebServer.py | 354 ++++++++++-------------------------------------- module/web/main.html | 29 ++++ module/web/main.js | 42 ++++++ 3 files changed, 139 insertions(+), 286 deletions(-) create mode 100644 module/web/main.html create mode 100644 module/web/main.js (limited to 'module') diff --git a/module/web/WebServer.py b/module/web/WebServer.py index 8cef0de2f..0712f1dce 100644 --- a/module/web/WebServer.py +++ b/module/web/WebServer.py @@ -1,290 +1,72 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2009 RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -### +import sys +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from xmlrpclib import ServerProxy +from time import time +import re -import random -import threading -import time - -import bottle -from bottle import abort -from bottle import redirect -from bottle import request -from bottle import response -from bottle import route -from bottle import run -from bottle import send_file -from bottle import template -from bottle import validate - - -core = None - -PATH = "./module/web/" -TIME = time.strftime("%a, %d %b %Y 00:00:00 +0000", time.localtime()) #set time to current day -USERS = {} - -@route('/login', method='POST') -def do_login(): - #print request.GET - - - username = core.config['webinterface']['username'] - pw = core.config['webinterface']['password'] - - if request.POST['u'] == username and request.POST['p'] == pw: - - id = int(random.getrandbits(16)) - ua = request.HEADER("HTTP_USER_AGENT") - ip = request.HEADER("REMOTE_ADDR") - - auth = {} - - auth['ua'] = ua - auth['ip'] = ip - auth['user'] = username - - USERS[id] = auth - - response.COOKIES['user'] = username - response.COOKIES['id'] = id - - return template('default', page='loggedin', user=username) - else: - return template('default', page='login') - -@route('/login') -def login(): - - if check_auth(request): - redirect("/") - - return template('default', page='login') - -@route('/logout') -def logout(): - try: - del USERS[int(request.COOKIES.get('id'))] - except: - pass +class Handler(BaseHTTPRequestHandler): - redirect("/login") - -@route('/') -def home(): - - if not check_auth(request): - redirect("/login") - - username = request.COOKIES.get('user') - - dls = core.get_downloads() - - for dl in dls: - dl['eta'] = str(core.format_time(dl['eta'])) - dl['wait_until'] = str(core.format_time(dl['wait_until'] - time.time())) - - - return template('default', page='home', links=dls, user=username, status=core.server_status()) - -@route('/queue') -def queue(): - - if not check_auth(request): - redirect("/login") - - username = request.COOKIES.get('user') - - return template('default', page='queue', links=core.get_links(), user=username, status=core.server_status()) - -@route('/downloads') -def downloads(): - - if not check_auth(request): - redirect("/login") - - username = request.COOKIES.get('user') - - return template('default', page='downloads', links=core.get_downloads(), user=username, status=core.server_status()) - - -@route('/logs') -def logs(): - - if not check_auth(request): - redirect("/login") - - username = request.COOKIES.get('user') - - return template('default', page='logs', links=core.get_downloads(), user=username, status=core.server_status()) - -@route('/json/links') -def get_links(): - response.header['Cache-Control'] = 'no-cache, must-revalidate' - response.content_type = 'application/json' - - if not check_auth(request): - abort(404, "No Access") - - json = '{ "downloads": [' - - downloads = core.get_downloads() - ids = [] - - for dl in downloads: - ids.append(dl['id']) - json += '{' - json += '"id": %s, "name": "%s", "speed": %s, "eta": "%s", "kbleft": %s, "size": %s, "percent": %s, "wait": "%s", "status": "%s"'\ - % (str(dl['id']), str(dl['name']), str(int(dl['speed'])), str(core.format_time(dl['eta'])), dl['kbleft'], dl['size'], dl['percent'], str(core.format_time(dl['wait_until'] - time.time())), dl['status']) - - json += "}," - - if json.endswith(","): json = json[:-1] - - json += '], "ids": %s }' % str(ids) - - return json - -@route('/json/status') -def get_status(): - response.header['Cache-Control'] = 'no-cache, must-revalidate' - response.content_type = 'application/json' - - if not check_auth(request): - abort(404, "No Access") - - data = core.server_status() - - if data['pause']: - status = "paused" - else: - status = "running" - - json = '{ "status": "%s", "speed": "%s", "queue": "%s" }' % (status, str(int(data['speed'])), str(data['queue'])) - - return json - -@route('/json/addlinks', method='POST') -def add_links(): - response.header['Cache-Control'] = 'no-cache, must-revalidate' - response.content_type = 'application/json' - - if not check_auth(request): - abort(404, "No Access") - - links = request.POST['links'].split('\n') - - core.add_links(links) - - return "{}" - -@route('/json/pause') -def pause(): - response.header['Cache-Control'] = 'no-cache, must-revalidate' - response.content_type = 'application/json' - - if not check_auth(request): - abort(404, "No Access") - - core.thread_list.pause = True - - return "{}" - - -@route('/json/play') -def play(): - response.header['Cache-Control'] = 'no-cache, must-revalidate' - response.content_type = 'application/json' - - if not check_auth(request): - abort(404, "No Access") - - core.thread_list.pause = False - - return "{}" - -@route('/favicon.ico') -def favicon(): - - if request.HEADER("HTTP_IF_MODIFIED_SINCE") == TIME: abort(304, "Not Modified") - - response.header['Last-Modified'] = TIME - - send_file('favicon.ico', root=(PATH + 'static/')) - -@route('static/:section/:filename') -def static_folder(section, filename): - - if request.HEADER("HTTP_IF_MODIFIED_SINCE") == TIME: abort(304, "Not Modified") - - response.header['Last-Modified'] = TIME - send_file(filename, root=(PATH + 'static/' + section)) - -@route('/static/:filename') -def static_file(filename): - - if request.HEADER("HTTP_IF_MODIFIED_SINCE") == TIME: abort(304, "Not Modified") - - response.header['Last-Modified'] = TIME - send_file(filename, root=(PATH + 'static/')) - - -def check_auth(req): - - try: - user = req.COOKIES.get('user') - id = int(req.COOKIES.get('id')) - ua = req.HEADER("HTTP_USER_AGENT") - ip = req.HEADER("REMOTE_ADDR") - - if USERS[id]['user'] == user and USERS[id]['ua'] == ua and USERS[id]['ip'] == ip: - return True - except: - return False - - return False - - -class WebServer(threading.Thread): - def __init__(self, pycore): - threading.Thread.__init__(self) - - global core, TIME - core = pycore - self.core = pycore - self.setDaemon(True) - - if pycore.config['general']['debug_mode']: - bottle.debug(True) - TIME = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.localtime()) + def do_GET(self): + global coreserver + stdout = sys.stdout + sys.stdout = self.wfile + if self.path == "/": + print "Server Runs" + elif self.path == "/downloads": + print self.get_downloads() + elif re.search("/add=.?", self.path): + if re.match(is_url, self.path.split("/add=")[1]): + coreserver.add_urls([self.path.split("/add=")[1]]) + print "Link Added" else: - bottle.debug(False) - - #@TODO remove - #TIME = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.localtime()) - - bottle.TEMPLATE_PATH.append('./module/web/templates/%s.tpl') - - def run(self): - self.core.logger.info("Starting Webinterface on %s port %s" % (self.core.config['webinterface']['listenaddr'],self.core.config['webinterface']['port'])) + try: + print open(self.path[1:], 'r').read() + except IOError: + self.send_error(404) + + def format_size(self, size): + return str(size / 1024) + " MiB" + + def format_time(self,seconds): + seconds = int(seconds) + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + + def get_downloads(self): + data = coreserver.status_downloads() + for download in data: + print "

%s

" % download["name"] + if download["status"] == "downloading": + percent = download["percent"] + z = percent / 4 + print "

%s

" % dl_name + print "[" + z * "#" + (25-z) * " " + "]" + str(percent) + "%
" + print "Speed: " + str(int(download['speed'])) + " kb/s" + print "Size: " + self.format_size(download['size']) + print "Finished in: " + self.format_time(download['eta']) + print "ID: " + str(download['id']) + dl_status = "[" + z * "#" + (25-z) * " " + "] " + str(percent) + "%" + " Speed: " + str(int(download['speed'])) + " kb/s" + " Size: " + self.format_size(download['size']) + " Finished in: " + self.format_time(download['eta']) + " ID: " + str(download['id']) + if download["status"] == "waiting": + print "waiting: " + self.format_time(download["wait_until"]- time()) + +is_url = re.compile("^(((https?|ftp)\:\/\/)?([\w\.\-]+(\:[\w\.\&%\$\-]+)*@)?((([^\s\(\)\<\>\\\"\.\[\]\,@;:]+)(\.[^\s\(\)\<\>\\\"\.\[\]\,@;:]+)*(\.[a-zA-Z]{2,4}))|((([01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}([01]?\d{1,2}|2[0-4]\d|25[0-5])))(\b\:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3}|0)\b)?((\/[^\/][\w\.\,\?\'\\\/\+&%\$#\=~_\-@]*)*[^\.\,\?\"\'\(\)\[\]!;<>{}\s\x7F-\xFF])?)$",re.IGNORECASE) + +coreserver = None + +class WebServer(): + + def start(self): try: - run(host=self.core.config['webinterface']['listenaddr'], port=int(self.core.config['webinterface']['port']), quiet=True) - except: - self.core.logger.error("Failed starting webserver, no webinterface available: Can't create socket") - exit() + global coreserver + coreserver = ServerProxy("https://testuser:testpw@localhost:1337", allow_none=True) + webserver = HTTPServer(('',8080),Handler) + print 'server started at port 8080' + webserver.serve_forever() + except KeyboardInterrupt: + webserver.socket.close() + +if __name__ == "__main__": + web = WebServer() + web.start() + diff --git a/module/web/main.html b/module/web/main.html new file mode 100644 index 000000000..87f0d7408 --- /dev/null +++ b/module/web/main.html @@ -0,0 +1,29 @@ +pyLoad - Webinterface + + + + + pyLoad - Webinterface + + + +

pyLoad - Webinterface

+
+ Status: running Speed: 600kb/s Files in queue: 0 +
+
+
+
+ + +
+
+ (START) (PAUSE) (ADD) +
+

Downloads

+
+ Lade Downloads +
+ + + diff --git a/module/web/main.js b/module/web/main.js new file mode 100644 index 000000000..a286df991 --- /dev/null +++ b/module/web/main.js @@ -0,0 +1,42 @@ +function getXmlHttpRequestObject() { + if (window.XMLHttpRequest) { + return new XMLHttpRequest(); //Not IE + } else if(window.ActiveXObject) { + return new ActiveXObject("Microsoft.XMLHTTP"); //IE + } else { + alert("Your browser doesn't support the XmlHttpRequest object. Better upgrade to Firefox."); + } +} +var req = getXmlHttpRequestObject(); + +function getDownloads() { + req.onreadystatechange = function() { + if (req.readyState == 4) { + if(req.status==200) { + document.getElementById('downloads').innerHTML = req.responseText; + } else { + alert("Fehler:\nHTTP-Status: "+req.status+"\nHTTP-Statustext: "+req.statusText); + } + }; + } + req.open("GET", '/downloads', true); + req.send(null); +} + +function addUrl(new_url) { + req.onreadystatechange = function() { + if (req.readyState == 4) { + if(req.status==200) { + document.getElementById('add_urls').innerHTML = req.responseText; + } else { + alert("Fehler:\nHTTP-Status: "+req.status+"\nHTTP-Statustext: "+req.statusText); + } + }; + } + url = "/add=" + new_url + req.open("GET", url, true); + req.send(null); +} + +window.setInterval("getDownloads()", 1000); + -- cgit v1.2.3 From 523e2857c47cdef1da6b43523bcf7871ed9e1d63 Mon Sep 17 00:00:00 2001 From: mkaay Date: Thu, 26 Nov 2009 22:05:11 +0100 Subject: complete new file_list, cleaned up --- module/file_list.py | 379 ++++++++++++++++++++++++++-------------- module/remote/ClientHandler.py | 24 --- module/remote/ClientSocket.py | 63 ------- module/remote/RequestHandler.py | 64 ------- module/remote/RequestObject.py | 18 -- module/thread_list.py | 25 +-- 6 files changed, 257 insertions(+), 316 deletions(-) delete mode 100644 module/remote/ClientHandler.py delete mode 100644 module/remote/ClientSocket.py delete mode 100644 module/remote/RequestHandler.py delete mode 100644 module/remote/RequestObject.py (limited to 'module') diff --git a/module/file_list.py b/module/file_list.py index 1701b801b..6e1984704 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -1,159 +1,274 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# -#Copyright (C) 2009 RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -### -LIST_VERSION = 3 +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay + @author: RaNaN + @version: v0.3 + @list-version: v4 +""" + +LIST_VERSION = 4 from threading import RLock from download_thread import Status import cPickle import re -from module.remote.RequestObject import RequestObject import module.Plugin +class NoSuchElementException(Exception): + pass + class File_List(object): def __init__(self, core): self.core = core - self.files = [] - self.data = {'version': LIST_VERSION, 'order': []} self.lock = RLock() - self.load() - - def new_pyfile(self, url, folder): - url = url.replace("\n", "") - pyfile = PyLoadFile(self.core, url) - pyfile.download_folder = self.core.config['general']['download_folder'] - pyfile.id = self.get_id() - pyfile.folder = folder - - return pyfile - - def append(self, url, folder=""): - if not url: - return False - #@TODO: filter non existence and invalid links - #re.compile("https?://[-a-z0-9\.]{4,}(?::\d+)?/[^#?]+(?:#\S+)?",re.IGNORECASE) - new_file = self.new_pyfile(url, folder) - self.files.append(new_file) - self.data[new_file.id] = Data(url, folder) - self.data['order'].append(int(new_file.id)) - - def extend(self, urls): - for url in urls: - self.append(url) - - def remove(self, pyfile): - if not self.core.config['general']['debug_mode']: - if pyfile in self.files: - self.files.remove(pyfile) - - self.data['order'].remove(pyfile.id) - del self.data[pyfile.id] - - def remove_id(self, pyid): - #also abort download - pyid = int(pyid) - found = False - for pyfile in self.files: - if pyfile.id == pyid: - self.files.remove(pyfile) - found = True - break - - if not found: - for pyfile in self.core.thread_list.py_downloading: - if pyfile.id == pyid: - pyfile.plugin.req.abort = True - break - return False - - self.data['order'].remove(pyid) - del self.data[pyid] - - def get_id(self): - """return a free id""" - id = 1 - while id in self.data.keys(): - id += 1 - - return id - - def move(self, id, offset=-1): - - for pyfile in self.files: - if pyfile.id == id: - index = self.files.index(pyfile) - pyfile = self.files.pop(index) - self.files.insert(index + offset, pyfile) - break - - index = self.data['order'].index(id) - pyfile = self.data['order'].pop(index) - self.data['order'].insert(index + offset, pyfile) - - def save(self): - self.lock.acquire() - - output = open('links.pkl', 'wb') - cPickle.dump(self.data, output, -1) - - #self.inform_client() + self.download_folder = self.core.config['general']['download_folder'] + self.collector = self.pyLoadCollector(self) + self.packager = self.pyLoadPackager(self) + + self.data = { + "version": LIST_VERSION, + "queue": [], + "packages": [], + "collector": [] + } - self.lock.release() - def load(self): + self.lock.acquire() try: pkl_file = open('links.pkl', 'rb') obj = cPickle.load(pkl_file) except: - obj = {'version': LIST_VERSION, 'order': []} - - if obj['version'] < LIST_VERSION: - obj = {'version': LIST_VERSION, 'order': []} - - for i in obj['order']: - self.append(obj[i].url, obj[i].folder) - - self.core.logger.info("Links loaded: " + str(int(len(obj) - 2))) - - def inform_client(self): - obj = RequestObject() - obj.command = "file_list" - obj.data = self.data + obj = False + if obj['version'] == LIST_VERSION and obj: + self.data = obj + self.lock.release() + + if len(self.data["collector"]) > 0: + self.core.logger.info("Found %s links in linkcollector" % len(self.data["collector"])) + if len(self.data["packages"]) > 0: + self.core.logger.info("Found %s unqueued packages" % len(self.data["packages"])) + if len(self.data["queue"]) > 0: + self.core.logger.info("Added %s packages to queue" % len(self.data["queue"])) + + def save(self): + self.lock.acquire() - self.core.server.push_all(obj) + output = open('links.pkl', 'wb') + cPickle.dump(self.data, output, -1) + + self.lock.release() + + def queueEmpty(self): + return (self.data["queue"] == []) + + def getDownloadList(self): + """ + for thread_list only + """ + files = [] + for pypack in self.data["queue"]: + for pyfile in pypack.files: + if pyfile.status.type == "reconnected" or pyfile.status.type == None: + files.append(pyfile) + return files + + class pyLoadCollector(): + def __init__(collector, file_list): + collector.file_list = file_list + + def _getFileFromID(collector, id): + """ + returns PyLoadFile instance and position in collector with given id + """ + for n, pyfile in enumerate(collector.file_list.data["collector"]): + if pyfile.id == id: + return (n, pyfile) + raise NoSuchElementException() + + def _getFreeID(collector): + """ + returns a free id + """ + ids = [] + for pyfile in collector.file_list.data["collector"]: + ids.append(pyfile.id) + id = 1 + while id in ids: + id += 1 + return id + + def getFile(collector, id): + """ + returns PyLoadFile instance from given id + """ + return collector._getFileFromID(id)[1] + + def popFile(collector, id): + """ + returns PyLoadFile instance given id and remove it from the collector + """ + collector.file_list.lock.acquire() + try: + n, pyfile = collector._getFileFromID(id) + del collector.file_list.data["collector"][n] + collector.file_list.lock.release() + except: + collector.file_list.lock.release() + else: + return pyfile + + def addLink(collector, url): + """ + appends a new PyLoadFile instance to the end of the collector + """ + pyfile = PyLoadFile(url) + pyfile.id = collector._getFreeID() + pyfile.download_folder = collector.file_list.download_folder + collector.file_list.lock.acquire() + collector.file_list.data["collector"].append(pyfile) + collector.file_list.lock.release() + return pyfile.id + + def removeFile(collector, id): + """ + removes PyLoadFile instance with the given id from collector + """ + collector.popFile(id) + + def replaceFile(collector, newpyfile): + """ + replaces PyLoadFile instance with the given PyLoadFile instance at the given id + """ + collector.file_list.lock.acquire() + try: + n, pyfile = collector._getFileFromID(newpyfile.id) + collector.file_list.data["collector"][n] = newpyfile + finally: + collector.file_list.lock.release() + + class pyLoadPackager(): + def __init__(packager, file_list): + packager.file_list = file_list + + def _getFreeID(packager): + """ + returns a free id + """ + ids = [] + for pypack in (packager.file_list.data["packages"] + packager.file_list.data["queue"]): + ids.append(pypack.id) + id = 1 + while id in ids: + id += 1 + return id + + def _getPackageFromID(packager, id): + """ + returns PyLoadPackage instance and position with given id + """ + for n, pypack in enumerate(packager.file_list.data["packages"]): + if pypack.id == id: + return ("packages", n, pypack) + for n, pypack in enumerate(packager.file_list.data["queue"]): + if pypack.id == id: + return ("queue", n, pypack) + raise NoSuchElementException() + + def addNewPackage(packager, package_name=None): + pypack = PyLoadPackage() + pypack.id = packager._getFreeID() + if package_name is not None: + pypack.data["package_name"] = package_name + packager.file_list.data["packages"].append(pypack) + return pypack.id + + def removePackage(packager, id): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + del packager.file_list.data[key][n] + finally: + packager.file_list.lock.release() + + def pushPackage2Queue(packager, id): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + if key == "packages": + del packager.file_list.data["packages"][n] + packager.file_list.data["queue"].append(pypack) + finally: + packager.file_list.lock.release() + + def pullOutPackage(packager, id): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + if key == "queue": + del packager.file_list.data["queue"][n] + packager.file_list.data["packages"].append(pypack) + finally: + packager.file_list.lock.release() + + def setPackageData(packager, id, package_name=None, folder=None): + packager.file_list.lock.acquire() + try: + key, n, pypack = packager._getPackageFromID(id) + if package_name is not None: + pypack.data["package_name"] = package_name + if folder is not None: + pypack.data["folder"] = folder + packager.file_list.data[key][n] = pypack + finally: + packager.file_list.lock.release() + + def addFileToPackage(packager, id, pyfile): + key, n, pypack = packager._getPackageFromID(id) + pyfile.package = pypack + pypack.files.append(pyfile) + packager.file_list.data[key][n] = pypack + + def removeFileFromPackage(packager, id, pid): + key, n, pypack = packager._getPackageFromID(pid) + for k, pyfile in enumerate(pypack.files): + if id == pyfile.id: + del pypack.files[k] + return True + raise NoSuchElementException() -class Data(): - def __init__(self, url, folder=""): - self.url = url - self.folder = folder +class PyLoadPackage(): + def __init__(self): + self.files = [] + self.data = { + "id": None, + "package_name": "", + "folder": "" + } -class PyLoadFile: - """ represents the url or file - """ - def __init__(self, parent, url): - self.parent = parent +class PyLoadFile(): + def __init__(self, url): self.id = None self.url = url self.folder = None + self.package = None self.filename = "filename" self.download_folder = "" + self.active = False pluginName = self._get_my_plugin() if pluginName: self.modul = __import__(pluginName) @@ -163,18 +278,12 @@ class PyLoadFile: pluginClass = module.Plugin.Plugin self.plugin = pluginClass(self) self.status = Status(self) + def _get_my_plugin(self): - - """ searches the right plugin for an url - """ for plugin, plugin_pattern in self.parent.plugins_avaible.items(): if re.match(plugin_pattern, self.url) != None: return plugin def init_download(self): - if self.parent.config['proxy']['activated']: self.plugin.req.add_proxy(self.parent.config['proxy']['protocol'], self.parent.config['proxy']['adress']) - - #@TODO: check dependicies, ocr etc - diff --git a/module/remote/ClientHandler.py b/module/remote/ClientHandler.py deleted file mode 100644 index 9653db9ae..000000000 --- a/module/remote/ClientHandler.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -authored by: RaNaN - -this module handels the incoming requests - -""" -import hashlib - -from Crypto.Cipher import Blowfish -from RequestHandler import RequestHandler - -class ClientHandler(RequestHandler): - def __init__(self, client, pw): - self.client = client - key = hashlib.sha256(pw) - self.bf = Blowfish.new(key.hexdigest(), Blowfish.MODE_ECB) - - def proceed(self, data): - obj = self.decrypt(data) - - self.client.data_arrived(obj) \ No newline at end of file diff --git a/module/remote/ClientSocket.py b/module/remote/ClientSocket.py deleted file mode 100644 index 0e6a5ed53..000000000 --- a/module/remote/ClientSocket.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -authored by: RaNaN - -socket for connecting to the core's server - -""" -import asynchat -import asyncore -import socket -import threading - -from ClientHandler import ClientHandler -from RequestObject import RequestObject - -class SocketThread(threading.Thread): - def __init__(self, adress, port, pw, client): - threading.Thread.__init__(self) - self.setDaemon(True) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((adress, port)) - self.socket = ClientSocket(sock, pw, client) - self.start() - - def run(self): - asyncore.loop() - print "loop closed" - - def push_exec(self, function, args=[]): - obj = RequestObject() - obj.command = "exec" - obj.function = function - obj.args = args - self.push(obj) - - def push(self, obj): - self.socket.push_obj(obj) - - -class ClientSocket(asynchat.async_chat): - def __init__(self, sock, pw, client): - asynchat.async_chat.__init__(self, sock) - self.data = "" - self.handler = ClientHandler(client, pw) - self.set_terminator("\n") - #self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - - def handle_close(self): - print "Disconnected from", self.getpeername() - self.close() - - def collect_incoming_data(self, data): - self.data += data - - def found_terminator(self): - self.handler.proceed(self.data) - self.data = "" - - def push_obj(self, obj): - string = self.handler.encrypt(obj) - self.push(string) \ No newline at end of file diff --git a/module/remote/RequestHandler.py b/module/remote/RequestHandler.py deleted file mode 100644 index a879d2297..000000000 --- a/module/remote/RequestHandler.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -authored by: RaNaN - -this module handels the incoming requests - -""" - -import base64 -import hashlib -import random -import string - -import cPickle -from Crypto.Cipher import Blowfish -from RequestObject import RequestObject - - -class RequestHandler: - def __init__(self, core): - self.core = core - key = hashlib.sha256(core.config['remote']['password']) - self.bf = Blowfish.new(key.hexdigest(), Blowfish.MODE_ECB) - - def proceed(self, data): - obj = self.decrypt(data) - - if obj.command == "exec": - func = getattr(self.core, obj.function) - obj.data = func( * obj.args) - else: - obj.data = "error happend" - - return self.encrypt(obj) - - - def decrypt(self, dec_str): - try: - dec_str = base64.standard_b64decode(dec_str) - dec_str = self.bf.decrypt(dec_str) - - dec_str = dec_str[:-(int(dec_str[-1], 16) + 1)] - obj = cPickle.loads(dec_str) - except: - obj = RequestObject() - - return obj - - def encrypt(self, obj): - enc_str = cPickle.dumps(obj, 1) - padding = len(enc_str) % 16 - padding = 16 - padding - - p_str = "" - for i in range(padding - 1): - p_str += random.choice(string.letters + string.digits + string.punctuation) - p_str += hex(len(p_str)).replace("0x", "") - enc_str += p_str - - enc_str = self.bf.encrypt(enc_str) - enc_str = base64.standard_b64encode(enc_str) - return enc_str + "\n" diff --git a/module/remote/RequestObject.py b/module/remote/RequestObject.py deleted file mode 100644 index 54ea837a6..000000000 --- a/module/remote/RequestObject.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -authored by: RaNaN - -represents the object for interaction - -""" -class RequestObject(object): - def __init__(self): - self.version = 0 - self.sender = "ip" - self.status = {} - self.command = None - self.function = "" - self.args = [] - self.data = None diff --git a/module/thread_list.py b/module/thread_list.py index da13afba6..aa50a654f 100644 --- a/module/thread_list.py +++ b/module/thread_list.py @@ -60,7 +60,7 @@ class Thread_List(object): def get_job(self): """return job if suitable, otherwise send thread idle""" - if not self.parent.is_time_download() or self.pause or self.reconnecting or not self.list.files: #conditions when threads dont download + if not self.parent.is_time_download() or self.pause or self.reconnecting or self.list.queueEmpty(): #conditions when threads dont download return None self.init_reconnect() @@ -68,15 +68,16 @@ class Thread_List(object): self.lock.acquire() pyfile = None - for i in range(len(self.list.files)): - if not self.list.files[i].modul.__name__ in self.occ_plugins: - pyfile = self.list.files.pop(i) + for f in self.list.getDownloadList(): + if not f.modul.__name__ in self.occ_plugins: + pyfile = f break if pyfile: self.py_downloading.append(pyfile) if not pyfile.plugin.multi_dl: self.occ_plugins.append(pyfile.modul.__name__) + pyfile.active = True self.parent.logger.info('Download starts: ' + pyfile.url) self.lock.release() @@ -89,6 +90,8 @@ class Thread_List(object): if not pyfile.plugin.multi_dl: self.occ_plugins.remove(pyfile.modul.__name__) + + pyfile.active = False if pyfile.plugin.req.curl and not pyfile.status == "reconnected": try: @@ -101,25 +104,23 @@ class Thread_List(object): if pyfile.status.type == "finished": self.parent.logger.info('Download finished: ' + pyfile.url + ' @' + str(pyfile.status.get_speed()) + 'kb/s') - self.list.remove(pyfile) - if pyfile.plugin.props['type'] == "container": - self.list.extend(pyfile.plugin.links) - + self.list.packager.removeFileFromPackage(pyfile.id, pyfile.package.id) + for link in pyfile.plugin.links: + id = self.list.collector.addLink(link) + pyfile.packager.pullOutPackage(pyfile.package.id) + pyfile.packager.addFileToPackage(pyfile.package.id, pyfile.collector.popFile(id)) - elif pyfile.status.type == "reconnected":#put it back in queque + elif pyfile.status.type == "reconnected": pyfile.plugin.req.init_curl() - self.list.files.insert(0, pyfile) elif pyfile.status.type == "failed": self.parent.logger.warning("Download failed: " + pyfile.url+ " | " + pyfile.status.error) with open(self.parent.config['general']['failed_file'], 'a') as f: f.write(pyfile.url + "\n") - self.list.remove(pyfile) elif pyfile.status.type == "aborted": self.parent.logger.info("Download aborted: " + pyfile.url) - self.list.remove(pyfile) self.list.save() -- cgit v1.2.3 From 80cd07afb67746a0279d6f194102e0d52bdbba29 Mon Sep 17 00:00:00 2001 From: mkaay Date: Fri, 27 Nov 2009 16:55:55 +0100 Subject: fixed file_list --- module/file_list.py | 124 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 113 insertions(+), 11 deletions(-) (limited to 'module') diff --git a/module/file_list.py b/module/file_list.py index 6e1984704..f3bfc0b8a 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -28,6 +28,7 @@ from download_thread import Status import cPickle import re import module.Plugin +import traceback class NoSuchElementException(Exception): pass @@ -46,6 +47,7 @@ class File_List(object): "packages": [], "collector": [] } + self.load() def load(self): self.lock.acquire() @@ -54,7 +56,26 @@ class File_List(object): obj = cPickle.load(pkl_file) except: obj = False - if obj['version'] == LIST_VERSION and obj: + traceback.print_exc() + if obj['version'] == LIST_VERSION and obj != False: + packages = [] + queue = [] + collector = [] + for n, pd in enumerate(obj["packages"]): + p = PyLoadPackage() + pd.get(p) + packages.append(p) + for pd in obj["queue"]: + p = PyLoadPackage() + pd.get(p) + queue.append(p) + for fd in obj["collector"]: + f = PyLoadFile("", self) + fd.get(f) + collector.append(f) + obj["packages"] = packages + obj["queue"] = queue + obj["collector"] = collector self.data = obj self.lock.release() @@ -67,9 +88,34 @@ class File_List(object): def save(self): self.lock.acquire() - + + pdata = { + "version": LIST_VERSION, + "queue": [], + "packages": [], + "collector": [] + } + packages = [] + queue = [] + collector = [] + for p in self.data["packages"]: + pd = PyLoadPackageData() + pd.set(p) + packages.append(pd) + for p in self.data["queue"]: + pd = PyLoadPackageData() + pd.set(p) + queue.append(pd) + for f in self.data["collector"]: + fd = PyLoadFileData() + fd.set(f) + collector.append(fd) + pdata["packages"] = packages + pdata["queue"] = queue + pdata["collector"] = collector + output = open('links.pkl', 'wb') - cPickle.dump(self.data, output, -1) + cPickle.dump(pdata, output, -1) self.lock.release() @@ -83,7 +129,7 @@ class File_List(object): files = [] for pypack in self.data["queue"]: for pyfile in pypack.files: - if pyfile.status.type == "reconnected" or pyfile.status.type == None: + if (pyfile.status.type == "reconnected" or pyfile.status.type == None) and not pyfile.active: files.append(pyfile) return files @@ -136,7 +182,7 @@ class File_List(object): """ appends a new PyLoadFile instance to the end of the collector """ - pyfile = PyLoadFile(url) + pyfile = PyLoadFile(url, collector.file_list) pyfile.id = collector._getFreeID() pyfile.download_folder = collector.file_list.download_folder collector.file_list.lock.acquire() @@ -256,15 +302,17 @@ class PyLoadPackage(): self.files = [] self.data = { "id": None, - "package_name": "", - "folder": "" + "package_name": "new_package", + "folder": None } class PyLoadFile(): - def __init__(self, url): + def __init__(self, url, file_list): self.id = None self.url = url self.folder = None + self.file_list = file_list + self.core = file_list.core self.package = None self.filename = "filename" self.download_folder = "" @@ -280,10 +328,64 @@ class PyLoadFile(): self.status = Status(self) def _get_my_plugin(self): - for plugin, plugin_pattern in self.parent.plugins_avaible.items(): + for plugin, plugin_pattern in self.core.plugins_avaible.items(): if re.match(plugin_pattern, self.url) != None: return plugin def init_download(self): - if self.parent.config['proxy']['activated']: - self.plugin.req.add_proxy(self.parent.config['proxy']['protocol'], self.parent.config['proxy']['adress']) + if self.core.config['proxy']['activated']: + self.plugin.req.add_proxy(self.core.config['proxy']['protocol'], self.core.config['proxy']['adress']) + +class PyLoadFileData(): + def __init__(self): + self.id = None + self.url = None + self.folder = None + self.pack_id = None + self.filename = None + self.status_type = None + self.status_url = None + + def set(self, pyfile): + self.id = pyfile.id + self.url = pyfile.url + self.folder = pyfile.folder + self.parsePackage(pyfile.package) + self.filename = pyfile.filename + self.status_type = pyfile.status.type + self.status_url = pyfile.status.url + self.status_filename = pyfile.status.filename + self.status_error = pyfile.status.error + + def get(self, pyfile): + pyfile.id = self.id + pyfile.url = self.url + pyfile.folder = self.folder + pyfile.filename = self.filename + pyfile.status.type = self.status_type + pyfile.status.url = self.status_url + pyfile.status.filename = self.status_filename + pyfile.status.error = self.status_error + + def parsePackage(self, pack): + if pack: + self.pack_id = pack.id + +class PyLoadPackageData(): + def __init__(self): + self.data = None + self.links = [] + + def set(self, pypack): + self.data = pypack.data + for pyfile in pypack.links: + fdata = PyLoadFileData() + fdata.set(pyfile) + self.links.append(fdata) + + def get(self, pypack): + for fdata in self.links: + pyfile = PyLoadFile() + fdata.get(pyfile) + pyfile.package = pypack + pypack.links.append(pyfile) -- cgit v1.2.3 From 7fb23f31ac129442f35d2d98a5ef3f0b387dc8e2 Mon Sep 17 00:00:00 2001 From: mkaay Date: Fri, 27 Nov 2009 23:51:44 +0100 Subject: updated xmlrpc methods --- module/file_list.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'module') diff --git a/module/file_list.py b/module/file_list.py index f3bfc0b8a..1bcfb8580 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -151,6 +151,9 @@ class File_List(object): returns a free id """ ids = [] + for pypack in (packager.file_list.data["packages"] + packager.file_list.data["queue"]): + for pyf in pypack.links: + ids.append(pyf.id) for pyfile in collector.file_list.data["collector"]: ids.append(pyfile.id) id = 1 @@ -235,6 +238,20 @@ class File_List(object): return ("queue", n, pypack) raise NoSuchElementException() + def _getFileFromID(packager, id): + """ + returns PyLoadFile instance and position with given id + """ + for n, pypack in enumerate(packager.file_list.data["packages"]): + for pyfile in pypack.files: + if pyfile.id == id: + return ("packages", n, pyfile, pypack, pid) + for n, pypack in enumerate(packager.file_list.data["queue"]): + for pyfile in pypack.files: + if pyfile.id == id: + return ("queue", n, pyfile, pypack, pid) + raise NoSuchElementException() + def addNewPackage(packager, package_name=None): pypack = PyLoadPackage() pypack.id = packager._getFreeID() @@ -251,6 +268,17 @@ class File_List(object): finally: packager.file_list.lock.release() + def removeFile(packager, id): + """ + removes PyLoadFile instance with the given id from package + """ + packager.file_list.lock.acquire() + try: + key, n, pyfile, pypack, pid = self._getFileFromID() + del pypack.files[n] + finally: + packager.file_list.lock.release() + def pushPackage2Queue(packager, id): packager.file_list.lock.acquire() try: -- cgit v1.2.3 From 54b787dd2a35b51952fdcb26df51cb18a0c97060 Mon Sep 17 00:00:00 2001 From: spoob Date: Sat, 28 Nov 2009 14:17:28 +0100 Subject: cleaned and fixed --- module/file_list.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'module') diff --git a/module/file_list.py b/module/file_list.py index 1bcfb8580..7b886cc2c 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -28,7 +28,6 @@ from download_thread import Status import cPickle import re import module.Plugin -import traceback class NoSuchElementException(Exception): pass @@ -56,8 +55,7 @@ class File_List(object): obj = cPickle.load(pkl_file) except: obj = False - traceback.print_exc() - if obj['version'] == LIST_VERSION and obj != False: + if obj != False and obj['version'] == LIST_VERSION: packages = [] queue = [] collector = [] @@ -151,7 +149,7 @@ class File_List(object): returns a free id """ ids = [] - for pypack in (packager.file_list.data["packages"] + packager.file_list.data["queue"]): + for pypack in (collector.file_list.data["packages"] + collector.file_list.data["queue"]): for pyf in pypack.links: ids.append(pyf.id) for pyfile in collector.file_list.data["collector"]: -- cgit v1.2.3 From 8effa6e3712a11104835aa031242b39a29f291a0 Mon Sep 17 00:00:00 2001 From: spoob Date: Sun, 29 Nov 2009 15:18:18 +0100 Subject: links.pkl now in module, nicer terminal kill --- module/file_list.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module') diff --git a/module/file_list.py b/module/file_list.py index 7b886cc2c..c95eefcec 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -28,6 +28,7 @@ from download_thread import Status import cPickle import re import module.Plugin +from os import sep class NoSuchElementException(Exception): pass @@ -51,7 +52,7 @@ class File_List(object): def load(self): self.lock.acquire() try: - pkl_file = open('links.pkl', 'rb') + pkl_file = open('module' + sep + 'links.pkl', 'rb') obj = cPickle.load(pkl_file) except: obj = False @@ -112,7 +113,7 @@ class File_List(object): pdata["queue"] = queue pdata["collector"] = collector - output = open('links.pkl', 'wb') + output = open('module' + sep + 'links.pkl', 'wb') cPickle.dump(pdata, output, -1) self.lock.release() -- cgit v1.2.3 From f7563727e1ccb8d764904806cb9e262ba555f824 Mon Sep 17 00:00:00 2001 From: spoob Date: Mon, 30 Nov 2009 15:18:04 +0100 Subject: Cleaned XMLRPC in Core --- module/network/Request.py | 4 +- module/plugins/FreakshareNet.py | 144 ++++++++++++++++++++++++++++++++++++++++ module/thread_list.py | 2 +- 3 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 module/plugins/FreakshareNet.py (limited to 'module') diff --git a/module/network/Request.py b/module/network/Request.py index 9f27af4ec..b80ea44da 100755 --- a/module/network/Request.py +++ b/module/network/Request.py @@ -51,7 +51,7 @@ class Request: self.lastURL = None self.auth = False - self.timeout = 5*3600 + self.timeout = 5 try: if pycurl: self.curl = True @@ -94,7 +94,7 @@ class Request: self.pycurl = pycurl.Curl() self.pycurl.setopt(pycurl.FOLLOWLOCATION, 1) self.pycurl.setopt(pycurl.MAXREDIRS, 5) - self.pycurl.setopt(pycurl.TIMEOUT, self.timeout) + self.pycurl.setopt(pycurl.TIMEOUT, (self.timeout*3600)) self.pycurl.setopt(pycurl.CONNECTTIMEOUT, 30) self.pycurl.setopt(pycurl.NOSIGNAL, 1) self.pycurl.setopt(pycurl.NOPROGRESS, 0) diff --git a/module/plugins/FreakshareNet.py b/module/plugins/FreakshareNet.py new file mode 100644 index 000000000..0768b5476 --- /dev/null +++ b/module/plugins/FreakshareNet.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +import urllib +import httplib +from module.Plugin import Plugin +from time import time + + +class FreakshareNet(Plugin): + + def __init__(self, parent): + Plugin.__init__(self, parent) + props = {} + props['name'] = "FreakshareNet" + props['type'] = "hoster" + props['pattern'] = r"http://(?:www\.)?freakshare\.net/files/\S*?/" + props['version'] = "0.1" + props['description'] = """Freakshare.com Download Plugin""" + props['author_name'] = ("sitacuisses","spoob","mkaay") + props['author_mail'] = ("sitacuisses@yahoo.de","spoob@pyload.org","mkaay@mkaay.de") + self.props = props + self.parent = parent + self.html = None + self.want_reconnect = False + self.multi_dl = False + self.req_opts = list() + + def prepare(self, thread): + pyfile = self.parent + + self.want_reconnect = False + + self.download_html() + + pyfile.status.exists = self.file_exists() + + if not pyfile.status.exists: + raise Exception, "The file was not found on the server." + return False + + self.get_waiting_time() + + pyfile.status.filename = self.get_file_name() + + pyfile.status.waituntil = self.time_plus_wait + thread.wait(self.parent) + pyfile.status.url = self.get_file_url() + pyfile.status.want_reconnect = self.want_reconnect + + return True + + def download_html(self): + url = self.parent.url + self.html = self.req.load(url, cookies=True) + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + if self.html == None: + self.download_html() + if not self.want_reconnect: + self.req_opts = self.get_download_options() # get the Post options for the Request + file_url = self.parent.url + return file_url + else: + return False + + def get_file_name(self): + if self.html == None: + self.download_html() + if not self.want_reconnect: + file_name = re.search(r"(.*?)<\/h1>", self.html).group(1) + return file_name + else: + return self.parent.url + + def get_waiting_time(self): + if self.html == None: + self.download_html() + timestring = re.search('\s*var\stime\s=\s(\d*?)\.\d*;', self.html).group(1) + if timestring: + sec = int(timestring) + 1 #add 1 sec as tenths of seconds are cut off + else: + sec = 0 + self.time_plus_wait = time() + sec + + def file_exists(self): + """ returns True or False + """ + if self.html == None: + self.download_html() + if re.search(r"Sorry, this Download doesnt exist anymore", self.html) != None: + return False + else: + return True + + def get_download_options(self): + re_envelope = re.search(r".*?value=\"Free\sDownload\".*?\n*?(.*?<.*?>\n*)*?\n*\s*?", self.html).group(0) #get the whole request + to_sort = re.findall(r"", re_envelope) + request_options = list() + for item in to_sort: #Name value pairs are output reversed from regex, so we reorder them + request_options.append((item[1], item[0])) + herewego = self.req.load(self.parent.url, None, request_options, cookies=True) # the actual download-Page + to_sort = None + to_sort = re.findall(r"", herewego) + request_options = list() + for item in to_sort: #Same as above + request_options.append((item[1], item[0])) + return request_options + + def proceed(self, url, location): + """ + request.download doesn't handle the 302 redirect correctly + that's why the data are posted "manually" via httplib + and the redirect-url is read from the header. + Important: The cookies may not be posted to the download-url + otherwise the downloaded file only contains "bad try" + Need to come up with a better idea to handle the redirect, + help is appreciated. + """ + temp_options = urllib.urlencode(self.req_opts) + temp_url = re.match(r"http://(.*?)/.*", url).group(1) # get the server name + temp_extended = re.match(r"http://.*?(/.*)", url).group(1) # get the url relative to serverroot + cookie_list = "" + for temp_cookie in self.req.cookies: #prepare cookies + cookie_list += temp_cookie.name + "=" + temp_cookie.value +";" + temp_headers = [ #create the additional header fields + ["Content-type", "application/x-www-form-urlencoded"], #this is very important + ["User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.10"], + ["Accept-Encoding", "deflate"], + ["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], + ["Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7"], + ["Connection", "keep-alive"], + ["Keep-Alive", "300"], + ["Referer", self.req.lastURL], + ["Cookie", cookie_list]] + temp_conn = httplib.HTTPConnection(temp_url) + temp_conn.request("POST", temp_extended, temp_options, dict(temp_headers)) + temp_response = temp_conn.getresponse() + new_url = temp_response.getheader("Location") # we need the Location-header + temp_conn.close + self.req.download(new_url, location, None, None, cookies=False) diff --git a/module/thread_list.py b/module/thread_list.py index aa50a654f..fc886e4b4 100644 --- a/module/thread_list.py +++ b/module/thread_list.py @@ -60,7 +60,7 @@ class Thread_List(object): def get_job(self): """return job if suitable, otherwise send thread idle""" - if not self.parent.is_time_download() or self.pause or self.reconnecting or self.list.queueEmpty(): #conditions when threads dont download + if not self.parent.server_methods.is_time_download() or self.pause or self.reconnecting or self.list.queueEmpty(): #conditions when threads dont download return None self.init_reconnect() -- cgit v1.2.3 From f3c2e597ebb63094c43ec39acb67a23a1cc2c141 Mon Sep 17 00:00:00 2001 From: mkaay Date: Mon, 30 Nov 2009 17:47:42 +0100 Subject: added xmlrpc auth without ssl --- module/remote/SecureXMLRPCServer.py | 186 ++++++++++++++++++++---------------- 1 file changed, 105 insertions(+), 81 deletions(-) (limited to 'module') diff --git a/module/remote/SecureXMLRPCServer.py b/module/remote/SecureXMLRPCServer.py index 158e44863..9dacd7b26 100644 --- a/module/remote/SecureXMLRPCServer.py +++ b/module/remote/SecureXMLRPCServer.py @@ -2,12 +2,12 @@ # which seems to be based on http://www.sabren.net/code/python/SecureXMLRPCServer.py # # Changes: -# 2007-01-06 Christian Hoffmann -# * Bugfix: replaced getattr by hasattr in the conditional -# (lead to an error otherwise) -# * SecureXMLRPCServer: added self.instance = None, otherwise a "wrong" -# exception is raised when calling unknown methods via xmlrpc -# * Added HTTP Basic authentication support +# 2007-01-06 Christian Hoffmann +# * Bugfix: replaced getattr by hasattr in the conditional +# (lead to an error otherwise) +# * SecureXMLRPCServer: added self.instance = None, otherwise a "wrong" +# exception is raised when calling unknown methods via xmlrpc +# * Added HTTP Basic authentication support # # Modified for the Sceradon project # @@ -21,86 +21,110 @@ import SocketServer import socket import base64 + +class SecureSocketConnection: + def __init__(self, connection): + self.__dict__["connection"] = connection + + def __getattr__(self, name): + return getattr(self.__dict__["connection"], name) + + def __setattr__(self, name, value): + setattr(self.__dict__["connection"], name, value) + + def shutdown(self, how=1): + self.__dict__["connection"].shutdown() + + def accept(self): + connection, address = self.__dict__["connection"].accept() + return (SecureSocketConnection(connection), address) + class SecureSocketServer(SocketServer.TCPServer, SocketServer.ThreadingMixIn): - def __init__(self, addr, cert, key, requestHandler, verify_cert_func=None): - SocketServer.TCPServer.__init__(self, addr, requestHandler) - ctx = SSL.Context(SSL.SSLv23_METHOD) - if not verify_cert_func and hasattr(self, 'verify_client_cert'): - verify_cert_func = getattr(self, 'verify_client_cert') - if verify_cert_func: - ctx.set_verify(SSL.VERIFY_PEER|SSL.VERIFY_FAIL_IF_NO_PEER_CERT, verify_cert_func) - ctx.use_privatekey_file(key) - ctx.use_certificate_file(cert) - - tmpConnection = SSL.Connection(ctx, socket.socket(socket.AF_INET, socket.SOCK_STREAM)) - self.socket = SecureSocketConnection(tmpConnection) - - self.server_bind() - self.server_activate() + def __init__(self, addr, cert, key, requestHandler, verify_cert_func=None): + SocketServer.TCPServer.__init__(self, addr, requestHandler) + ctx = SSL.Context(SSL.SSLv23_METHOD) + if not verify_cert_func and hasattr(self, 'verify_client_cert'): + verify_cert_func = getattr(self, 'verify_client_cert') + if verify_cert_func: + ctx.set_verify(SSL.VERIFY_PEER|SSL.VERIFY_FAIL_IF_NO_PEER_CERT, verify_cert_func) + ctx.use_privatekey_file(key) + ctx.use_certificate_file(cert) + + tmpConnection = SSL.Connection(ctx, socket.socket(socket.AF_INET, socket.SOCK_STREAM)) + self.socket = SecureSocketConnection(tmpConnection) + + self.server_bind() + self.server_activate() - def finish_request(self, request, client_address): - """Finish one request by instantiating RequestHandlerClass.""" - self.RequestHandlerClass(request, client_address, self) + def finish_request(self, request, client_address): + """Finish one request by instantiating RequestHandlerClass.""" + self.RequestHandlerClass(request, client_address, self) + +####################################### +########### Request Handler ########### +####################################### +class AuthXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): + def __init__(self, request, client_address, server): + self.authMap = server.getAuthenticationMap() + SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server) + + def setup(self): + self.connection = self.request + self.rfile = socket._fileobject(self.request, "rb", self.rbufsize) + self.wfile = socket._fileobject(self.request, "wb", self.wbufsize) + + def do_POST(self): + # authentication + if self.authMap != None: # explicit None! + if self.headers.has_key('authorization') and self.headers['authorization'].startswith('Basic '): + authenticationString = base64.b64decode(self.headers['authorization'].split(' ')[1]) + if authenticationString.find(':') != -1: + username, password = authenticationString.split(':', 1) + if self.authMap.has_key(username) and self.verifyPassword(username, password): + return SimpleXMLRPCRequestHandler.do_POST(self) + self.send_response(401) + self.end_headers() + return False + return SimpleXMLRPCRequestHandler.do_POST(self) + + def verifyPassword(self, username, givenPassword): + return self.authMap[username] == givenPassword -class SecureXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): - def __init__(self, request, client_address, server, client_digest=None): - self.authMap = server.getAuthenticationMap() - SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server) - self.client_digest = client_digest - - def setup(self): - self.connection = self.request - self.rfile = socket._fileobject(self.request, "rb", self.rbufsize) - self.wfile = socket._fileobject(self.request, "wb", self.wbufsize) - - def do_POST(self): - # authentication - if self.authMap != None: # explicit None! - if self.headers.has_key('authorization') and self.headers['authorization'].startswith('Basic '): - authenticationString = base64.b64decode(self.headers['authorization'].split(' ')[1]) - if authenticationString.find(':') != -1: - username, password = authenticationString.split(':', 1) - if self.authMap.has_key(username) and self.verifyPassword(username, password): - return SimpleXMLRPCRequestHandler.do_POST(self) - self.send_response(401) - self.end_headers() - return False - return SimpleXMLRPCRequestHandler.do_POST(self) - - def verifyPassword(self, username, givenPassword): - return self.authMap[username] == givenPassword +class SecureXMLRPCRequestHandler(AuthXMLRPCRequestHandler): + def __init__(self, request, client_address, server, client_digest=None): + self.authMap = server.getAuthenticationMap() + SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server) + self.client_digest = client_digest -class SecureXMLRPCServer(SecureSocketServer, SimpleXMLRPCServer): - def __init__(self, address, cert, key, authenticationMap = None, handler=SecureXMLRPCRequestHandler, verify_cert_func=None): - self.logRequests = False - self._send_traceback_header = False - self.encoding = "utf-8" - self.allow_none = True - SecureSocketServer.__init__(self, address, cert, key, handler, verify_cert_func) - # This comes from SimpleXMLRPCServer.__init__()->SimpleXMLRPCDispatcher.__init__() - self.funcs = {} - self.instance = None - self.authenticationMap = authenticationMap - - def getAuthenticationMap(self): - return self.authenticationMap +##################################### +########### XMLRPC Server ########### +##################################### +class AuthXMLRPCServer(SimpleXMLRPCServer): + def __init__(self, address, authenticationMap = None, handler=AuthXMLRPCRequestHandler): + SimpleXMLRPCServer.__init__(self, address, requestHandler=handler) + self.logRequests = False + self._send_traceback_header = False + self.encoding = "utf-8" + self.allow_none = True + self.authenticationMap = authenticationMap + + def getAuthenticationMap(self): + return self.authenticationMap -class SecureSocketConnection: - def __init__(self, connection): - self.__dict__["connection"] = connection - - def __getattr__(self, name): - return getattr(self.__dict__["connection"], name) - - def __setattr__(self, name, value): - setattr(self.__dict__["connection"], name, value) - - def shutdown(self, how=1): - self.__dict__["connection"].shutdown() - - def accept(self): - connection, address = self.__dict__["connection"].accept() - return (SecureSocketConnection(connection), address) +class SecureXMLRPCServer(AuthXMLRPCServer, SecureSocketServer): + def __init__(self, address, cert, key, authenticationMap = None, handler=SecureXMLRPCRequestHandler, verify_cert_func=None): + self.logRequests = False + self._send_traceback_header = False + self.encoding = "utf-8" + self.allow_none = True + SecureSocketServer.__init__(self, address, cert, key, handler, verify_cert_func) + # This comes from SimpleXMLRPCServer.__init__()->SimpleXMLRPCDispatcher.__init__() + self.funcs = {} + self.instance = None + self.authenticationMap = authenticationMap + + def getAuthenticationMap(self): + return self.authenticationMap -- cgit v1.2.3 From 0511d85691e8cf1c0c70045cf23e8abc6fc7cf40 Mon Sep 17 00:00:00 2001 From: mkaay Date: Mon, 30 Nov 2009 19:24:07 +0100 Subject: WIP: package system for cli --- module/file_list.py | 52 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 12 deletions(-) (limited to 'module') diff --git a/module/file_list.py b/module/file_list.py index c95eefcec..1b9526eb8 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -132,6 +132,22 @@ class File_List(object): files.append(pyfile) return files + def getFileInfo(self, id): + try: + n, pyfile = self.collector._getFileFromID(id) + except NoSuchElementException: + key, n, pyfile, pypack, pid = self.packager._getFileFromID() + info = {} + info["id"] = pyfile.id + info["url"] = pyfile.url + info["folder"] = pyfile.folder + info["filename"] = pyfile.filename + info["status_type"] = pyfile.status.type + info["status_url"] = pyfile.status.url + info["status_filename"] = pyfile.status.filename + info["status_error"] = pyfile.status.error + return info + class pyLoadCollector(): def __init__(collector, file_list): collector.file_list = file_list @@ -151,7 +167,7 @@ class File_List(object): """ ids = [] for pypack in (collector.file_list.data["packages"] + collector.file_list.data["queue"]): - for pyf in pypack.links: + for pyf in pypack.files: ids.append(pyf.id) for pyfile in collector.file_list.data["collector"]: ids.append(pyfile.id) @@ -219,7 +235,7 @@ class File_List(object): """ ids = [] for pypack in (packager.file_list.data["packages"] + packager.file_list.data["queue"]): - ids.append(pypack.id) + ids.append(pypack.data["id"]) id = 1 while id in ids: id += 1 @@ -230,10 +246,10 @@ class File_List(object): returns PyLoadPackage instance and position with given id """ for n, pypack in enumerate(packager.file_list.data["packages"]): - if pypack.id == id: + if pypack.data["id"] == id: return ("packages", n, pypack) for n, pypack in enumerate(packager.file_list.data["queue"]): - if pypack.id == id: + if pypack.data["id"] == id: return ("queue", n, pypack) raise NoSuchElementException() @@ -253,11 +269,11 @@ class File_List(object): def addNewPackage(packager, package_name=None): pypack = PyLoadPackage() - pypack.id = packager._getFreeID() + pypack.data["id"] = packager._getFreeID() if package_name is not None: pypack.data["package_name"] = package_name packager.file_list.data["packages"].append(pypack) - return pypack.id + return pypack.data["id"] def removePackage(packager, id): packager.file_list.lock.acquire() @@ -310,6 +326,17 @@ class File_List(object): finally: packager.file_list.lock.release() + def getPackageData(packager, id): + key, n, pypack = packager._getPackageFromID(id) + return pypack.data + + def getPackageFiles(packager, id): + key, n, pypack = packager._getPackageFromID(id) + ids = [] + for pyfile in pypack: + ids.append(pyfile.id) + return ids + def addFileToPackage(packager, id, pyfile): key, n, pypack = packager._getPackageFromID(id) pyfile.package = pypack @@ -396,23 +423,24 @@ class PyLoadFileData(): def parsePackage(self, pack): if pack: - self.pack_id = pack.id + self.pack_id = pack.data["id"] class PyLoadPackageData(): def __init__(self): self.data = None - self.links = [] + self.files = [] def set(self, pypack): self.data = pypack.data - for pyfile in pypack.links: + for pyfile in pypack.files: fdata = PyLoadFileData() fdata.set(pyfile) - self.links.append(fdata) + self.files.append(fdata) def get(self, pypack): - for fdata in self.links: + pypack.data = self.data + for fdata in self.files: pyfile = PyLoadFile() fdata.get(pyfile) pyfile.package = pypack - pypack.links.append(pyfile) + pypack.files.append(pyfile) -- cgit v1.2.3 From 7caec14b0a307df9f2bd9ea6a9db6977a836145b Mon Sep 17 00:00:00 2001 From: mkaay Date: Mon, 30 Nov 2009 21:25:33 +0100 Subject: WIP: package system second draft - unstable --- module/file_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module') diff --git a/module/file_list.py b/module/file_list.py index 1b9526eb8..585c58ddf 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -333,7 +333,7 @@ class File_List(object): def getPackageFiles(packager, id): key, n, pypack = packager._getPackageFromID(id) ids = [] - for pyfile in pypack: + for pyfile in pypack.files: ids.append(pyfile.id) return ids -- cgit v1.2.3 From b3c1f830aaba0c22de22693e6b8cd81fe392f21a Mon Sep 17 00:00:00 2001 From: mkaay Date: Mon, 30 Nov 2009 22:00:04 +0100 Subject: fixed file_list --- module/file_list.py | 8 ++++---- module/plugins/RapidshareCom.py | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'module') diff --git a/module/file_list.py b/module/file_list.py index 585c58ddf..f66567e0f 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -62,11 +62,11 @@ class File_List(object): collector = [] for n, pd in enumerate(obj["packages"]): p = PyLoadPackage() - pd.get(p) + pd.get(p, self) packages.append(p) for pd in obj["queue"]: p = PyLoadPackage() - pd.get(p) + pd.get(p, self) queue.append(p) for fd in obj["collector"]: f = PyLoadFile("", self) @@ -437,10 +437,10 @@ class PyLoadPackageData(): fdata.set(pyfile) self.files.append(fdata) - def get(self, pypack): + def get(self, pypack, fl): pypack.data = self.data for fdata in self.files: - pyfile = PyLoadFile() + pyfile = PyLoadFile(fdata.url, fl) fdata.get(pyfile) pyfile.package = pypack pypack.files.append(pyfile) diff --git a/module/plugins/RapidshareCom.py b/module/plugins/RapidshareCom.py index 275fa761c..29e9a7c1f 100644 --- a/module/plugins/RapidshareCom.py +++ b/module/plugins/RapidshareCom.py @@ -6,6 +6,7 @@ from time import time from module.Plugin import Plugin import hashlib +import logging class RapidshareCom(Plugin): @@ -26,6 +27,7 @@ class RapidshareCom(Plugin): self.time_plus_wait = None #time() + wait in seconds self.want_reconnect = False self.no_slots = True + self.logger = logging.getLogger("log") self.read_config() if self.config['premium']: self.multi_dl = True @@ -141,7 +143,7 @@ class RapidshareCom(Plugin): except: if re.search(r"(Currently a lot of users|There are no more download slots)", self.html[1], re.I) != None: self.time_plus_wait = time() + 130 - self.parent.parent.logger.info("Rapidshare: No free slots!") + self.logger.info("Rapidshare: No free slots!") self.no_slots = True return True self.no_slots = False -- cgit v1.2.3 From dcf9df576d0e30e016bb6ecb9fb67decc04d2761 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 30 Nov 2009 23:31:37 +0100 Subject: Fixed EOL Errors, beautified js --- module/web/static/default/home.js | 95 ++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 51 deletions(-) (limited to 'module') diff --git a/module/web/static/default/home.js b/module/web/static/default/home.js index 0d74aa17a..025dcfcc7 100644 --- a/module/web/static/default/home.js +++ b/module/web/static/default/home.js @@ -6,12 +6,12 @@ var dwProgressBar = new Class({ //options options: { container: $$('body')[0], - boxID:'', - percentageID:'', - displayID:'', + boxID: '', + percentageID: '', + displayID: '', startPercentage: 0, displayText: false, - speed:10 + speed: 10 }, //initialization @@ -25,20 +25,20 @@ var dwProgressBar = new Class({ //creates the box and percentage elements createElements: function() { var box = new Element('div', { - id:this.options.boxID + this.options.id, - 'class' : this.options.boxID + id: this.options.boxID + this.options.id, + 'class': this.options.boxID }); var perc = new Element('div', { - id:this.options.percentageID + this.options.id, - 'style':'width:0px;', - 'class' : this.options.percentageID + id: this.options.percentageID + this.options.id, + 'style': 'width:0px;', + 'class': this.options.percentageID }); perc.inject(box); box.inject(this.options.container); - if(this.options.displayText) { + if (this.options.displayText) { var text = new Element('div', { - id:this.options.displayID + this.options.id, - 'class' : this.options.displayID + id: this.options.displayID + this.options.id, + 'class': this.options.displayID }); text.inject(this.options.container); } @@ -47,19 +47,19 @@ var dwProgressBar = new Class({ //calculates width in pixels from percentage calculate: function(percentage) { - return (document.id(this.options.boxID+ this.options.id).getStyle('width').replace('px','') * (percentage / 100)).toInt(); + return (document.id(this.options.boxID + this.options.id).getStyle('width').replace('px', '') * (percentage / 100)).toInt(); }, //animates the change in percentage animate: function(to) { - document.id(this.options.percentageID+ this.options.id).set('morph', { + document.id(this.options.percentageID + this.options.id).set('morph', { duration: this.options.speed, - link:'cancel' + link: 'cancel' }).morph({ - width:this.calculate(to.toInt()) + width: this.calculate(to.toInt()) }); - if(this.options.displayText) { - document.id(this.options.displayID+ this.options.id).set('text', to.toInt() + '%'); + if (this.options.displayText) { + document.id(this.options.displayID + this.options.id).set('text', to.toInt() + '%'); } }, @@ -70,7 +70,6 @@ var dwProgressBar = new Class({ }); - req = new Request.JSON({ onSuccess: renderTable, method: 'get', @@ -80,48 +79,46 @@ req = new Request.JSON({ limit: 20000 }); - var dls = [] var pbs = [] -function renderTable(data){ +function renderTable(data) { - data.downloads.each(function(dl){ + data.downloads.each(function(dl) { - if (dls.contains(dl.id)){ + if (dls.contains(dl.id)) { - var div = $('dl'+dl.id) + var div = $('dl' + dl.id) pbs[dl.id].set(dl.percent) div.getChildren("b")[0].textContent = dl.name - if (dl.status == "downloading"){ + if (dl.status == "downloading") { - size = Math.round((dl.size - dl.kbleft) / 1024) + "/" + Math.round(dl.size / 1024) + " MB" - speed = Math.round(dl.speed) + " kb/s" - eta = dl.eta - + size = Math.round((dl.size - dl.kbleft) / 1024) + "/" + Math.round(dl.size / 1024) + " MB"; + speed = Math.round(dl.speed) + " kb/s"; + eta = dl.eta; - }else if (dl.status == "waiting"){ + } else if (dl.status == "waiting") { - size = "waiting "+ dl.wait - speed = "" - eta = "" + size = "waiting " + dl.wait; + speed = ""; + eta = ""; } - div.getChildren(".dlsize")[0].textContent = size - div.getChildren(".dlspeed")[0].textContent = speed - div.getChildren(".dltime")[0].textContent = eta + div.getChildren(".dlsize")[0].textContent = size; + div.getChildren(".dlspeed")[0].textContent = speed; + div.getChildren(".dltime")[0].textContent = eta; - }else{ + } else { dls.push(dl.id) container = $("dlcontainer") dldiv = new Element('div', { - 'id' : 'dl'+dl.id, + 'id': 'dl' + dl.id, 'class': 'download', 'styles': { 'display': 'None' @@ -158,28 +155,27 @@ function renderTable(data){ 'class': 'dlspeed', 'html': Math.round(dl.speed) + " kb/s" }).inject(dldiv) - + new Element('div', { 'class': 'dltime', 'html': dl.eta }).inject(dldiv) //dldiv.dissolve({duration : 0}) - dldiv.reveal() } }) - dls.each(function(id, index){ + dls.each(function(id, index) { - if (data.ids.contains(id)){ + if (data.ids.contains(id)) { - }else{ +} else { //$("dl"+id).reveal() - dls.erase(id) - $('dl'+id).nix() + dls.erase(id); + $('dl' + id).nix() } @@ -187,7 +183,8 @@ function renderTable(data){ } -window.addEvent('domready', function(){ +window.addEvent('domready', +function() { /* //create the progress bar for example 1 @@ -205,8 +202,4 @@ pb = new dwProgressBar({ req.startTimer(); - -}); - - - +}); \ No newline at end of file -- cgit v1.2.3 From 9ecbb58e7ea06cb604745e25627e2fb4709fa442 Mon Sep 17 00:00:00 2001 From: spoob Date: Wed, 2 Dec 2009 15:37:25 +0100 Subject: New Update Function, pycurl able to just load headers, little fixes --- module/Unzip.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++ module/network/Request.py | 9 ++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 module/Unzip.py (limited to 'module') diff --git a/module/Unzip.py b/module/Unzip.py new file mode 100644 index 000000000..f56fbe751 --- /dev/null +++ b/module/Unzip.py @@ -0,0 +1,50 @@ +import zipfile +import os + +class Unzip: + def __init__(self): + pass + + def extract(self, file, dir): + if not dir.endswith(':') and not os.path.exists(dir): + os.mkdir(dir) + + zf = zipfile.ZipFile(file) + + # create directory structure to house files + self._createstructure(file, dir) + + # extract files to directory structure + for i, name in enumerate(zf.namelist()): + + if not name.endswith('/') and not name.endswith("config"): + print "extracting", name.replace("pyload/","") + outfile = open(os.path.join(dir, name.replace("pyload/","")), 'wb') + outfile.write(zf.read(name)) + outfile.flush() + outfile.close() + + def _createstructure(self, file, dir): + self._makedirs(self._listdirs(file), dir) + + def _makedirs(self, directories, basedir): + """ Create any directories that don't currently exist """ + for dir in directories: + curdir = os.path.join(basedir, dir) + if not os.path.exists(curdir): + os.mkdir(curdir) + + def _listdirs(self, file): + """ Grabs all the directories in the zip structure + This is necessary to create the structure before trying + to extract the file to it. """ + zf = zipfile.ZipFile(file) + + dirs = [] + + for name in zf.namelist(): + if name.endswith('/'): + dirs.append(name.replace("pyload/","")) + + dirs.sort() + return dirs diff --git a/module/network/Request.py b/module/network/Request.py index b80ea44da..cda8e50f1 100755 --- a/module/network/Request.py +++ b/module/network/Request.py @@ -110,7 +110,7 @@ class Request: "Connection: keep-alive", "Keep-Alive: 300"]) - def load(self, url, get={}, post={}, ref=True, cookies=False): + def load(self, url, get={}, post={}, ref=True, cookies=False, just_header=False): if post: post = urllib.urlencode(post) @@ -136,6 +136,13 @@ class Request: if ref and self.lastURL is not None: self.pycurl.setopt(pycurl.REFERER, self.lastURL) + if just_header: + self.pycurl.setopt(pycurl.NOPROGRESS, 1) + self.pycurl.setopt(pycurl.NOBODY, 1) + self.pycurl.perform() + self.pycurl.setopt(pycurl.NOPROGRESS, 0) + self.pycurl.setopt(pycurl.NOBODY, 0) + return self.header self.pycurl.perform() -- cgit v1.2.3 From 106e79456886563e4ee4ed43027bc69984f65928 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Wed, 2 Dec 2009 20:36:43 +0100 Subject: new bottle.py, re implemented webserver(not ready yet) --- module/web/WebServer.py | 428 +++++++++++++++++++++++----- module/web/bottle.py | 727 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 896 insertions(+), 259 deletions(-) (limited to 'module') diff --git a/module/web/WebServer.py b/module/web/WebServer.py index 0712f1dce..29b0aafe8 100644 --- a/module/web/WebServer.py +++ b/module/web/WebServer.py @@ -1,72 +1,364 @@ -import sys -from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler -from xmlrpclib import ServerProxy -from time import time -import re +#import sys +#from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +#from xmlrpclib import ServerProxy +#from time import time +#import re +# +#class Handler(BaseHTTPRequestHandler): +# +# def do_GET(self): +# global coreserver +# stdout = sys.stdout +# sys.stdout = self.wfile +# if self.path == "/": +# print "Server Runs" +# elif self.path == "/downloads": +# print self.get_downloads() +# elif re.search("/add=.?", self.path): +# if re.match(is_url, self.path.split("/add=")[1]): +# coreserver.add_urls([self.path.split("/add=")[1]]) +# print "Link Added" +# else: +# try: +# print open(self.path[1:], 'r').read() +# except IOError: +# self.send_error(404) +# +# def format_size(self, size): +# return str(size / 1024) + " MiB" +# +# def format_time(self,seconds): +# seconds = int(seconds) +# hours, seconds = divmod(seconds, 3600) +# minutes, seconds = divmod(seconds, 60) +# return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) +# +# def get_downloads(self): +# data = coreserver.status_downloads() +# for download in data: +# print "

%s

" % download["name"] +# if download["status"] == "downloading": +# percent = download["percent"] +# z = percent / 4 +# print "

%s

" % dl_name +# print "[" + z * "#" + (25-z) * " " + "]" + str(percent) + "%
" +# print "Speed: " + str(int(download['speed'])) + " kb/s" +# print "Size: " + self.format_size(download['size']) +# print "Finished in: " + self.format_time(download['eta']) +# print "ID: " + str(download['id']) +# dl_status = "[" + z * "#" + (25-z) * " " + "] " + str(percent) + "%" + " Speed: " + str(int(download['speed'])) + " kb/s" + " Size: " + self.format_size(download['size']) + " Finished in: " + self.format_time(download['eta']) + " ID: " + str(download['id']) +# if download["status"] == "waiting": +# print "waiting: " + self.format_time(download["wait_until"]- time()) +# +#is_url = re.compile("^(((https?|ftp)\:\/\/)?([\w\.\-]+(\:[\w\.\&%\$\-]+)*@)?((([^\s\(\)\<\>\\\"\.\[\]\,@;:]+)(\.[^\s\(\)\<\>\\\"\.\[\]\,@;:]+)*(\.[a-zA-Z]{2,4}))|((([01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}([01]?\d{1,2}|2[0-4]\d|25[0-5])))(\b\:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3}|0)\b)?((\/[^\/][\w\.\,\?\'\\\/\+&%\$#\=~_\-@]*)*[^\.\,\?\"\'\(\)\[\]!;<>{}\s\x7F-\xFF])?)$",re.IGNORECASE) +# +#coreserver = None +# +#class WebServer(): +# +# def start(self): +# try: +# global coreserver +# coreserver = ServerProxy("https://testuser:testpw@localhost:1337", allow_none=True) +# webserver = HTTPServer(('',8080),Handler) +# print 'server started at port 8080' +# webserver.serve_forever() +# except KeyboardInterrupt: +# webserver.socket.close() +# +#if __name__ == "__main__": +# web = WebServer() +# web.start() -class Handler(BaseHTTPRequestHandler): + +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2009 RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### +import random +import threading +import time + +import bottle +from bottle import abort +from bottle import redirect +from bottle import request +from bottle import response +from bottle import route +from bottle import run +from bottle import send_file +from bottle import template +from bottle import validate + + +core = None +core_methods = None + +PATH = "./module/web/" +TIME = time.strftime("%a, %d %b %Y 00:00:00 +0000", time.localtime()) #set time to current day +USERS = {} +# TODO: Implement new server methods +@route('/login', method='POST') +def do_login(): + #print request.GET + + + username = core.config['webinterface']['username'] + pw = core.config['webinterface']['password'] + + if request.POST['u'] == username and request.POST['p'] == pw: + + id = int(random.getrandbits(16)) + ua = request.HEADER("HTTP_USER_AGENT") + ip = request.HEADER("REMOTE_ADDR") + + auth = {} + + auth['ua'] = ua + auth['ip'] = ip + auth['user'] = username + + USERS[id] = auth + + response.COOKIES['user'] = username + response.COOKIES['id'] = id + + return template('default', page='loggedin', user=username) + else: + return template('default', page='login') + +@route('/login') +def login(): + + if check_auth(request): + redirect("/") + + return template('default', page='login') + +@route('/logout') +def logout(): + try: + del USERS[int(request.COOKIES.get('id'))] + except: + pass - def do_GET(self): - global coreserver - stdout = sys.stdout - sys.stdout = self.wfile - if self.path == "/": - print "Server Runs" - elif self.path == "/downloads": - print self.get_downloads() - elif re.search("/add=.?", self.path): - if re.match(is_url, self.path.split("/add=")[1]): - coreserver.add_urls([self.path.split("/add=")[1]]) - print "Link Added" + redirect("/login") + +@route('/') +def home(): + + if not check_auth(request): + redirect("/login") + + username = request.COOKIES.get('user') + + dls = core_methods.status_downloads() + + for dl in dls: + dl['eta'] = str(core.format_time(dl['eta'])) + dl['wait_until'] = str(core.format_time(dl['wait_until'] - time.time())) + + + return template('default', page='home', links=dls, user=username, status=core_methods.status_server()) + +@route('/queue') +def queue(): + + if not check_auth(request): + redirect("/login") + + username = request.COOKIES.get('user') + + return template('default', page='queue', links=core.get_links(), user=username, status=core_methods.status_server()) + +@route('/downloads') +def downloads(): + + if not check_auth(request): + redirect("/login") + + username = request.COOKIES.get('user') + + return template('default', page='downloads', links=core_methods.status_downloads(), user=username, status=core_methods.status_server()) + + +@route('/logs') +def logs(): + + if not check_auth(request): + redirect("/login") + + username = request.COOKIES.get('user') + + return template('default', page='logs', links=core_methods.status_downloads(), user=username, status=core_methods.status_server()) + +@route('/json/links') +def get_links(): + response.header['Cache-Control'] = 'no-cache, must-revalidate' + response.content_type = 'application/json' + + if not check_auth(request): + abort(404, "No Access") + + json = '{ "downloads": [' + + downloads = core_methods.status_downloads() + ids = [] + + for dl in downloads: + ids.append(dl['id']) + json += '{' + json += '"id": %s, "name": "%s", "speed": %s, "eta": "%s", "kbleft": %s, "size": %s, "percent": %s, "wait": "%s", "status": "%s"'\ + % (str(dl['id']), str(dl['name']), str(int(dl['speed'])), str(core.format_time(dl['eta'])), dl['kbleft'], dl['size'], dl['percent'], str(core.format_time(dl['wait_until'] - time.time())), dl['status']) + + json += "}," + + if json.endswith(","): json = json[:-1] + + json += '], "ids": %s }' % str(ids) + + return json + +@route('/json/status') +def get_status(): + response.header['Cache-Control'] = 'no-cache, must-revalidate' + response.content_type = 'application/json' + + if not check_auth(request): + abort(404, "No Access") + + data = core_methods.status_server() + + if data['pause']: + status = "paused" + else: + status = "running" + + json = '{ "status": "%s", "speed": "%s", "queue": "%s" }' % (status, str(int(data['speed'])), str(data['queue'])) + + return json + +@route('/json/addlinks', method='POST') +def add_links(): + response.header['Cache-Control'] = 'no-cache, must-revalidate' + response.content_type = 'application/json' + + if not check_auth(request): + abort(404, "No Access") + + links = request.POST['links'].split('\n') + + core.add_links(links) + + return "{}" + +@route('/json/pause') +def pause(): + response.header['Cache-Control'] = 'no-cache, must-revalidate' + response.content_type = 'application/json' + + if not check_auth(request): + abort(404, "No Access") + + core.thread_list.pause = True + + return "{}" + + +@route('/json/play') +def play(): + response.header['Cache-Control'] = 'no-cache, must-revalidate' + response.content_type = 'application/json' + + if not check_auth(request): + abort(404, "No Access") + + core.thread_list.pause = False + + return "{}" + +@route('/favicon.ico') +def favicon(): + + if request.HEADER("HTTP_IF_MODIFIED_SINCE") == TIME: abort(304, "Not Modified") + + response.header['Last-Modified'] = TIME + + send_file('favicon.ico', root=(PATH + 'static/')) + +@route('static/:section/:filename') +def static_folder(section, filename): + + if request.HEADER("HTTP_IF_MODIFIED_SINCE") == TIME: abort(304, "Not Modified") + + response.header['Last-Modified'] = TIME + send_file(filename, root=(PATH + 'static/' + section)) + +@route('/static/:filename') +def static_file(filename): + + if request.HEADER("HTTP_IF_MODIFIED_SINCE") == TIME: abort(304, "Not Modified") + + response.header['Last-Modified'] = TIME + send_file(filename, root=(PATH + 'static/')) + + +def check_auth(req): + + try: + user = req.COOKIES.get('user') + id = int(req.COOKIES.get('id')) + ua = req.HEADER("HTTP_USER_AGENT") + ip = req.HEADER("REMOTE_ADDR") + + if USERS[id]['user'] == user and USERS[id]['ua'] == ua and USERS[id]['ip'] == ip: + return True + except: + return False + + return False + + +class WebServer(threading.Thread): + def __init__(self, pycore): + threading.Thread.__init__(self) + + global core, core_methods, TIME + core = pycore + core_methods = pycore.server_methods + self.core = pycore + self.setDaemon(True) + + if pycore.config['general']['debug_mode']: + bottle.debug(True) + TIME = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.localtime()) else: - try: - print open(self.path[1:], 'r').read() - except IOError: - self.send_error(404) - - def format_size(self, size): - return str(size / 1024) + " MiB" - - def format_time(self,seconds): - seconds = int(seconds) - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - - def get_downloads(self): - data = coreserver.status_downloads() - for download in data: - print "

%s

" % download["name"] - if download["status"] == "downloading": - percent = download["percent"] - z = percent / 4 - print "

%s

" % dl_name - print "[" + z * "#" + (25-z) * " " + "]" + str(percent) + "%
" - print "Speed: " + str(int(download['speed'])) + " kb/s" - print "Size: " + self.format_size(download['size']) - print "Finished in: " + self.format_time(download['eta']) - print "ID: " + str(download['id']) - dl_status = "[" + z * "#" + (25-z) * " " + "] " + str(percent) + "%" + " Speed: " + str(int(download['speed'])) + " kb/s" + " Size: " + self.format_size(download['size']) + " Finished in: " + self.format_time(download['eta']) + " ID: " + str(download['id']) - if download["status"] == "waiting": - print "waiting: " + self.format_time(download["wait_until"]- time()) - -is_url = re.compile("^(((https?|ftp)\:\/\/)?([\w\.\-]+(\:[\w\.\&%\$\-]+)*@)?((([^\s\(\)\<\>\\\"\.\[\]\,@;:]+)(\.[^\s\(\)\<\>\\\"\.\[\]\,@;:]+)*(\.[a-zA-Z]{2,4}))|((([01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}([01]?\d{1,2}|2[0-4]\d|25[0-5])))(\b\:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3}|0)\b)?((\/[^\/][\w\.\,\?\'\\\/\+&%\$#\=~_\-@]*)*[^\.\,\?\"\'\(\)\[\]!;<>{}\s\x7F-\xFF])?)$",re.IGNORECASE) - -coreserver = None - -class WebServer(): - - def start(self): - try: - global coreserver - coreserver = ServerProxy("https://testuser:testpw@localhost:1337", allow_none=True) - webserver = HTTPServer(('',8080),Handler) - print 'server started at port 8080' - webserver.serve_forever() - except KeyboardInterrupt: - webserver.socket.close() - -if __name__ == "__main__": - web = WebServer() - web.start() + bottle.debug(False) + + #@TODO remove + #TIME = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.localtime()) + bottle.TEMPLATE_PATH.append('./module/web/templates/') + + def run(self): + self.core.logger.info("Starting Webinterface on %s port %s" % (self.core.config['webinterface']['listenaddr'],self.core.config['webinterface']['port'])) + try: + run(host=self.core.config['webinterface']['listenaddr'], port=int(self.core.config['webinterface']['port']), quiet=True) + except: + self.core.logger.error("Failed starting webserver, no webinterface available: Can't create socket") + exit() \ No newline at end of file diff --git a/module/web/bottle.py b/module/web/bottle.py index 66ceb527f..41a8c8fc0 100644 --- a/module/web/bottle.py +++ b/module/web/bottle.py @@ -1,15 +1,15 @@ # -*- coding: utf-8 -*- """ -Bottle is a fast and simple mirco-framework for small web-applications. It -offers request dispatching (Routes) with url parameter support, Templates, -key/value Databases, a build-in HTTP Server and adapters for many third party -WSGI/HTTP-server and template engines. All in a single file and with no +Bottle is a fast and simple micro-framework for small web applications. It +offers request dispatching (Routes) with url parameter support, templates, +key/value databases, a built-in HTTP Server and adapters for many third party +WSGI/HTTP-server and template engines - all in a single file and with no dependencies other than the Python Standard Library. Homepage and documentation: http://wiki.github.com/defnull/bottle Special thanks to Stefan Matthias Aust [http://github.com/sma] - for his contribution to SimpelTemplate + for his contribution to SimpleTemplate Licence (MIT) ------------- @@ -62,9 +62,10 @@ Example """ __author__ = 'Marcel Hellkamp' -__version__ = '0.5.7' +__version__ = '0.6.4' __license__ = 'MIT' +import types import sys import cgi import mimetypes @@ -75,26 +76,31 @@ import re import random import threading import time +import warnings +import email.utils from wsgiref.headers import Headers as HeaderWrapper from Cookie import SimpleCookie import anydbm as dbm +import subprocess +import thread + try: from urlparse import parse_qs -except ImportError: +except ImportError: # pragma: no cover from cgi import parse_qs try: import cPickle as pickle -except ImportError: +except ImportError: # pragma: no cover import pickle as pickle try: try: from json import dumps as json_dumps - except ImportError: + except ImportError: # pragma: no cover from simplejson import dumps as json_dumps -except ImportError: +except ImportError: # pragma: no cover json_dumps = None @@ -105,38 +111,37 @@ except ImportError: # Exceptions and Events class BottleException(Exception): - """ A base class for exceptions used by bottle.""" + """ A base class for exceptions used by bottle. """ pass class HTTPError(BottleException): - """ A way to break the execution and instantly jump to an error handler. """ + """ + A way to break the execution and instantly jump to an error handler. + """ def __init__(self, status, text): self.output = text self.http_status = int(status) + BottleException.__init__(self, status, text) def __repr__(self): - return "HTTPError(%d,%s)" % (self.http_status, repr(self.output)) + return 'HTTPError(%d,%s)' % (self.http_status, repr(self.output)) def __str__(self): - out = [] - status = self.http_status - name = HTTP_CODES.get(status,'Unknown').title() - url = request.path - out.append('') - out.append('Error %d: %s' % (status, name)) - out.append('

Error %d: %s

' % (status, name)) - out.append('

Sorry, the requested URL "%s" caused an error.

' % url) - out.append(''.join(list(self.output))) - out.append('') - return "\n".join(out) + return HTTP_ERROR_TEMPLATE % { + 'status' : self.http_status, + 'url' : request.path, + 'error_name' : HTTP_CODES.get(self.http_status, 'Unknown').title(), + 'error_message' : ''.join(self.output) + } class BreakTheBottle(BottleException): - """ Not an exception, but a straight jump out of the controller code. - + """ + Not an exception, but a straight jump out of the controller code. Causes the Bottle to instantly call start_response() and return the - content of output """ + content of output + """ def __init__(self, output): self.output = output @@ -149,8 +154,10 @@ class BreakTheBottle(BottleException): _default_app = None def default_app(newapp = None): - ''' Returns the current default app or sets a new one. - Defaults to an instance of Bottle ''' + """ + Returns the current default app or sets a new one. + Defaults to an instance of Bottle + """ global _default_app if newapp: _default_app = newapp @@ -161,17 +168,20 @@ def default_app(newapp = None): class Bottle(object): - def __init__(self, catchall=True, debug=False, optimize=False, autojson=True): + def __init__(self, catchall=True, optimize=False, autojson=True): self.simple_routes = {} self.regexp_routes = {} + self.default_route = None self.error_handler = {} self.optimize = optimize - self.debug = debug self.autojson = autojson self.catchall = catchall + self.serve = True def match_url(self, url, method='GET'): - """Returns the first matching handler and a parameter dict or (None, None) """ + """ + Returns the first matching handler and a parameter dict or (None, None) + """ url = url.strip().lstrip("/ ") # Search for static routes first route = self.simple_routes.get(method,{}).get(url,None) @@ -186,47 +196,116 @@ class Bottle(object): if i > 0 and self.optimize and random.random() <= 0.001: routes[i-1], routes[i] = routes[i], routes[i-1] return (handler, match.groupdict()) - return (None, None) - - def add_route(self, route, handler, method='GET', simple=False): + if self.default_route: + return (self.default_route, {}) + if method == 'HEAD': # Fall back to GET + return self.match_url(url) + else: + return (None, None) + + def add_controller(self, route, controller, **kargs): + """ Adds a controller class or object """ + if '{action}' not in route and 'action' not in kargs: + raise BottleException("Routes to controller classes or object MUST" + " contain an {action} placeholder or use the action-parameter") + for action in (m for m in dir(controller) if not m.startswith('_')): + handler = getattr(controller, action) + if callable(handler) and action == kargs.get('action', action): + self.add_route(route.replace('{action}', action), handler, **kargs) + + def add_route(self, route, handler, method='GET', simple=False, **kargs): """ Adds a new route to the route mappings. """ + if isinstance(handler, type) and issubclass(handler, BaseController): + handler = handler() + if isinstance(handler, BaseController): + self.add_controller(route, handler, method=method, simple=simple, **kargs) + return method = method.strip().upper() route = route.strip().lstrip('$^/ ').rstrip('$^ ') if re.match(r'^(\w+/)*\w*$', route) or simple: self.simple_routes.setdefault(method, {})[route] = handler else: - route = re.sub(r':([a-zA-Z_]+)(?P[^\w/])(?P.+?)(?P=uniq)',r'(?P<\1>\g)',route) - route = re.sub(r':([a-zA-Z_]+)',r'(?P<\1>[^/]+)', route) + route = re.sub(r':([a-zA-Z_]+)(?P[^\w/])(?P.+?)(?P=uniq)', + r'(?P<\1>\g)',route) + route = re.sub(r':([a-zA-Z_]+)', r'(?P<\1>[^/]+)', route) route = re.compile('^%s$' % route) self.regexp_routes.setdefault(method, []).append([route, handler]) def route(self, url, **kargs): - """ Decorator for request handler. Same as add_route(url, handler, **kargs).""" + """ + Decorator for request handler. + Same as add_route(url, handler, **kargs). + """ def wrapper(handler): self.add_route(url, handler, **kargs) return handler return wrapper + def set_default(self, handler): + self.default_route = handler + + def default(self): + """ Decorator for request handler. Same as add_defroute( handler ).""" + def wrapper(handler): + self.set_default(handler) + return handler + return wrapper + def set_error_handler(self, code, handler): """ Adds a new error handler. """ - code = int(code) - self.error_handler[code] = handler + self.error_handler[int(code)] = handler def error(self, code=500): - """ Decorator for error handler. Same as set_error_handler(code, handler).""" + """ + Decorator for error handler. + Same as set_error_handler(code, handler). + """ def wrapper(handler): self.set_error_handler(code, handler) return handler return wrapper + def cast(self, out): + """ + Cast the output to an iterable of strings or something WSGI can handle. + Set Content-Type and Content-Length when possible. Then clear output + on HEAD requests. + Supports: False, str, unicode, list(unicode), dict(), open() + """ + if not out: + out = [] + response.header['Content-Length'] = '0' + elif isinstance(out, types.StringType): + out = [out] + elif isinstance(out, unicode): + out = [out.encode(response.charset)] + elif isinstance(out, list) and isinstance(out[0], unicode): + out = map(lambda x: x.encode(response.charset), out) + elif self.autojson and json_dumps and isinstance(out, dict): + out = [json_dumps(out)] + response.content_type = 'application/json' + elif hasattr(out, 'read'): + out = request.environ.get('wsgi.file_wrapper', + lambda x: iter(lambda: x.read(8192), ''))(out) + if isinstance(out, list) and len(out) == 1: + response.header['Content-Length'] = str(len(out[0])) + if not hasattr(out, '__iter__'): + raise TypeError('Request handler for route "%s" returned [%s] ' + 'which is not iterable.' % (request.path, type(out).__name__)) + return out + + def __call__(self, environ, start_response): - """ The bottle WSGI-interface .""" + """ The bottle WSGI-interface. """ request.bind(environ) response.bind() try: # Unhandled Exceptions try: # Bottle Error Handling + if not self.serve: + abort(503, "Server stopped") handler, args = self.match_url(request.path, request.method) - if not handler: raise HTTPError(404, "Not found") + if not handler: + raise HTTPError(404, "Not found") output = handler(**args) db.close() except BreakTheBottle, e: @@ -234,26 +313,18 @@ class Bottle(object): except HTTPError, e: response.status = e.http_status output = self.error_handler.get(response.status, str)(e) - # output casting - if hasattr(output, 'read'): - output = environ.get('wsgi.file_wrapper', lambda x: iter(lambda: x.read(8192), ''))(output) - elif self.autojson and json_dumps and isinstance(output, dict): - output = json_dumps(output) - response.content_type = 'application/json' - if isinstance(output, str): - response.header['Content-Length'] = str(len(output)) - output = [output] + output = self.cast(output) + if response.status in (100, 101, 204, 304) or request.method == 'HEAD': + output = [] # rfc2616 section 4.3 except (KeyboardInterrupt, SystemExit, MemoryError): raise except Exception, e: response.status = 500 if self.catchall: err = "Unhandled Exception: %s\n" % (repr(e)) - if self.debug: - err += "

Traceback:

\n
\n"
-                    err += traceback.format_exc(10)
-                    err += "\n
" - output = str(HTTPError(500, err)) + if DEBUG: + err += TRACEBACK_TEMPLATE % traceback.format_exc(10) + output = [str(HTTPError(500, err))] request._environ['wsgi.errors'].write(err) else: raise @@ -267,8 +338,11 @@ class Request(threading.local): """ Represents a single request using thread-local namespace. """ def bind(self, environ): - """ Binds the enviroment of the current request to this request handler """ + """ + Binds the enviroment of the current request to this request handler + """ self._environ = environ + self.environ = self._environ self._GET = None self._POST = None self._GETPOST = None @@ -279,25 +353,25 @@ class Request(threading.local): @property def method(self): - ''' Returns the request method (GET,POST,PUT,DELETE,...) ''' + """ Get the request method (GET,POST,PUT,DELETE,...) """ return self._environ.get('REQUEST_METHOD', 'GET').upper() @property def query_string(self): - ''' Content of QUERY_STRING ''' + """ Get content of QUERY_STRING """ return self._environ.get('QUERY_STRING', '') @property def input_length(self): - ''' Content of CONTENT_LENGTH ''' + """ Get content of CONTENT_LENGTH """ try: - return int(self._environ.get('CONTENT_LENGTH', '0')) + return max(0,int(self._environ.get('CONTENT_LENGTH', '0'))) except ValueError: return 0 @property def GET(self): - """Returns a dict with GET parameters.""" + """ Get a dict with GET parameters. """ if self._GET is None: data = parse_qs(self.query_string, keep_blank_values=True) self._GET = {} @@ -310,9 +384,10 @@ class Request(threading.local): @property def POST(self): - """Returns a dict with parsed POST or PUT data.""" + """ Get a dict with parsed POST or PUT data. """ if self._POST is None: - data = cgi.FieldStorage(fp=self._environ['wsgi.input'], environ=self._environ, keep_blank_values=True) + data = cgi.FieldStorage(fp=self._environ['wsgi.input'], + environ=self._environ, keep_blank_values=True) self._POST = {} for item in data.list: name = item.name @@ -326,7 +401,7 @@ class Request(threading.local): @property def params(self): - ''' Returns a mix of GET and POST data. POST overwrites GET ''' + """ Returns a mix of GET and POST data. POST overwrites GET """ if self._GETPOST is None: self._GETPOST = dict(self.GET) self._GETPOST.update(dict(self.POST)) @@ -334,7 +409,7 @@ class Request(threading.local): @property def COOKIES(self): - """Returns a dict with COOKIES.""" + """ Returns a dict with COOKIES. """ if self._COOKIES is None: raw_dict = SimpleCookie(self._environ.get('HTTP_COOKIE','')) self._COOKIES = {} @@ -357,6 +432,7 @@ class Response(threading.local): self.header = HeaderWrapper(self.header_list) self.content_type = 'text/html' self.error = None + self.charset = 'utf8' def wsgiheaders(self): ''' Returns a wsgi conform list of header/value pairs ''' @@ -371,19 +447,33 @@ class Response(threading.local): return self._COOKIES def set_cookie(self, key, value, **kargs): - """ Sets a Cookie. Optional settings: expires, path, comment, domain, max-age, secure, version, httponly """ + """ + Sets a Cookie. Optional settings: + expires, path, comment, domain, max-age, secure, version, httponly + """ self.COOKIES[key] = value - for k in kargs: - self.COOKIES[key][k] = kargs[k] + for k, v in kargs.iteritems(): + self.COOKIES[key][k] = v def get_content_type(self): - '''Gives access to the 'Content-Type' header and defaults to 'text/html'.''' + """ Get the current 'Content-Type' header. """ return self.header['Content-Type'] def set_content_type(self, value): + if 'charset=' in value: + self.charset = value.split('charset=')[-1].split(';')[0].strip() self.header['Content-Type'] = value - - content_type = property(get_content_type, set_content_type, None, get_content_type.__doc__) + + content_type = property(get_content_type, set_content_type, None, + get_content_type.__doc__) + + +class BaseController(object): + _singleton = None + def __new__(cls, *a, **k): + if not cls._singleton: + cls._singleton = object.__new__(cls, *a, **k) + return cls._singleton def abort(code=500, text='Unknown Error: Appliction stopped.'): @@ -398,12 +488,11 @@ def redirect(url, code=307): raise BreakTheBottle("") -def send_file(filename, root, guessmime = True, mimetype = 'text/plain'): +def send_file(filename, root, guessmime = True, mimetype = None): """ Aborts execution and sends a static files as response. """ - root = os.path.abspath(root) + '/' - filename = os.path.normpath(filename).strip('/') - filename = os.path.join(root, filename) - + root = os.path.abspath(root) + os.sep + filename = os.path.abspath(os.path.join(root, filename.strip('/\\'))) + if not filename.startswith(root): abort(401, "Access denied.") if not os.path.exists(filename) or not os.path.isfile(filename): @@ -411,25 +500,41 @@ def send_file(filename, root, guessmime = True, mimetype = 'text/plain'): if not os.access(filename, os.R_OK): abort(401, "You do not have permission to access this file.") - if guessmime: - guess = mimetypes.guess_type(filename)[0] - if guess: - response.content_type = guess - elif mimetype: - response.content_type = mimetype - elif mimetype: - response.content_type = mimetype + if guessmime and not mimetype: + mimetype = mimetypes.guess_type(filename)[0] + if not mimetype: mimetype = 'text/plain' + response.content_type = mimetype stats = os.stat(filename) - # TODO: HTTP_IF_MODIFIED_SINCE -> 304 (Thu, 02 Jul 2009 23:16:31 CEST) - if 'Content-Length' not in response.header: - response.header['Content-Length'] = stats.st_size if 'Last-Modified' not in response.header: - ts = time.gmtime(stats.st_mtime) - ts = time.strftime("%a, %d %b %Y %H:%M:%S +0000", ts) - response.header['Last-Modified'] = ts + lm = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(stats.st_mtime)) + response.header['Last-Modified'] = lm + if 'HTTP_IF_MODIFIED_SINCE' in request.environ: + ims = request.environ['HTTP_IF_MODIFIED_SINCE'] + # IE sends "; length=146" + ims = ims.split(";")[0].strip() + ims = parse_date(ims) + if ims is not None and ims >= stats.st_mtime: + abort(304, "Not modified") + if 'Content-Length' not in response.header: + response.header['Content-Length'] = str(stats.st_size) + raise BreakTheBottle(open(filename, 'rb')) + - raise BreakTheBottle(open(filename, 'r')) +def parse_date(ims): + """ + Parses date strings usually found in HTTP header and returns UTC epoch. + Understands rfc1123, rfc850 and asctime. + """ + try: + ts = email.utils.parsedate_tz(ims) + if ts is not None: + if ts[9] is None: + return time.mktime(ts[:8] + (0,)) - time.timezone + else: + return time.mktime(ts[:8] + (0,)) - ts[9] - time.timezone + except (ValueError, IndexError): + return None @@ -439,15 +544,17 @@ def send_file(filename, root, guessmime = True, mimetype = 'text/plain'): # Decorators def validate(**vkargs): - ''' Validates and manipulates keyword arguments by user defined callables - and handles ValueError and missing arguments by raising HTTPError(403) ''' + """ + Validates and manipulates keyword arguments by user defined callables. + Handles ValueError and missing arguments by raising HTTPError(403). + """ def decorator(func): def wrapper(**kargs): - for key in vkargs: + for key, value in vkargs.iteritems(): if key not in kargs: abort(403, 'Missing parameter: %s' % key) try: - kargs[key] = vkargs[key](kargs[key]) + kargs[key] = value(kargs[key]) except ValueError, e: abort(403, 'Wrong parameter format for: %s' % key) return func(**kargs) @@ -456,12 +563,21 @@ def validate(**vkargs): def route(url, **kargs): - """ Decorator for request handler. Same as add_route(url, handler, **kargs).""" + """ + Decorator for request handler. Same as add_route(url, handler, **kargs). + """ return default_app().route(url, **kargs) +def default(): + """ + Decorator for request handler. Same as set_default(handler). + """ + return default_app().default() def error(code=500): - """ Decorator for error handler. Same as set_error_handler(code, handler).""" + """ + Decorator for error handler. Same as set_error_handler(code, handler). + """ return default_app().error(code) @@ -471,8 +587,23 @@ def error(code=500): # Server adapter -class ServerAdapter(object): +class WSGIAdapter(object): + def run(self, handler): # pragma: no cover + pass + + def __repr__(self): + return "%s()" % (self.__class__.__name__) + + +class CGIServer(WSGIAdapter): + def run(self, handler): + from wsgiref.handlers import CGIHandler + CGIHandler().run(handler) + + +class ServerAdapter(WSGIAdapter): def __init__(self, host='127.0.0.1', port=8080, **kargs): + WSGIAdapter.__init__(self) self.host = host self.port = int(port) self.options = kargs @@ -480,9 +611,6 @@ class ServerAdapter(object): def __repr__(self): return "%s (%s:%d)" % (self.__class__.__name__, self.host, self.port) - def run(self, handler): - pass - class WSGIRefServer(ServerAdapter): def run(self, handler): @@ -513,12 +641,14 @@ class PasteServer(ServerAdapter): class FapwsServer(ServerAdapter): - """ Extreamly fast Webserver using libev (see http://william-os4y.livejournal.com/) - Experimental ... """ + """ + Extremly fast webserver using libev. + See http://william-os4y.livejournal.com/ + Experimental ... + """ def run(self, handler): import fapws._evwsgi as evwsgi from fapws import base - import sys evwsgi.start(self.host, self.port) evwsgi.set_base_module(base) def app(environ, start_response): @@ -528,35 +658,75 @@ class FapwsServer(ServerAdapter): evwsgi.run() -def run(app=None, server=WSGIRefServer, host='127.0.0.1', port=8080, **kargs): - """ Runs bottle as a web server, using Python's built-in wsgiref implementation by default. - - You may choose between WSGIRefServer, CherryPyServer, FlupServer and - PasteServer or write your own server adapter. - """ +def run(app=None, server=WSGIRefServer, host='127.0.0.1', port=8080, + interval=1, reloader=False, **kargs): + """ Runs bottle as a web server. """ if not app: app = default_app() - quiet = bool('quiet' in kargs and kargs['quiet']) - - # Instanciate server, if it is a class instead of an instance - if isinstance(server, type) and issubclass(server, ServerAdapter): - server = server(host=host, port=port, **kargs) + quiet = bool(kargs.get('quiet', False)) - if not isinstance(server, ServerAdapter): - raise RuntimeError("Server must be a subclass of ServerAdapter") + # Instantiate server, if it is a class instead of an instance + if isinstance(server, type): + if issubclass(server, CGIServer): + server = server() + elif issubclass(server, ServerAdapter): + server = server(host=host, port=port, **kargs) - if not quiet: - print 'Bottle server starting up (using %s)...' % repr(server) - print 'Listening on http://%s:%d/' % (server.host, server.port) - print 'Use Ctrl-C to quit.' - print + if not isinstance(server, WSGIAdapter): + raise RuntimeError("Server must be a subclass of WSGIAdapter") + + if not quiet and isinstance(server, ServerAdapter): # pragma: no cover + if not reloader or os.environ.get('BOTTLE_CHILD') == 'true': + print "Bottle server starting up (using %s)..." % repr(server) + print "Listening on http://%s:%d/" % (server.host, server.port) + print "Use Ctrl-C to quit." + print + else: + print "Bottle auto reloader starting up..." try: - server.run(app) + if reloader and interval: + reloader_run(server, app, interval) + else: + server.run(app) except KeyboardInterrupt: - print "Shuting down..." - + if not quiet: # pragma: no cover + print "Shutting Down..." + + +#TODO: If the parent process is killed (with SIGTERM) the childs survive... +def reloader_run(server, app, interval): + if os.environ.get('BOTTLE_CHILD') == 'true': + # We are a child process + files = dict() + for module in sys.modules.values(): + file_path = getattr(module, '__file__', None) + if file_path and os.path.isfile(file_path): + file_split = os.path.splitext(file_path) + if file_split[1] in ('.py', '.pyc', '.pyo'): + file_path = file_split[0] + '.py' + files[file_path] = os.stat(file_path).st_mtime + thread.start_new_thread(server.run, (app,)) + while True: + time.sleep(interval) + for file_path, file_mtime in files.iteritems(): + if not os.path.exists(file_path): + print "File changed: %s (deleted)" % file_path + elif os.stat(file_path).st_mtime > file_mtime: + print "File changed: %s (modified)" % file_path + else: continue + print "Restarting..." + app.serve = False + time.sleep(interval) # be nice and wait for running requests + sys.exit(3) + while True: + args = [sys.executable] + sys.argv + environ = os.environ.copy() + environ['BOTTLE_CHILD'] = 'true' + exit_status = subprocess.call(args, env=environ) + if exit_status != 3: + sys.exit(exit_status) @@ -564,89 +734,183 @@ def run(app=None, server=WSGIRefServer, host='127.0.0.1', port=8080, **kargs): # Templates +class TemplateError(HTTPError): + def __init__(self, message): + HTTPError.__init__(self, 500, message) + class BaseTemplate(object): - def __init__(self, template='', filename=None): - self.source = filename - if self.source: - fp = open(filename) - template = fp.read() - fp.close() - self.parse(template) - - def parse(self, template): raise NotImplementedError - def render(self, **args): raise NotImplementedError - - @classmethod - def find(cls, name): - for path in TEMPLATE_PATH: - if os.path.isfile(path % name): - return cls(filename = path % name) - return None + def __init__(self, template='', name=None, filename=None, lookup=[]): + """ + Create a new template. + If a name is provided, but no filename and no template string, the + filename is guessed using the lookup path list. + Subclasses can assume that either self.template or self.filename is set. + If both are present, self.template should be used. + """ + self.name = name + self.filename = filename + self.template = template + self.lookup = lookup + if self.name and not self.filename: + for path in self.lookup: + fpath = os.path.join(path, self.name+'.tpl') + if os.path.isfile(fpath): + self.filename = fpath + if not self.template and not self.filename: + raise TemplateError('Template (%s) not found.' % self.name) + self.prepare() + + def prepare(self): + """ + Run preparatios (parsing, caching, ...). + It should be possible to call this multible times to refresh a template. + """ + raise NotImplementedError + + def render(self, **args): + """ + Render the template with the specified local variables and return an + iterator of strings (bytes). This must be thread save! + """ + raise NotImplementedError class MakoTemplate(BaseTemplate): - def parse(self, template): + output_encoding=None + input_encoding=None + default_filters=None + global_variables={} + + def prepare(self): from mako.template import Template - self.tpl = Template(template) + from mako.lookup import TemplateLookup + #TODO: This is a hack... http://github.com/defnull/bottle/issues#issue/8 + mylookup = TemplateLookup(directories=map(os.path.abspath, self.lookup)+['./']) + if self.template: + self.tpl = Template(self.template, + lookup=mylookup, + output_encoding=MakoTemplate.output_encoding, + input_encoding=MakoTemplate.input_encoding, + default_filters=MakoTemplate.default_filters + ) + else: + self.tpl = Template(filename=self.filename, + lookup=mylookup, + output_encoding=MakoTemplate.output_encoding, + input_encoding=MakoTemplate.input_encoding, + default_filters=MakoTemplate.default_filters + ) def render(self, **args): - return self.tpl.render(**args) + _defaults = MakoTemplate.global_variables.copy() + _defaults.update(args) + return self.tpl.render(**_defaults) class CheetahTemplate(BaseTemplate): - def parse(self, template): + def prepare(self): from Cheetah.Template import Template self.context = threading.local() self.context.vars = {} - self.tpl = Template(source = template, searchList=[self.context.vars]) + if self.template: + self.tpl = Template(source=self.template, searchList=[self.context.vars]) + else: + self.tpl = Template(file=self.filename, searchList=[self.context.vars]) def render(self, **args): self.context.vars.update(args) out = str(self.tpl) self.context.vars.clear() - return out + return [out] + + +class Jinja2Template(BaseTemplate): + env = None # hopefully, a Jinja environment is actually thread-safe + + def prepare(self): + if not self.env: + from jinja2 import Environment, FunctionLoader + self.env = Environment(line_statement_prefix="#", loader=FunctionLoader(self.loader)) + if self.template: + self.tpl = self.env.from_string(self.template) + else: + self.tpl = self.env.get_template(self.filename) + + def render(self, **args): + return self.tpl.render(**args).encode("utf-8") + + def loader(self, name): + if not name.endswith(".tpl"): + for path in self.lookup: + fpath = os.path.join(path, name+'.tpl') + if os.path.isfile(fpath): + name = fpath + break + f = open(name) + try: return f.read() + finally: f.close() class SimpleTemplate(BaseTemplate): - re_python = re.compile(r'^\s*%\s*(?:(if|elif|else|try|except|finally|for|while|with|def|class)|(include)|(end)|(.*))') + re_python = re.compile(r'^\s*%\s*(?:(if|elif|else|try|except|finally|for|' + 'while|with|def|class)|(include|rebase)|(end)|(.*))') re_inline = re.compile(r'\{\{(.*?)\}\}') dedent_keywords = ('elif', 'else', 'except', 'finally') + def prepare(self): + if self.template: + code = self.translate(self.template) + self.co = compile(code, '', 'exec') + else: + code = self.translate(open(self.filename).read()) + self.co = compile(code, self.filename, 'exec') + def translate(self, template): indent = 0 strbuffer = [] code = [] - self.subtemplates = {} + self.includes = dict() class PyStmt(str): def __repr__(self): return 'str(' + self + ')' def flush(allow_nobreak=False): if len(strbuffer): if allow_nobreak and strbuffer[-1].endswith("\\\\\n"): strbuffer[-1]=strbuffer[-1][:-3] - code.append(" " * indent + "stdout.append(%s)" % repr(''.join(strbuffer))) - code.append((" " * indent + "\n") * len(strbuffer)) # to preserve line numbers + code.append(' ' * indent + "_stdout.append(%s)" % repr(''.join(strbuffer))) + code.append((' ' * indent + '\n') * len(strbuffer)) # to preserve line numbers del strbuffer[:] for line in template.splitlines(True): m = self.re_python.match(line) if m: flush(allow_nobreak=True) - keyword, include, end, statement = m.groups() + keyword, subtpl, end, statement = m.groups() if keyword: if keyword in self.dedent_keywords: indent -= 1 code.append(" " * indent + line[m.start(1):]) indent += 1 - elif include: + elif subtpl: tmp = line[m.end(2):].strip().split(None, 1) - name = tmp[0] - args = tmp[1:] and tmp[1] or '' - self.subtemplates[name] = SimpleTemplate.find(name) - code.append(" " * indent + "stdout.append(_subtemplates[%s].render(%s))\n" % (repr(name), args)) + if not tmp: + code.append(' ' * indent + "_stdout.extend(_base)\n") + else: + name = tmp[0] + args = tmp[1:] and tmp[1] or '' + if name not in self.includes: + self.includes[name] = SimpleTemplate(name=name, lookup=self.lookup) + if subtpl == 'include': + code.append(' ' * indent + + "_ = _includes[%s].execute(_stdout, %s)\n" + % (repr(name), args)) + else: + code.append(' ' * indent + + "_tpl['_rebase'] = (_includes[%s], dict(%s))\n" + % (repr(name), args)) elif end: indent -= 1 - code.append(" " * indent + '#' + line[m.start(3):]) + code.append(' ' * indent + '#' + line[m.start(3):]) elif statement: - code.append(" " * indent + line[m.start(4):]) + code.append(' ' * indent + line[m.start(4):]) else: splits = self.re_inline.split(line) # text, (expr, text)* if len(splits) == 1: @@ -656,40 +920,86 @@ class SimpleTemplate(BaseTemplate): for i in range(1, len(splits), 2): splits[i] = PyStmt(splits[i]) splits = [x for x in splits if bool(x)] - code.append(" " * indent + "stdout.extend(%s)\n" % repr(splits)) + code.append(' ' * indent + "_stdout.extend(%s)\n" % repr(splits)) flush() return ''.join(code) - def parse(self, template): - code = self.translate(template) - self.co = compile(code, self.source or '