diff options
author | cp1 <christopherpool1@googlemail.com> | 2009-06-11 16:50:48 +0200 |
---|---|---|
committer | cp1 <christopherpool1@googlemail.com> | 2009-06-11 16:50:48 +0200 |
commit | fd6e286ee089cf75276f2581471aba254d69fcb5 (patch) | |
tree | 1f91e2063f3182d172d7c7f70c4fdb98b407a60b /module/network | |
parent | add wxversion selector to ensure wxWidgets 2.8 is used (diff) | |
download | pyload-fd6e286ee089cf75276f2581471aba254d69fcb5.tar.xz |
correcting false identation (tabs to spaces)
Diffstat (limited to 'module/network')
-rw-r--r-- | module/network/Keepalive.py | 46 | ||||
-rw-r--r-- | module/network/MultipartPostHandler.py | 11 | ||||
-rwxr-xr-x | module/network/Request.py | 34 | ||||
-rw-r--r-- | module/network/sslfactory.py | 12 |
4 files changed, 51 insertions, 52 deletions
diff --git a/module/network/Keepalive.py b/module/network/Keepalive.py index 68abe087d..dbf4d94cb 100644 --- a/module/network/Keepalive.py +++ b/module/network/Keepalive.py @@ -9,9 +9,9 @@ # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, +# License along with this library; if not, write to the +# Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, # Boston, MA 02111-1307 USA # This file is part of urlgrabber, a high-level cross-protocol url-grabber @@ -24,7 +24,7 @@ >>> keepalive_handler = HTTPHandler() >>> opener = urllib2.build_opener(keepalive_handler) >>> urllib2.install_opener(opener) ->>> +>>> >>> fo = urllib2.urlopen('http://www.python.org') If a connection to a given host is requested, and all of the existing @@ -113,7 +113,7 @@ import sslfactory import sys if sys.version_info < (2, 4): HANDLE_ERRORS = 1 else: HANDLE_ERRORS = 0 - + class ConnectionManager: """ The connection manager must be able to: @@ -153,7 +153,7 @@ class ConnectionManager: def set_ready(self, connection, ready): try: self._readymap[connection] = ready except KeyError: pass - + def get_ready_conn(self, host): conn = None self._lock.acquire() @@ -177,7 +177,7 @@ class ConnectionManager: class KeepAliveHandler: def __init__(self): self._cm = ConnectionManager() - + #### Connection Management def open_connections(self): """return a list of connected hosts and the number of connections @@ -191,14 +191,14 @@ class KeepAliveHandler: for h in self._cm.get_all(host): self._cm.remove(h) h.close() - + def close_all(self): """close all open connections""" for host, conns in self._cm.get_all().items(): for h in conns: self._cm.remove(h) h.close() - + def _request_closed(self, request, host, connection): """tells us that this request is now closed and the the connection is ready for another request""" @@ -207,7 +207,7 @@ class KeepAliveHandler: def _remove_connection(self, host, connection, close=0): if close: connection.close() self._cm.remove(connection) - + #### Transaction Execution def do_open(self, req): host = req.get_host() @@ -238,7 +238,7 @@ class KeepAliveHandler: r = h.getresponse() except (socket.error, httplib.HTTPException), err: raise urllib2.URLError(err) - + # if not a persistent connection, don't try to reuse it if r.will_close: self._cm.remove(h) @@ -250,7 +250,7 @@ class KeepAliveHandler: r.code = r.status r.headers = r.msg r.msg = r.reason - + if r.status == 200 or not HANDLE_ERRORS: return r else: @@ -286,7 +286,7 @@ class KeepAliveHandler: self._cm.remove(h) h.close() raise - + if r is None or r.version == 9: # httplib falls back to assuming HTTP 0.9 if it gets a # bad header back. This is most likely to happen if @@ -342,7 +342,7 @@ class HTTPSHandler(KeepAliveHandler, urllib2.HTTPSHandler): if not ssl_factory: ssl_factory = sslfactory.get_factory() self._ssl_factory = ssl_factory - + def https_open(self, req): return self.do_open(req) @@ -367,7 +367,7 @@ class HTTPResponse(httplib.HTTPResponse): # although read() never adds to the buffer. # Both readline and readlines have been stolen with almost no # modification from socket.py - + def __init__(self, sock, debuglevel=0, strict=0, method=None): if method: # the httplib in python 2.3 uses the method arg @@ -396,7 +396,7 @@ class HTTPResponse(httplib.HTTPResponse): def close_connection(self): self._handler._remove_connection(self._host, self._connection, close=1) self.close() - + def info(self): return self.headers @@ -453,7 +453,7 @@ class HTTPConnection(httplib.HTTPConnection): class HTTPSConnection(httplib.HTTPSConnection): response_class = HTTPResponse - + ######################################################################### ##### TEST FUNCTIONS ######################################################################### @@ -487,7 +487,7 @@ def error_handler(url): def continuity(url): import md5 format = '%25s: %s' - + # first fetch the file with the normal http handler opener = urllib2.build_opener() urllib2.install_opener(opener) @@ -534,7 +534,7 @@ def comp(N, url): t2 = fetch(N, url) print ' TIME: %.3f s' % t2 print ' improvement factor: %.2f' % (t1/t2, ) - + def fetch(N, url, delay=0): import time lens = [] @@ -566,7 +566,7 @@ def test_timeout(url): fo = urllib2.urlopen(url) data1 = fo.read() fo.close() - + i = 20 print " waiting %i seconds for the server to close the connection" % i while i > 0: @@ -588,7 +588,7 @@ def test_timeout(url): DEBUG = dbbackup - + def test(url, N=10): print "checking error hander (do this on a non-200)" try: error_handler(url) @@ -604,7 +604,7 @@ def test(url, N=10): print print "performing dropped-connection check" test_timeout(url) - + if __name__ == '__main__': import time import sys @@ -614,4 +614,4 @@ if __name__ == '__main__': except: print "%s <integer> <url>" % sys.argv[0] else: - test(url, N) + test(url, N)
\ No newline at end of file diff --git a/module/network/MultipartPostHandler.py b/module/network/MultipartPostHandler.py index f25c6fa1b..b6980ae18 100644 --- a/module/network/MultipartPostHandler.py +++ b/module/network/MultipartPostHandler.py @@ -2,18 +2,18 @@ #### # 02/2006 Will Holcomb <wholcomb@gmail.com> -# +# # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # -# 7/26/07 Slightly modified by Brian Schneider +# 7/26/07 Slightly modified by Brian Schneider # in order to support unicode files ( multipart_encode function ) """ Usage: @@ -84,7 +84,7 @@ class MultipartPostHandler(urllib2.BaseHandler): request.add_unredirected_header('Content-Type', contenttype) request.add_data(data) - + return request def multipart_encode(vars, files, boundary = None, buf = None): @@ -135,5 +135,4 @@ def main(): validateFile("http://www.google.com") if __name__=="__main__": - main() - + main()
\ No newline at end of file diff --git a/module/network/Request.py b/module/network/Request.py index de9000ab3..2a0199fb8 100755 --- a/module/network/Request.py +++ b/module/network/Request.py @@ -20,7 +20,7 @@ from cStringIO import StringIO additionaly you can firstly pass the get and secondly the post data in form of a dictonary when the last argument is true the handler simulate a http referer with the last called url. retrieveUrl returns response as string - + """ class Request: def __init__(self): @@ -38,13 +38,13 @@ class Request: self.opener = urllib2.build_opener(handler, urllib2.HTTPCookieProcessor(self.cj)) self.downloader = urllib2.build_opener() #self.opener.add_handler() - + self.opener.addheaders = [ ("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.10"), ("Accept-Encoding", "gzip,deflate"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7"), - ("Connection", "keep-alive"), + ("Connection", "keep-alive"), ("Keep-Alive", "300")] self.downloader.addheaders = [ @@ -52,20 +52,20 @@ class Request: ("Accept-Encoding", "gzip,deflate"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7")] - - + + def load(self, url, get={}, post={}, ref=True, cookies=False): - + if post: post = urllib.urlencode(post) else: post = None - + if get: get = urllib.urlencode(get) else: get = "" - + url = url + get req = urllib2.Request(url, data=post) @@ -85,13 +85,13 @@ class Request: self.cookies.append(cookie) output = rep.read() - + if rep.headers.has_key("content-encoding"): if rep.headers["content-encoding"] == "gzip": output = GzipFile('', 'r', 0, StringIO(output)).read() - + self.lastURL = url - + return output def add_auth(self, user, pw): @@ -111,12 +111,12 @@ class Request: self.downloader.add_handler(handler) def download(self, url, filename, post={}): - + if post: post = urllib.urlencode(post) else: post = None - + if not self.dl: self.dl = True file = open(filename, 'wb') @@ -128,14 +128,14 @@ class Request: self.dl_size = 0 self.dl_arrived = 0 self.dl_time = time.time() - for chunk in conn: + for chunk in conn: self.dl_arrived += len(chunk) file.write(chunk) file.close() self.dl = False self.dl_finished = time.time() return True - + def get_speed(self): try: return (self.dl_arrived / ((time.time() if self.dl else self.dl_finished) - self.dl_time)) / 1024 @@ -144,7 +144,7 @@ class Request: def get_ETA(self): try: - return (self.dl_size - self.dl_arrived) / (self.dl_arrived / (time.time() - self.dl_time)) + return (self.dl_size - self.dl_arrived) / (self.dl_arrived / (time.time() - self.dl_time)) except: return 0 @@ -153,4 +153,4 @@ class Request: if __name__ == "__main__": import doctest - doctest.testmod() + doctest.testmod()
\ No newline at end of file diff --git a/module/network/sslfactory.py b/module/network/sslfactory.py index f7e6d3d7e..14903cd2a 100644 --- a/module/network/sslfactory.py +++ b/module/network/sslfactory.py @@ -9,9 +9,9 @@ # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, +# License along with this library; if not, write to the +# Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, # Boston, MA 02111-1307 USA # This file is part of urlgrabber, a high-level cross-protocol url-grabber @@ -31,7 +31,7 @@ except ImportError: DEBUG = None if have_m2crypto: - + class M2SSLFactory: def __init__(self, ssl_ca_cert, ssl_context): @@ -74,7 +74,7 @@ class SSLFactory: def create_opener(self, *handlers): return urllib2.build_opener(*handlers) - + def get_factory(ssl_ca_cert = None, ssl_context = None): """ Return an SSLFactory, based on if M2Crypto is available. """ @@ -86,4 +86,4 @@ def get_factory(ssl_ca_cert = None, ssl_context = None): if DEBUG: DEBUG.warning("SSL arguments supplied, but M2Crypto is not available. " "Using Python SSL.") - return SSLFactory() + return SSLFactory()
\ No newline at end of file |