diff options
Diffstat (limited to 'module/network')
| -rw-r--r-- | module/network/Browser.py | 9 | ||||
| -rw-r--r-- | module/network/Bucket.py | 12 | ||||
| -rw-r--r-- | module/network/CookieJar.py | 20 | ||||
| -rw-r--r-- | module/network/HTTPChunk.py | 25 | ||||
| -rw-r--r-- | module/network/HTTPDownload.py | 31 | ||||
| -rw-r--r-- | module/network/HTTPRequest.py | 38 | ||||
| -rw-r--r-- | module/network/RequestFactory.py | 31 | ||||
| -rw-r--r-- | module/network/XDCCRequest.py | 2 |
8 files changed, 96 insertions, 72 deletions
diff --git a/module/network/Browser.py b/module/network/Browser.py index d68a23687..9cf6c2f30 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -16,7 +16,7 @@ class Browser(object): self.options = options #holds pycurl options self.bucket = bucket - self.cj = None # needs to be setted later + self.cj = None # needs to be set later self._size = 0 self.renewHTTPRequest() @@ -55,6 +55,13 @@ class Browser(object): return 0 @property + def name(self): + if self.dl: + return self.dl.name + else: + return "" + + @property def arrived(self): if self.dl: return self.dl.arrived diff --git a/module/network/Bucket.py b/module/network/Bucket.py index 69da277ae..db67faa4a 100644 --- a/module/network/Bucket.py +++ b/module/network/Bucket.py @@ -20,15 +20,18 @@ from time import time from threading import Lock +# 10kb minimum rate +MIN_RATE = 10240 + class Bucket: def __init__(self): - self.rate = 0 + self.rate = 0 # bytes per second, maximum targeted throughput self.tokens = 0 self.timestamp = time() self.lock = Lock() def __nonzero__(self): - return False if self.rate < 10240 else True + return False if self.rate < MIN_RATE else True def setRate(self, rate): self.lock.acquire() @@ -36,8 +39,8 @@ class Bucket: self.lock.release() def consumed(self, amount): - """ return time the process have to sleep, after consumed specified amount """ - if self.rate < 10240: return 0 #min. 10kb, may become unresponsive otherwise + """ return the time the process has to sleep, after it consumed a specified amount """ + if self.rate < MIN_RATE: return 0 #May become unresponsive otherwise self.lock.acquire() self.calc_tokens() @@ -47,7 +50,6 @@ class Bucket: time = -self.tokens/float(self.rate) else: time = 0 - self.lock.release() return time diff --git a/module/network/CookieJar.py b/module/network/CookieJar.py index c05812334..a020d6f9e 100644 --- a/module/network/CookieJar.py +++ b/module/network/CookieJar.py @@ -20,10 +20,12 @@ from time import time class CookieJar(): - def __init__(self, pluginname, account=None): + def __init__(self, pluginname): self.cookies = {} - self.plugin = pluginname - self.account = account + self.pluginname = pluginname + + def __repr__(self): + return ("<CookieJar plugin=%s>\n\t" % self.pluginname) + "\n\t".join(self.cookies.values()) def addCookies(self, clist): for c in clist: @@ -33,18 +35,18 @@ class CookieJar(): def getCookies(self): return self.cookies.values() - def parseCookie(self, name): + def getCookie(self, name): if name in self.cookies: return self.cookies[name].split("\t")[6] else: return None - def getCookie(self, name): - return self.parseCookie(name) + def setCookie(self, domain, name, value, path="/", exp=None): + if not exp: exp = time() + 3600 * 24 * 180 - def setCookie(self, domain, name, value, path="/", exp=time()+3600*24*180): - s = ".%s TRUE %s FALSE %s %s %s" % (domain, path, exp, name, value) + # dot makes it valid on all subdomains + s = ".%s TRUE %s FALSE %s %s %s" % (domain.strip("."), path, exp, name, value) self.cookies[name] = s def clear(self): - self.cookies = {} + self.cookies = {}
\ No newline at end of file diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index b637aef32..d17177ee7 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -20,10 +20,13 @@ from os import remove, stat, fsync from os.path import exists from time import sleep from re import search -from module.utils import fs_encode + import codecs import pycurl +from module.utils import remove_chars +from module.utils.fs import fs_encode + from HTTPRequest import HTTPRequest class WrongFormat(Exception): @@ -204,7 +207,7 @@ class HTTPChunk(HTTPRequest): def writeHeader(self, buf): self.header += buf - #@TODO forward headers?, this is possibly unneeeded, when we just parse valid 200 headers + #@TODO forward headers?, this is possibly unneeded, when we just parse valid 200 headers # as first chunk, we will parse the headers if not self.range and self.header.endswith("\r\n\r\n"): self.parseHeader() @@ -233,8 +236,8 @@ class HTTPChunk(HTTPRequest): sleep(self.p.bucket.consumed(size)) else: # Avoid small buffers, increasing sleep time slowly if buffer size gets smaller - # otherwise reduce sleep time percentual (values are based on tests) - # So in general cpu time is saved without reducing bandwith too much + # otherwise reduce sleep time percentile (values are based on tests) + # So in general cpu time is saved without reducing bandwidth too much if size < self.lastSize: self.sleep += 0.002 @@ -250,17 +253,19 @@ class HTTPChunk(HTTPRequest): def parseHeader(self): - """parse data from recieved header""" + """parse data from received header""" for orgline in self.decodeResponse(self.header).splitlines(): line = orgline.strip().lower() if line.startswith("accept-ranges") and "bytes" in line: self.p.chunkSupport = True - if line.startswith("content-disposition") and "filename=" in line: - name = orgline.partition("filename=")[2] - name = name.replace('"', "").replace("'", "").replace(";", "").strip() - self.p.nameDisposition = name - self.log.debug("Content-Disposition: %s" % name) + if "content-disposition" in line: + + m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", line) + if m: + name = remove_chars(m.groupdict()['name'], "\"';").strip() + self.p._name = name + self.log.debug("Content-Disposition: %s" % name) if not self.resume and line.startswith("content-length"): self.p.size = int(line.split(":")[1]) diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index fe8075539..c6d2e1547 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -17,9 +17,9 @@ @author: RaNaN """ -from os import remove, fsync +from os import remove from os.path import dirname -from time import sleep, time +from time import time from shutil import move from logging import getLogger @@ -28,11 +28,13 @@ import pycurl from HTTPChunk import ChunkInfo, HTTPChunk from HTTPRequest import BadHeader -from module.plugins.Plugin import Abort -from module.utils import save_join, fs_encode +from module.plugins.Base import Abort +from module.utils.fs import save_join, fs_encode + +# TODO: save content-disposition for resuming class HTTPDownload(): - """ loads a url http + ftp """ + """ loads an url, http + ftp supported """ def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, options={}, progressNotify=None, disposition=False): @@ -49,7 +51,7 @@ class HTTPDownload(): self.abort = False self.size = 0 - self.nameDisposition = None #will be parsed from content disposition + self._name = ""# will be parsed from content disposition self.chunks = [] @@ -87,6 +89,10 @@ class HTTPDownload(): if not self.size: return 0 return (self.arrived * 100) / self.size + @property + def name(self): + return self._name if self.disposition else "" + def _copyChunks(self): init = fs_encode(self.info.getChunkName(0)) #initial chunk name @@ -113,8 +119,8 @@ class HTTPDownload(): remove(fname) #remove chunk fo.close() - if self.nameDisposition and self.disposition: - self.filename = save_join(dirname(self.filename), self.nameDisposition) + if self.name: + self.filename = save_join(dirname(self.filename), self.name) move(init, fs_encode(self.filename)) self.info.remove() #remove info file @@ -144,8 +150,7 @@ class HTTPDownload(): finally: self.close() - if self.nameDisposition and self.disposition: return self.nameDisposition - return None + return self.name def _download(self, chunks, resume): if not resume: @@ -169,7 +174,7 @@ class HTTPDownload(): while 1: #need to create chunks - if not chunksCreated and self.chunkSupport and self.size: #will be setted later by first chunk + if not chunksCreated and self.chunkSupport and self.size: #will be set later by first chunk if not resume: self.info.setSize(self.size) @@ -188,7 +193,7 @@ class HTTPDownload(): self.chunks.append(c) self.m.add_handle(handle) else: - #close immediatly + #close immediately self.log.debug("Invalid curl handle -> closed") c.close() @@ -286,7 +291,7 @@ class HTTPDownload(): if self.abort: raise Abort() - #sleep(0.003) #supress busy waiting - limits dl speed to (1 / x) * buffersize + #sleep(0.003) #suppress busy waiting - limits dl speed to (1 / x) * buffersize self.m.select(1) for chunk in self.chunks: diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 4747d937f..4b45e10a2 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -25,21 +25,21 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Plugin import Abort +from module.plugins.Base import Abort def myquote(url): - return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") + return quote(url.encode('utf8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") def myurlencode(data): data = dict(data) - return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ - y.encode('utf_8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) + return urlencode(dict((x.encode('utf8') if isinstance(x, unicode) else x, \ + y.encode('utf8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) bad_headers = range(400, 404) + range(405, 418) + range(500, 506) class BadHeader(Exception): def __init__(self, code, content=""): - Exception.__init__(self, "Bad server response: %s %s" % (code, responses[int(code)])) + Exception.__init__(self, "Bad server response: %s %s" % (code, responses.get(int(code), "Unknown Header"))) self.code = code self.content = content @@ -62,6 +62,7 @@ class HTTPRequest(): self.initHandle() self.setInterface(options) + self.setOptions(options) self.c.setopt(pycurl.WRITEFUNCTION, self.write) self.c.setopt(pycurl.HEADERFUNCTION, self.writeHeader) @@ -79,7 +80,8 @@ class HTTPRequest(): if hasattr(pycurl, "AUTOREFERER"): self.c.setopt(pycurl.AUTOREFERER, 1) self.c.setopt(pycurl.SSL_VERIFYPEER, 0) - self.c.setopt(pycurl.LOW_SPEED_TIME, 30) + # Interval for low speed, detects connection loss, but can abort dl if hoster stalls the download + self.c.setopt(pycurl.LOW_SPEED_TIME, 45) self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5) #self.c.setopt(pycurl.VERBOSE, 1) @@ -127,6 +129,11 @@ class HTTPRequest(): if "timeout" in options: self.c.setopt(pycurl.LOW_SPEED_TIME, options["timeout"]) + def setOptions(self, options): + """ Sets same options as available in pycurl """ + for k, v in options.iteritems(): + if hasattr(pycurl, k): + self.c.setopt(getattr(pycurl, k), v) def addCookies(self): """ put cookies from curl handle to cj """ @@ -193,11 +200,20 @@ class HTTPRequest(): if just_header: self.c.setopt(pycurl.FOLLOWLOCATION, 0) self.c.setopt(pycurl.NOBODY, 1) - self.c.perform() - rep = self.header - self.c.setopt(pycurl.FOLLOWLOCATION, 1) - self.c.setopt(pycurl.NOBODY, 0) + # overwrite HEAD request, we want a common request type + if post: + self.c.setopt(pycurl.CUSTOMREQUEST, "POST") + else: + self.c.setopt(pycurl.CUSTOMREQUEST, "GET") + + try: + self.c.perform() + rep = self.header + finally: + self.c.setopt(pycurl.FOLLOWLOCATION, 1) + self.c.setopt(pycurl.NOBODY, 0) + self.c.unsetopt(pycurl.CUSTOMREQUEST) else: self.c.perform() @@ -262,7 +278,7 @@ class HTTPRequest(): #TODO: html_unescape as default except LookupError: - self.log.debug("No Decoder foung for %s" % encoding) + self.log.debug("No Decoder found for %s" % encoding) except Exception: self.log.debug("Error when decoding string from %s." % encoding) diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 5b1528281..932184678 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -24,38 +24,29 @@ from Bucket import Bucket from HTTPRequest import HTTPRequest from CookieJar import CookieJar -from XDCCRequest import XDCCRequest - class RequestFactory(): def __init__(self, core): self.lock = Lock() self.core = core self.bucket = Bucket() self.updateBucket() - self.cookiejars = {} + @property def iface(self): return self.core.config["download"]["interface"] - def getRequest(self, pluginName, account=None, type="HTTP"): - self.lock.acquire() - - if type == "XDCC": - return XDCCRequest(proxies=self.getProxies()) - + def getRequest(self, pluginName, cj=None): req = Browser(self.bucket, self.getOptions()) - if account: - cj = self.getCookieJar(pluginName, account) + if cj: req.setCookieJar(cj) else: req.setCookieJar(CookieJar(pluginName)) - self.lock.release() return req def getHTTPRequest(self, **kwargs): - """ returns a http request, dont forget to close it ! """ + """ returns a http request, don't forget to close it ! """ options = self.getOptions() options.update(kwargs) # submit kwargs as additional options return HTTPRequest(CookieJar(None), options) @@ -67,16 +58,12 @@ class RequestFactory(): rep = h.load(*args, **kwargs) finally: h.close() - - return rep - def getCookieJar(self, pluginName, account=None): - if (pluginName, account) in self.cookiejars: - return self.cookiejars[(pluginName, account)] + return rep - cj = CookieJar(pluginName, account) - self.cookiejars[(pluginName, account)] = cj - return cj + def openCookieJar(self, pluginname): + """Create new CookieJar""" + return CookieJar(pluginname) def getProxies(self): """ returns a proxy list for the request classes """ @@ -106,7 +93,7 @@ class RequestFactory(): def getOptions(self): """returns options needed for pycurl""" - return {"interface": self.iface(), + return {"interface": self.iface, "proxies": self.getProxies(), "ipv6": self.core.config["download"]["ipv6"]} diff --git a/module/network/XDCCRequest.py b/module/network/XDCCRequest.py index f03798c17..7a1a98cb5 100644 --- a/module/network/XDCCRequest.py +++ b/module/network/XDCCRequest.py @@ -119,7 +119,7 @@ class XDCCRequest(): fh.write(data) - # acknowledge data by sending number of recceived bytes + # acknowledge data by sending number of received bytes dccsock.send(struct.pack('!I', self.recv)) dccsock.close() |