diff options
Diffstat (limited to 'module/network')
| -rw-r--r-- | module/network/Browser.py | 7 | ||||
| -rw-r--r-- | module/network/Bucket.py | 10 | ||||
| -rw-r--r-- | module/network/CookieJar.py | 20 | ||||
| -rw-r--r-- | module/network/HTTPChunk.py | 17 | ||||
| -rw-r--r-- | module/network/HTTPDownload.py | 21 | ||||
| -rw-r--r-- | module/network/HTTPRequest.py | 30 | ||||
| -rw-r--r-- | module/network/RequestFactory.py | 29 |
7 files changed, 75 insertions, 59 deletions
diff --git a/module/network/Browser.py b/module/network/Browser.py index d68a23687..3452184d8 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -55,6 +55,13 @@ class Browser(object): return 0 @property + def name(self): + if self.dl: + return self.dl.name + else: + return "" + + @property def arrived(self): if self.dl: return self.dl.arrived diff --git a/module/network/Bucket.py b/module/network/Bucket.py index 69da277ae..ff80bda55 100644 --- a/module/network/Bucket.py +++ b/module/network/Bucket.py @@ -20,15 +20,18 @@ from time import time from threading import Lock +# 10kb minimum rate +MIN_RATE = 10240 + class Bucket: def __init__(self): - self.rate = 0 + self.rate = 0 # bytes per second, maximum targeted throughput self.tokens = 0 self.timestamp = time() self.lock = Lock() def __nonzero__(self): - return False if self.rate < 10240 else True + return False if self.rate < MIN_RATE else True def setRate(self, rate): self.lock.acquire() @@ -37,7 +40,7 @@ class Bucket: def consumed(self, amount): """ return time the process have to sleep, after consumed specified amount """ - if self.rate < 10240: return 0 #min. 10kb, may become unresponsive otherwise + if self.rate < MIN_RATE: return 0 #May become unresponsive otherwise self.lock.acquire() self.calc_tokens() @@ -47,7 +50,6 @@ class Bucket: time = -self.tokens/float(self.rate) else: time = 0 - self.lock.release() return time diff --git a/module/network/CookieJar.py b/module/network/CookieJar.py index c05812334..a020d6f9e 100644 --- a/module/network/CookieJar.py +++ b/module/network/CookieJar.py @@ -20,10 +20,12 @@ from time import time class CookieJar(): - def __init__(self, pluginname, account=None): + def __init__(self, pluginname): self.cookies = {} - self.plugin = pluginname - self.account = account + self.pluginname = pluginname + + def __repr__(self): + return ("<CookieJar plugin=%s>\n\t" % self.pluginname) + "\n\t".join(self.cookies.values()) def addCookies(self, clist): for c in clist: @@ -33,18 +35,18 @@ class CookieJar(): def getCookies(self): return self.cookies.values() - def parseCookie(self, name): + def getCookie(self, name): if name in self.cookies: return self.cookies[name].split("\t")[6] else: return None - def getCookie(self, name): - return self.parseCookie(name) + def setCookie(self, domain, name, value, path="/", exp=None): + if not exp: exp = time() + 3600 * 24 * 180 - def setCookie(self, domain, name, value, path="/", exp=time()+3600*24*180): - s = ".%s TRUE %s FALSE %s %s %s" % (domain, path, exp, name, value) + # dot makes it valid on all subdomains + s = ".%s TRUE %s FALSE %s %s %s" % (domain.strip("."), path, exp, name, value) self.cookies[name] = s def clear(self): - self.cookies = {} + self.cookies = {}
\ No newline at end of file diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index b637aef32..3380fb733 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -20,10 +20,13 @@ from os import remove, stat, fsync from os.path import exists from time import sleep from re import search -from module.utils import fs_encode + import codecs import pycurl +from module.utils import remove_chars +from module.utils.fs import fs_encode + from HTTPRequest import HTTPRequest class WrongFormat(Exception): @@ -256,11 +259,13 @@ class HTTPChunk(HTTPRequest): if line.startswith("accept-ranges") and "bytes" in line: self.p.chunkSupport = True - if line.startswith("content-disposition") and "filename=" in line: - name = orgline.partition("filename=")[2] - name = name.replace('"', "").replace("'", "").replace(";", "").strip() - self.p.nameDisposition = name - self.log.debug("Content-Disposition: %s" % name) + if "content-disposition" in line: + + m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", line) + if m: + name = remove_chars(m.groupdict()['name'], "\"';").strip() + self.p._name = name + self.log.debug("Content-Disposition: %s" % name) if not self.resume and line.startswith("content-length"): self.p.size = int(line.split(":")[1]) diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index fe8075539..59d38beee 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -17,9 +17,9 @@ @author: RaNaN """ -from os import remove, fsync +from os import remove from os.path import dirname -from time import sleep, time +from time import time from shutil import move from logging import getLogger @@ -28,8 +28,8 @@ import pycurl from HTTPChunk import ChunkInfo, HTTPChunk from HTTPRequest import BadHeader -from module.plugins.Plugin import Abort -from module.utils import save_join, fs_encode +from module.plugins.Hoster import Abort +from module.utils.fs import save_join, fs_encode class HTTPDownload(): """ loads a url http + ftp """ @@ -49,7 +49,7 @@ class HTTPDownload(): self.abort = False self.size = 0 - self.nameDisposition = None #will be parsed from content disposition + self._name = ""# will be parsed from content disposition self.chunks = [] @@ -87,6 +87,10 @@ class HTTPDownload(): if not self.size: return 0 return (self.arrived * 100) / self.size + @property + def name(self): + return self._name if self.disposition else "" + def _copyChunks(self): init = fs_encode(self.info.getChunkName(0)) #initial chunk name @@ -113,8 +117,8 @@ class HTTPDownload(): remove(fname) #remove chunk fo.close() - if self.nameDisposition and self.disposition: - self.filename = save_join(dirname(self.filename), self.nameDisposition) + if self.name: + self.filename = save_join(dirname(self.filename), self.name) move(init, fs_encode(self.filename)) self.info.remove() #remove info file @@ -144,8 +148,7 @@ class HTTPDownload(): finally: self.close() - if self.nameDisposition and self.disposition: return self.nameDisposition - return None + return self.name def _download(self, chunks, resume): if not resume: diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 40f18f2a5..2f084efb5 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -25,20 +25,21 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Plugin import Abort +from module.plugins.Hoster import Abort def myquote(url): - return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") + return quote(url.encode('utf8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") def myurlencode(data): - return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ - y.encode('utf_8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) + data = dict(data) + return urlencode(dict((x.encode('utf8') if isinstance(x, unicode) else x, \ + y.encode('utf8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) bad_headers = range(400, 404) + range(405, 418) + range(500, 506) class BadHeader(Exception): def __init__(self, code, content=""): - Exception.__init__(self, "Bad server response: %s %s" % (code, responses[int(code)])) + Exception.__init__(self, "Bad server response: %s %s" % (code, responses.get(int(code), "Unknown Header"))) self.code = code self.content = content @@ -192,11 +193,20 @@ class HTTPRequest(): if just_header: self.c.setopt(pycurl.FOLLOWLOCATION, 0) self.c.setopt(pycurl.NOBODY, 1) - self.c.perform() - rep = self.header - self.c.setopt(pycurl.FOLLOWLOCATION, 1) - self.c.setopt(pycurl.NOBODY, 0) + # overwrite HEAD request, we want a common request type + if post: + self.c.setopt(pycurl.CUSTOMREQUEST, "POST") + else: + self.c.setopt(pycurl.CUSTOMREQUEST, "GET") + + try: + self.c.perform() + rep = self.header + finally: + self.c.setopt(pycurl.FOLLOWLOCATION, 1) + self.c.setopt(pycurl.NOBODY, 0) + self.c.unsetopt(pycurl.CUSTOMREQUEST) else: self.c.perform() @@ -261,7 +271,7 @@ class HTTPRequest(): #TODO: html_unescape as default except LookupError: - self.log.debug("No Decoder foung for %s" % encoding) + self.log.debug("No Decoder found for %s" % encoding) except Exception: self.log.debug("Error when decoding string from %s." % encoding) diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 5b1528281..12fd66c95 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -24,34 +24,25 @@ from Bucket import Bucket from HTTPRequest import HTTPRequest from CookieJar import CookieJar -from XDCCRequest import XDCCRequest - class RequestFactory(): def __init__(self, core): self.lock = Lock() self.core = core self.bucket = Bucket() self.updateBucket() - self.cookiejars = {} + @property def iface(self): return self.core.config["download"]["interface"] - def getRequest(self, pluginName, account=None, type="HTTP"): - self.lock.acquire() - - if type == "XDCC": - return XDCCRequest(proxies=self.getProxies()) - + def getRequest(self, pluginName, cj=None): req = Browser(self.bucket, self.getOptions()) - if account: - cj = self.getCookieJar(pluginName, account) + if cj: req.setCookieJar(cj) else: req.setCookieJar(CookieJar(pluginName)) - self.lock.release() return req def getHTTPRequest(self, **kwargs): @@ -67,16 +58,12 @@ class RequestFactory(): rep = h.load(*args, **kwargs) finally: h.close() - - return rep - def getCookieJar(self, pluginName, account=None): - if (pluginName, account) in self.cookiejars: - return self.cookiejars[(pluginName, account)] + return rep - cj = CookieJar(pluginName, account) - self.cookiejars[(pluginName, account)] = cj - return cj + def openCookieJar(self, pluginname): + """Create new CookieJar""" + return CookieJar(pluginname) def getProxies(self): """ returns a proxy list for the request classes """ @@ -106,7 +93,7 @@ class RequestFactory(): def getOptions(self): """returns options needed for pycurl""" - return {"interface": self.iface(), + return {"interface": self.iface, "proxies": self.getProxies(), "ipv6": self.core.config["download"]["ipv6"]} |