From 958bf611f5d9d117f19f824990ec6fd6b537e967 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 22 Dec 2011 23:45:38 +0100 Subject: accountmanager v2, delete your accounts.conf and re-enter them in pyload, new nice debug functions, try core.shell() and core.breakpoint() --- module/network/Bucket.py | 10 ++++++---- module/network/CookieJar.py | 20 +++++++++++--------- module/network/RequestFactory.py | 29 ++++++++--------------------- 3 files changed, 25 insertions(+), 34 deletions(-) (limited to 'module/network') diff --git a/module/network/Bucket.py b/module/network/Bucket.py index 69da277ae..ff80bda55 100644 --- a/module/network/Bucket.py +++ b/module/network/Bucket.py @@ -20,15 +20,18 @@ from time import time from threading import Lock +# 10kb minimum rate +MIN_RATE = 10240 + class Bucket: def __init__(self): - self.rate = 0 + self.rate = 0 # bytes per second, maximum targeted throughput self.tokens = 0 self.timestamp = time() self.lock = Lock() def __nonzero__(self): - return False if self.rate < 10240 else True + return False if self.rate < MIN_RATE else True def setRate(self, rate): self.lock.acquire() @@ -37,7 +40,7 @@ class Bucket: def consumed(self, amount): """ return time the process have to sleep, after consumed specified amount """ - if self.rate < 10240: return 0 #min. 10kb, may become unresponsive otherwise + if self.rate < MIN_RATE: return 0 #May become unresponsive otherwise self.lock.acquire() self.calc_tokens() @@ -47,7 +50,6 @@ class Bucket: time = -self.tokens/float(self.rate) else: time = 0 - self.lock.release() return time diff --git a/module/network/CookieJar.py b/module/network/CookieJar.py index c05812334..a020d6f9e 100644 --- a/module/network/CookieJar.py +++ b/module/network/CookieJar.py @@ -20,10 +20,12 @@ from time import time class CookieJar(): - def __init__(self, pluginname, account=None): + def __init__(self, pluginname): self.cookies = {} - self.plugin = pluginname - self.account = account + self.pluginname = pluginname + + def __repr__(self): + return ("\n\t" % self.pluginname) + "\n\t".join(self.cookies.values()) def addCookies(self, clist): for c in clist: @@ -33,18 +35,18 @@ class CookieJar(): def getCookies(self): return self.cookies.values() - def parseCookie(self, name): + def getCookie(self, name): if name in self.cookies: return self.cookies[name].split("\t")[6] else: return None - def getCookie(self, name): - return self.parseCookie(name) + def setCookie(self, domain, name, value, path="/", exp=None): + if not exp: exp = time() + 3600 * 24 * 180 - def setCookie(self, domain, name, value, path="/", exp=time()+3600*24*180): - s = ".%s TRUE %s FALSE %s %s %s" % (domain, path, exp, name, value) + # dot makes it valid on all subdomains + s = ".%s TRUE %s FALSE %s %s %s" % (domain.strip("."), path, exp, name, value) self.cookies[name] = s def clear(self): - self.cookies = {} + self.cookies = {} \ No newline at end of file diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 5b1528281..12fd66c95 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -24,34 +24,25 @@ from Bucket import Bucket from HTTPRequest import HTTPRequest from CookieJar import CookieJar -from XDCCRequest import XDCCRequest - class RequestFactory(): def __init__(self, core): self.lock = Lock() self.core = core self.bucket = Bucket() self.updateBucket() - self.cookiejars = {} + @property def iface(self): return self.core.config["download"]["interface"] - def getRequest(self, pluginName, account=None, type="HTTP"): - self.lock.acquire() - - if type == "XDCC": - return XDCCRequest(proxies=self.getProxies()) - + def getRequest(self, pluginName, cj=None): req = Browser(self.bucket, self.getOptions()) - if account: - cj = self.getCookieJar(pluginName, account) + if cj: req.setCookieJar(cj) else: req.setCookieJar(CookieJar(pluginName)) - self.lock.release() return req def getHTTPRequest(self, **kwargs): @@ -67,16 +58,12 @@ class RequestFactory(): rep = h.load(*args, **kwargs) finally: h.close() - - return rep - def getCookieJar(self, pluginName, account=None): - if (pluginName, account) in self.cookiejars: - return self.cookiejars[(pluginName, account)] + return rep - cj = CookieJar(pluginName, account) - self.cookiejars[(pluginName, account)] = cj - return cj + def openCookieJar(self, pluginname): + """Create new CookieJar""" + return CookieJar(pluginname) def getProxies(self): """ returns a proxy list for the request classes """ @@ -106,7 +93,7 @@ class RequestFactory(): def getOptions(self): """returns options needed for pycurl""" - return {"interface": self.iface(), + return {"interface": self.iface, "proxies": self.getProxies(), "ipv6": self.core.config["download"]["ipv6"]} -- cgit v1.2.3 From 0df39b89d4625e1671603fbc7e2a94b045673378 Mon Sep 17 00:00:00 2001 From: zoidberg10 Date: Thu, 5 Jan 2012 18:26:40 +0100 Subject: fix post urlencode regression --- module/network/HTTPRequest.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/network') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 40f18f2a5..d4c33bbff 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -31,6 +31,7 @@ def myquote(url): return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") def myurlencode(data): + data = dict(data) return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ y.encode('utf_8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) -- cgit v1.2.3 From 1bb6ebf544b43cacf7c0755c5a8608b79b95e2d6 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 20:11:16 +0100 Subject: MultiHoster plugin type, some fixes, new documentation structure --- module/network/HTTPDownload.py | 6 +++--- module/network/HTTPRequest.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'module/network') diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index fe8075539..0d5fc59c9 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -17,9 +17,9 @@ @author: RaNaN """ -from os import remove, fsync +from os import remove from os.path import dirname -from time import sleep, time +from time import time from shutil import move from logging import getLogger @@ -28,7 +28,7 @@ import pycurl from HTTPChunk import ChunkInfo, HTTPChunk from HTTPRequest import BadHeader -from module.plugins.Plugin import Abort +from module.plugins.Hoster import Abort from module.utils import save_join, fs_encode class HTTPDownload(): diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index d4c33bbff..cd13dd01f 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -25,7 +25,7 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Plugin import Abort +from module.plugins.Hoster import Abort def myquote(url): return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") -- cgit v1.2.3 From 1ecdd9f6b53fec45e1d48592e3ff56aa7a576bec Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 8 Jan 2012 16:47:52 +0100 Subject: some cleanups, closed #490 --- module/network/HTTPChunk.py | 2 +- module/network/HTTPDownload.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/network') diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index b637aef32..add2cc094 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -20,7 +20,7 @@ from os import remove, stat, fsync from os.path import exists from time import sleep from re import search -from module.utils import fs_encode +from module.utils.fs import fs_encode import codecs import pycurl diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index 0d5fc59c9..6ac39a051 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -29,7 +29,7 @@ from HTTPChunk import ChunkInfo, HTTPChunk from HTTPRequest import BadHeader from module.plugins.Hoster import Abort -from module.utils import save_join, fs_encode +from module.utils.fs import save_join, fs_encode class HTTPDownload(): """ loads a url http + ftp """ -- cgit v1.2.3 From c654f31efc548957f10e3f4c1a5060dcea1dcdec Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 10 Jan 2012 00:04:24 +0100 Subject: changed HEAD request --- module/network/HTTPRequest.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'module/network') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index cd13dd01f..8d65b025f 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -193,11 +193,19 @@ class HTTPRequest(): if just_header: self.c.setopt(pycurl.FOLLOWLOCATION, 0) self.c.setopt(pycurl.NOBODY, 1) + + # overwrite HEAD request, we want a common request type + if post: + self.c.setopt(pycurl.CUSTOMREQUEST, "POST") + else: + self.c.setopt(pycurl.CUSTOMREQUEST, "GET") + self.c.perform() rep = self.header self.c.setopt(pycurl.FOLLOWLOCATION, 1) self.c.setopt(pycurl.NOBODY, 0) + self.c.setopt(pycurl.CUSTOMREQUEST, 0) else: self.c.perform() -- cgit v1.2.3 From 692d015627ecf03fbc23cfdb4afcf398b9a09a51 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 12 Jan 2012 17:26:28 +0100 Subject: scripts for testing and syntax unit test --- module/network/HTTPRequest.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'module/network') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 8d65b025f..4684397d9 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -39,7 +39,7 @@ bad_headers = range(400, 404) + range(405, 418) + range(500, 506) class BadHeader(Exception): def __init__(self, code, content=""): - Exception.__init__(self, "Bad server response: %s %s" % (code, responses[int(code)])) + Exception.__init__(self, "Bad server response: %s %s" % (code, responses.get(int(code), "Unknown Header"))) self.code = code self.content = content @@ -200,12 +200,13 @@ class HTTPRequest(): else: self.c.setopt(pycurl.CUSTOMREQUEST, "GET") - self.c.perform() - rep = self.header - - self.c.setopt(pycurl.FOLLOWLOCATION, 1) - self.c.setopt(pycurl.NOBODY, 0) - self.c.setopt(pycurl.CUSTOMREQUEST, 0) + try: + self.c.perform() + rep = self.header + finally: + self.c.setopt(pycurl.FOLLOWLOCATION, 1) + self.c.setopt(pycurl.NOBODY, 0) + self.c.setopt(pycurl.CUSTOMREQUEST, 0) else: self.c.perform() -- cgit v1.2.3 From c7ad1cc5b4a5d190a060e3ddd9274c3065da6708 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Fri, 13 Jan 2012 23:24:21 +0100 Subject: plugin unit test, closed #499, #500 --- module/network/HTTPRequest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/network') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 4684397d9..7887081e7 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -206,7 +206,7 @@ class HTTPRequest(): finally: self.c.setopt(pycurl.FOLLOWLOCATION, 1) self.c.setopt(pycurl.NOBODY, 0) - self.c.setopt(pycurl.CUSTOMREQUEST, 0) + self.c.unsetopt(pycurl.CUSTOMREQUEST) else: self.c.perform() -- cgit v1.2.3 From 17b3595dc5db8b3270e6bcd07176ed4b7b47930a Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 15 Jan 2012 19:28:12 +0100 Subject: improved handling of content-disposition --- module/network/Browser.py | 7 +++++++ module/network/HTTPChunk.py | 17 +++++++++++------ module/network/HTTPDownload.py | 13 ++++++++----- module/network/HTTPRequest.py | 2 +- 4 files changed, 27 insertions(+), 12 deletions(-) (limited to 'module/network') diff --git a/module/network/Browser.py b/module/network/Browser.py index d68a23687..3452184d8 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -54,6 +54,13 @@ class Browser(object): return self.dl.size return 0 + @property + def name(self): + if self.dl: + return self.dl.name + else: + return "" + @property def arrived(self): if self.dl: diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index add2cc094..3380fb733 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -20,10 +20,13 @@ from os import remove, stat, fsync from os.path import exists from time import sleep from re import search -from module.utils.fs import fs_encode + import codecs import pycurl +from module.utils import remove_chars +from module.utils.fs import fs_encode + from HTTPRequest import HTTPRequest class WrongFormat(Exception): @@ -256,11 +259,13 @@ class HTTPChunk(HTTPRequest): if line.startswith("accept-ranges") and "bytes" in line: self.p.chunkSupport = True - if line.startswith("content-disposition") and "filename=" in line: - name = orgline.partition("filename=")[2] - name = name.replace('"', "").replace("'", "").replace(";", "").strip() - self.p.nameDisposition = name - self.log.debug("Content-Disposition: %s" % name) + if "content-disposition" in line: + + m = search("filename(?P=|\*=(?P.+)'')(?P.*)", line) + if m: + name = remove_chars(m.groupdict()['name'], "\"';").strip() + self.p._name = name + self.log.debug("Content-Disposition: %s" % name) if not self.resume and line.startswith("content-length"): self.p.size = int(line.split(":")[1]) diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index 6ac39a051..59d38beee 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -49,7 +49,7 @@ class HTTPDownload(): self.abort = False self.size = 0 - self.nameDisposition = None #will be parsed from content disposition + self._name = ""# will be parsed from content disposition self.chunks = [] @@ -87,6 +87,10 @@ class HTTPDownload(): if not self.size: return 0 return (self.arrived * 100) / self.size + @property + def name(self): + return self._name if self.disposition else "" + def _copyChunks(self): init = fs_encode(self.info.getChunkName(0)) #initial chunk name @@ -113,8 +117,8 @@ class HTTPDownload(): remove(fname) #remove chunk fo.close() - if self.nameDisposition and self.disposition: - self.filename = save_join(dirname(self.filename), self.nameDisposition) + if self.name: + self.filename = save_join(dirname(self.filename), self.name) move(init, fs_encode(self.filename)) self.info.remove() #remove info file @@ -144,8 +148,7 @@ class HTTPDownload(): finally: self.close() - if self.nameDisposition and self.disposition: return self.nameDisposition - return None + return self.name def _download(self, chunks, resume): if not resume: diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 7887081e7..a0b419763 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -271,7 +271,7 @@ class HTTPRequest(): #TODO: html_unescape as default except LookupError: - self.log.debug("No Decoder foung for %s" % encoding) + self.log.debug("No Decoder found for %s" % encoding) except Exception: self.log.debug("Error when decoding string from %s." % encoding) -- cgit v1.2.3 From 995b6ace8598f22fe8b21b67c587797baa6f7f21 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 16 Jan 2012 18:31:59 +0100 Subject: correct utf8 conversion for urls --- module/network/HTTPRequest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/network') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index a0b419763..2f084efb5 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -28,12 +28,12 @@ from cStringIO import StringIO from module.plugins.Hoster import Abort def myquote(url): - return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") + return quote(url.encode('utf8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") def myurlencode(data): data = dict(data) - return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ - y.encode('utf_8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) + return urlencode(dict((x.encode('utf8') if isinstance(x, unicode) else x, \ + y.encode('utf8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) bad_headers = range(400, 404) + range(405, 418) + range(500, 506) -- cgit v1.2.3 From 7bd2ffe0e50efea468efaec28abace2055dab42d Mon Sep 17 00:00:00 2001 From: Jeix Date: Sun, 5 Feb 2012 14:31:50 +0100 Subject: closed #261 --- module/network/XDCCRequest.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'module/network') diff --git a/module/network/XDCCRequest.py b/module/network/XDCCRequest.py index 92ccb4839..f03798c17 100644 --- a/module/network/XDCCRequest.py +++ b/module/network/XDCCRequest.py @@ -63,8 +63,9 @@ class XDCCRequest(): return socket.socket() - def download(self, ip, port, filename, progressNotify=None): + def download(self, ip, port, filename, irc, progressNotify=None): + ircbuffer = "" lastUpdate = time() cumRecvLen = 0 @@ -94,6 +95,8 @@ class XDCCRequest(): remove(filename) raise Abort() + self._keepAlive(irc, ircbuffer) + data = dccsock.recv(4096) dataLen = len(data) self.recv += dataLen @@ -124,7 +127,21 @@ class XDCCRequest(): return filename - + def _keepAlive(self, sock, readbuffer): + fdset = select([sock], [], [], 0) + if sock not in fdset[0]: + return + + readbuffer += sock.recv(1024) + temp = readbuffer.split("\n") + readbuffer = temp.pop() + + for line in temp: + line = line.rstrip() + first = line.split() + if first[0] == "PING": + sock.send("PONG %s\r\n" % first[1]) + def abortDownloads(self): self.abort = True -- cgit v1.2.3 From 4df2b77fdf42046fe19bd371be7c7255986b5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 6 Mar 2012 13:36:39 +0100 Subject: renamed hooks to addons, new filemanager and database, many new api methods you will loose ALL your LINKS, webinterface will NOT work --- module/network/HTTPDownload.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'module/network') diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index 59d38beee..520a4e5f4 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -31,6 +31,8 @@ from HTTPRequest import BadHeader from module.plugins.Hoster import Abort from module.utils.fs import save_join, fs_encode +# TODO: save content-disposition for resuming + class HTTPDownload(): """ loads a url http + ftp """ -- cgit v1.2.3 From 50d4df8b4d48b855bd18e9922355b7f3f2b4da4e Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 20 Mar 2012 14:57:45 +0100 Subject: captcha decrypting for all plugin types, new interaction manager --- module/network/HTTPDownload.py | 2 +- module/network/HTTPRequest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/network') diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index 520a4e5f4..05ca44406 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -28,7 +28,7 @@ import pycurl from HTTPChunk import ChunkInfo, HTTPChunk from HTTPRequest import BadHeader -from module.plugins.Hoster import Abort +from module.plugins.Base import Abort from module.utils.fs import save_join, fs_encode # TODO: save content-disposition for resuming diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 2f084efb5..774c0e64d 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -25,7 +25,7 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Hoster import Abort +from module.plugins.Base import Abort def myquote(url): return quote(url.encode('utf8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") -- cgit v1.2.3 From b40b32ee05f611323a7827fad2a25fa0a28dcb24 Mon Sep 17 00:00:00 2001 From: X3n0m0rph59 Date: Sun, 22 Apr 2012 19:56:17 +0200 Subject: a huge pile of spelling fixes --- module/network/Browser.py | 2 +- module/network/Bucket.py | 2 +- module/network/HTTPChunk.py | 8 ++++---- module/network/HTTPDownload.py | 8 ++++---- module/network/RequestFactory.py | 2 +- module/network/XDCCRequest.py | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) (limited to 'module/network') diff --git a/module/network/Browser.py b/module/network/Browser.py index 3452184d8..9cf6c2f30 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -16,7 +16,7 @@ class Browser(object): self.options = options #holds pycurl options self.bucket = bucket - self.cj = None # needs to be setted later + self.cj = None # needs to be set later self._size = 0 self.renewHTTPRequest() diff --git a/module/network/Bucket.py b/module/network/Bucket.py index ff80bda55..db67faa4a 100644 --- a/module/network/Bucket.py +++ b/module/network/Bucket.py @@ -39,7 +39,7 @@ class Bucket: self.lock.release() def consumed(self, amount): - """ return time the process have to sleep, after consumed specified amount """ + """ return the time the process has to sleep, after it consumed a specified amount """ if self.rate < MIN_RATE: return 0 #May become unresponsive otherwise self.lock.acquire() diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index 3380fb733..d17177ee7 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -207,7 +207,7 @@ class HTTPChunk(HTTPRequest): def writeHeader(self, buf): self.header += buf - #@TODO forward headers?, this is possibly unneeeded, when we just parse valid 200 headers + #@TODO forward headers?, this is possibly unneeded, when we just parse valid 200 headers # as first chunk, we will parse the headers if not self.range and self.header.endswith("\r\n\r\n"): self.parseHeader() @@ -236,8 +236,8 @@ class HTTPChunk(HTTPRequest): sleep(self.p.bucket.consumed(size)) else: # Avoid small buffers, increasing sleep time slowly if buffer size gets smaller - # otherwise reduce sleep time percentual (values are based on tests) - # So in general cpu time is saved without reducing bandwith too much + # otherwise reduce sleep time percentile (values are based on tests) + # So in general cpu time is saved without reducing bandwidth too much if size < self.lastSize: self.sleep += 0.002 @@ -253,7 +253,7 @@ class HTTPChunk(HTTPRequest): def parseHeader(self): - """parse data from recieved header""" + """parse data from received header""" for orgline in self.decodeResponse(self.header).splitlines(): line = orgline.strip().lower() if line.startswith("accept-ranges") and "bytes" in line: diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index 05ca44406..c6d2e1547 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -34,7 +34,7 @@ from module.utils.fs import save_join, fs_encode # TODO: save content-disposition for resuming class HTTPDownload(): - """ loads a url http + ftp """ + """ loads an url, http + ftp supported """ def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, options={}, progressNotify=None, disposition=False): @@ -174,7 +174,7 @@ class HTTPDownload(): while 1: #need to create chunks - if not chunksCreated and self.chunkSupport and self.size: #will be setted later by first chunk + if not chunksCreated and self.chunkSupport and self.size: #will be set later by first chunk if not resume: self.info.setSize(self.size) @@ -193,7 +193,7 @@ class HTTPDownload(): self.chunks.append(c) self.m.add_handle(handle) else: - #close immediatly + #close immediately self.log.debug("Invalid curl handle -> closed") c.close() @@ -291,7 +291,7 @@ class HTTPDownload(): if self.abort: raise Abort() - #sleep(0.003) #supress busy waiting - limits dl speed to (1 / x) * buffersize + #sleep(0.003) #suppress busy waiting - limits dl speed to (1 / x) * buffersize self.m.select(1) for chunk in self.chunks: diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 12fd66c95..932184678 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -46,7 +46,7 @@ class RequestFactory(): return req def getHTTPRequest(self, **kwargs): - """ returns a http request, dont forget to close it ! """ + """ returns a http request, don't forget to close it ! """ options = self.getOptions() options.update(kwargs) # submit kwargs as additional options return HTTPRequest(CookieJar(None), options) diff --git a/module/network/XDCCRequest.py b/module/network/XDCCRequest.py index f03798c17..7a1a98cb5 100644 --- a/module/network/XDCCRequest.py +++ b/module/network/XDCCRequest.py @@ -119,7 +119,7 @@ class XDCCRequest(): fh.write(data) - # acknowledge data by sending number of recceived bytes + # acknowledge data by sending number of received bytes dccsock.send(struct.pack('!I', self.recv)) dccsock.close() -- cgit v1.2.3 From c64d929c6339226e6c29e3bfba195fbc5610865b Mon Sep 17 00:00:00 2001 From: zoidberg10 Date: Fri, 17 Aug 2012 01:02:44 +0200 Subject: fix multipart post encoding --- module/network/HTTPRequest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/network') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 774c0e64d..290a27a40 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -167,7 +167,7 @@ class HTTPRequest(): self.c.setopt(pycurl.POSTFIELDS, post) else: - post = [(x, str(quote(y)) if type(y) in (str, unicode) else y ) for x, y in post.iteritems()] + post = [(x, y.encode('utf8') if type(y) == unicode else y ) for x, y in post.iteritems()] self.c.setopt(pycurl.HTTPPOST, post) else: self.c.setopt(pycurl.POST, 0) -- cgit v1.2.3 From 7fe05500cc738eda76216510e110eb118ad04dee Mon Sep 17 00:00:00 2001 From: RaNaN Date: Wed, 12 Sep 2012 09:56:44 +0200 Subject: higher low speed time, easier way to set curl options --- module/network/HTTPRequest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'module/network') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 290a27a40..4b45e10a2 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -62,6 +62,7 @@ class HTTPRequest(): self.initHandle() self.setInterface(options) + self.setOptions(options) self.c.setopt(pycurl.WRITEFUNCTION, self.write) self.c.setopt(pycurl.HEADERFUNCTION, self.writeHeader) @@ -79,7 +80,8 @@ class HTTPRequest(): if hasattr(pycurl, "AUTOREFERER"): self.c.setopt(pycurl.AUTOREFERER, 1) self.c.setopt(pycurl.SSL_VERIFYPEER, 0) - self.c.setopt(pycurl.LOW_SPEED_TIME, 30) + # Interval for low speed, detects connection loss, but can abort dl if hoster stalls the download + self.c.setopt(pycurl.LOW_SPEED_TIME, 45) self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5) #self.c.setopt(pycurl.VERBOSE, 1) @@ -127,6 +129,11 @@ class HTTPRequest(): if "timeout" in options: self.c.setopt(pycurl.LOW_SPEED_TIME, options["timeout"]) + def setOptions(self, options): + """ Sets same options as available in pycurl """ + for k, v in options.iteritems(): + if hasattr(pycurl, k): + self.c.setopt(getattr(pycurl, k), v) def addCookies(self): """ put cookies from curl handle to cj """ -- cgit v1.2.3