diff options
Diffstat (limited to 'pyload/network')
| -rw-r--r-- | pyload/network/Browser.py | 23 | ||||
| -rw-r--r-- | pyload/network/CookieJar.py | 1 | ||||
| -rw-r--r-- | pyload/network/HTTPChunk.py | 24 | ||||
| -rw-r--r-- | pyload/network/HTTPDownload.py | 13 | ||||
| -rw-r--r-- | pyload/network/HTTPRequest.py | 18 | ||||
| -rw-r--r-- | pyload/network/JsEngine.py | 4 | ||||
| -rw-r--r-- | pyload/network/RequestFactory.py | 9 | ||||
| -rw-r--r-- | pyload/network/XDCCRequest.py | 12 |
8 files changed, 104 insertions, 0 deletions
diff --git a/pyload/network/Browser.py b/pyload/network/Browser.py index 89341ff25..1c0596ccc 100644 --- a/pyload/network/Browser.py +++ b/pyload/network/Browser.py @@ -9,6 +9,7 @@ from pyload.network.HTTPDownload import HTTPDownload class Browser(object): __slots__ = ("log", "options", "bucket", "cj", "_size", "http", "dl") + def __init__(self, bucket=None, options={}): self.log = getLogger("log") @@ -26,6 +27,7 @@ class Browser(object): if hasattr(self, "http"): self.http.close() self.http = HTTPRequest(self.cj, self.options) + def setLastURL(self, val): self.http.lastURL = val @@ -35,17 +37,22 @@ class Browser(object): code = property(lambda self: self.http.code) cookieJar = property(lambda self: self.cj) + def setCookieJar(self, cj): self.cj = cj self.http.cj = cj @property + + def speed(self): if self.dl: return self.dl.speed return 0 @property + + def size(self): if self._size: return self._size @@ -54,30 +61,38 @@ class Browser(object): return 0 @property + + def arrived(self): if self.dl: return self.dl.arrived return 0 @property + + def percent(self): if not self.size: return 0 return (self.arrived * 100) / self.size + def clearCookies(self): if self.cj: self.cj.clear() self.http.clearCookies() + def clearReferer(self): self.http.lastURL = None + def abortDownloads(self): self.http.abort = True if self.dl: self._size = self.dl.size self.dl.abort = True + def httpDownload(self, url, filename, get={}, post={}, ref=True, cookies=True, chunks=1, resume=False, progressNotify=None, disposition=False): """ this can also download ftp """ @@ -91,14 +106,17 @@ class Browser(object): return name + def load(self, *args, **kwargs): """ retrieves page """ return self.http.load(*args, **kwargs) + def putHeader(self, name, value): """ add a header to the request """ self.http.putHeader(name, value) + def addAuth(self, pwd): """Adds user and pw for http auth @@ -107,20 +125,25 @@ class Browser(object): self.options["auth"] = pwd self.renewHTTPRequest() #we need a new request + def removeAuth(self): if "auth" in self.options: del self.options["auth"] self.renewHTTPRequest() + def setOption(self, name, value): """Adds an option to the request, see HTTPRequest for existing ones""" self.options[name] = value + def deleteOption(self, name): if name in self.options: del self.options[name] + def clearHeaders(self): self.http.clearHeaders() + def close(self): """ cleanup """ if hasattr(self, "http"): diff --git a/pyload/network/CookieJar.py b/pyload/network/CookieJar.py index a2b302776..35d7fa6ef 100644 --- a/pyload/network/CookieJar.py +++ b/pyload/network/CookieJar.py @@ -21,6 +21,7 @@ class CookieJar(Cookie.SimpleCookie): def getCookie(self, name): return self[name].value + def setCookie(self, domain, name, value, path="/", exp=None, secure="FALSE"): self[name] = value self[name]["domain"] = domain diff --git a/pyload/network/HTTPChunk.py b/pyload/network/HTTPChunk.py index 41752b940..5395b7bda 100644 --- a/pyload/network/HTTPChunk.py +++ b/pyload/network/HTTPChunk.py @@ -17,12 +17,14 @@ class WrongFormat(Exception): class ChunkInfo(object): + def __init__(self, name): self.name = unicode(name) self.size = 0 self.resume = False self.chunks = [] + def __repr__(self): ret = "ChunkInfo: %s, %s\n" % (self.name, self.size) for i, c in enumerate(self.chunks): @@ -30,15 +32,19 @@ class ChunkInfo(object): return ret + def setSize(self, size): self.size = int(size) + def addChunk(self, name, range): self.chunks.append((name, range)) + def clear(self): self.chunks = [] + def createChunks(self, chunks): self.clear() chunk_size = self.size / chunks @@ -62,6 +68,8 @@ class ChunkInfo(object): fh.close() @staticmethod + + def load(name): fs_name = fs_encode("%s.chunks" % name) if not exists(fs_name): @@ -93,21 +101,26 @@ class ChunkInfo(object): fh.close() return ci + def remove(self): fs_name = fs_encode("%s.chunks" % self.name) if exists(fs_name): remove(fs_name) + def getCount(self): return len(self.chunks) + def getChunkName(self, index): return self.chunks[index][0] + def getChunkRange(self, index): return self.chunks[index][1] class HTTPChunk(HTTPRequest): + def __init__(self, id, parent, range=None, resume=False): self.id = id self.p = parent # HTTPDownload instance @@ -136,13 +149,17 @@ class HTTPChunk(HTTPRequest): self.sleep = 0.000 self.lastSize = 0 + def __repr__(self): return "<HTTPChunk id=%d, size=%d, arrived=%d>" % (self.id, self.size, self.arrived) @property + + def cj(self): return self.p.cj + def getHandle(self): """ returns a Curl handle ready to use for perform/multiperform """ @@ -188,6 +205,7 @@ class HTTPChunk(HTTPRequest): return self.c + def writeHeader(self, buf): self.header += buf #@TODO forward headers?, this is possibly unneeeded, when we just parse valid 200 headers @@ -202,6 +220,7 @@ class HTTPChunk(HTTPRequest): self.headerParsed = True + def writeBody(self, buf): #ignore BOM, it confuses unrar if not self.BOMChecked: @@ -263,25 +282,30 @@ class HTTPChunk(HTTPRequest): self.headerParsed = True + def stop(self): """The download will not proceed after next call of writeBody""" self.range = [0, 0] self.size = 0 + def resetRange(self): """ Reset the range, so the download will load all data available """ self.range = None + def setRange(self, range): self.range = range self.size = range[1] - range[0] + def flushFile(self): """ flush and close file """ self.fp.flush() fsync(self.fp.fileno()) #make sure everything was written to disk self.fp.close() #needs to be closed, or merging chunks will fail + def close(self): """ closes everything, unusable after this """ if self.fp: self.fp.close() diff --git a/pyload/network/HTTPDownload.py b/pyload/network/HTTPDownload.py index 0d331ca99..89e594640 100644 --- a/pyload/network/HTTPDownload.py +++ b/pyload/network/HTTPDownload.py @@ -59,19 +59,26 @@ class HTTPDownload(object): self.progress = progress @property + + def speed(self): last = [sum(x) for x in self.lastSpeeds if x] return (sum(self.speeds) + sum(last)) / (1 + len(last)) @property + + def arrived(self): return sum([c.arrived for c in self.chunks]) @property + + def percent(self): if not self.size: return 0 return (self.arrived * 100) / self.size + def _copyChunks(self): init = fs_encode(self.info.getChunkName(0)) #initial chunk name @@ -104,6 +111,7 @@ class HTTPDownload(object): move(init, fs_encode(self.filename)) self.info.remove() #remove info file + def download(self, chunks=1, resume=False): """ returns new filename or None """ @@ -132,6 +140,7 @@ class HTTPDownload(object): if self.nameDisposition and self.disposition: return self.nameDisposition return None + def _download(self, chunks, resume): if not resume: self.info.clear() @@ -280,15 +289,18 @@ class HTTPDownload(object): self._copyChunks() + def updateProgress(self): if self.progress: self.progress(self.percent) + def findChunk(self, handle): """ linear search to find a chunk (should be ok since chunk size is usually low) """ for chunk in self.chunks: if chunk.c == handle: return chunk + def closeChunk(self, chunk): try: self.m.remove_handle(chunk.c) @@ -297,6 +309,7 @@ class HTTPDownload(object): finally: chunk.close() + def close(self): """ cleanup """ for chunk in self.chunks: diff --git a/pyload/network/HTTPRequest.py b/pyload/network/HTTPRequest.py index fe7e26c48..3e5903df3 100644 --- a/pyload/network/HTTPRequest.py +++ b/pyload/network/HTTPRequest.py @@ -28,6 +28,7 @@ bad_headers = range(400, 404) + range(405, 418) + range(500, 506) class BadHeader(Exception): + def __init__(self, code, content=""): Exception.__init__(self, "Bad server response: %s %s" % (code, responses[int(code)])) self.code = code @@ -35,6 +36,7 @@ class BadHeader(Exception): class HTTPRequest(object): + def __init__(self, cookies=None, options=None): self.c = pycurl.Curl() self.rep = StringIO() @@ -58,6 +60,7 @@ class HTTPRequest(object): self.log = getLogger("log") + def initHandle(self): """ sets common options to curl handle """ self.c.setopt(pycurl.FOLLOWLOCATION, 1) @@ -87,6 +90,7 @@ class HTTPRequest(object): "Keep-Alive: 300", "Expect:"]) + def setInterface(self, options): interface, proxy, ipv6 = options["interface"], options["proxies"], options["ipv6"] @@ -119,11 +123,13 @@ class HTTPRequest(object): if "timeout" in options: self.c.setopt(pycurl.LOW_SPEED_TIME, options["timeout"]) + def addCookies(self): """ put cookies from curl handle to cj """ if self.cj: self.cj.addCookies(self.c.getinfo(pycurl.INFO_COOKIELIST)) + def getCookies(self): """ add cookies from cj to curl handle """ if self.cj: @@ -131,9 +137,11 @@ class HTTPRequest(object): self.c.setopt(pycurl.COOKIELIST, c) return + def clearCookies(self): self.c.setopt(pycurl.COOKIELIST, "") + def setRequestContext(self, url, get, post, referer, cookies, multipart=False): """ sets everything needed for the request """ @@ -171,6 +179,7 @@ class HTTPRequest(object): self.c.setopt(pycurl.COOKIEJAR, "") self.getCookies() + def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False, follow_location=True, save_cookies=True): """ load and returns a given page """ @@ -212,6 +221,7 @@ class HTTPRequest(object): return rep + def verifyHeader(self): """ raise an exceptions on bad headers """ code = int(self.c.getinfo(pycurl.RESPONSE_CODE)) @@ -220,10 +230,12 @@ class HTTPRequest(object): raise BadHeader(code, self.getResponse()) return code + def checkHeader(self): """ check if header indicates failure""" return int(self.c.getinfo(pycurl.RESPONSE_CODE)) not in bad_headers + def getResponse(self): """ retrieve response from string io """ if self.rep is None: @@ -234,6 +246,7 @@ class HTTPRequest(object): self.rep = StringIO() return value + def decodeResponse(self, rep): """ decode with correct encoding, relies on header """ header = self.header.splitlines() @@ -269,6 +282,7 @@ class HTTPRequest(object): return rep + def write(self, buf): """ writes response """ if self.rep.tell() > 1000000 or self.abort: @@ -283,16 +297,20 @@ class HTTPRequest(object): else: self.rep.write(buf) + def writeHeader(self, buf): """ writes header """ self.header += buf + def putHeader(self, name, value): self.headers.append("%s: %s" % (name, value)) + def clearHeaders(self): self.headers = [] + def close(self): """ cleanup, unusable after this """ self.rep.close() diff --git a/pyload/network/JsEngine.py b/pyload/network/JsEngine.py index c64e8c490..bcf6ae0e2 100644 --- a/pyload/network/JsEngine.py +++ b/pyload/network/JsEngine.py @@ -33,6 +33,8 @@ class JsEngine(object): @classmethod + + def find(cls): """ Check if there is any engine available """ return [E for E in ENGINES if E.find()] @@ -120,6 +122,8 @@ class AbstractEngine(object): @classmethod + + def find(cls): """ Check if the engine is available """ try: diff --git a/pyload/network/RequestFactory.py b/pyload/network/RequestFactory.py index 579eafea7..5f8e7e206 100644 --- a/pyload/network/RequestFactory.py +++ b/pyload/network/RequestFactory.py @@ -11,6 +11,7 @@ from pyload.network.CookieJar import CookieJar from pyload.network.XDCCRequest import XDCCRequest class RequestFactory(object): + def __init__(self, core): self.lock = Lock() self.core = core @@ -18,9 +19,11 @@ class RequestFactory(object): self.updateBucket() self.cookiejars = {} + def iface(self): return self.core.config["download"]["interface"] + def getRequest(self, pluginName, account=None, type="HTTP"): self.lock.acquire() @@ -38,12 +41,14 @@ class RequestFactory(object): self.lock.release() return req + def getHTTPRequest(self, **kwargs): """ returns a http request, dont forget to close it ! """ options = self.getOptions() options.update(kwargs) # submit kwargs as additional options return HTTPRequest(CookieJar(None), options) + def getURL(self, *args, **kwargs): """ see HTTPRequest for argument list """ cj = None @@ -65,6 +70,7 @@ class RequestFactory(object): return rep + def getCookieJar(self, pluginName, account=None): if (pluginName, account) in self.cookiejars: return self.cookiejars[(pluginName, account)] @@ -73,6 +79,7 @@ class RequestFactory(object): self.cookiejars[(pluginName, account)] = cj return cj + def getProxies(self): """ returns a proxy list for the request classes """ if not self.core.config["proxy"]["proxy"]: @@ -99,12 +106,14 @@ class RequestFactory(object): "password": pw, } + def getOptions(self): """returns options needed for pycurl""" return {"interface": self.iface(), "proxies": self.getProxies(), "ipv6": self.core.config["download"]["ipv6"]} + def updateBucket(self): """ set values in the bucket according to settings""" if not self.core.config["download"]["limit_speed"]: diff --git a/pyload/network/XDCCRequest.py b/pyload/network/XDCCRequest.py index c49f418c4..a4813e039 100644 --- a/pyload/network/XDCCRequest.py +++ b/pyload/network/XDCCRequest.py @@ -16,6 +16,7 @@ from pyload.plugin.Plugin import Abort class XDCCRequest(object): + def __init__(self, timeout=30, proxies={}): self.proxies = proxies @@ -27,6 +28,7 @@ class XDCCRequest(object): self.abort = False + def createSocket(self): # proxytype = None # proxy = None @@ -46,6 +48,7 @@ class XDCCRequest(object): return socket.socket() + def download(self, ip, port, filename, irc, progress=None): ircbuffer = "" @@ -109,6 +112,7 @@ class XDCCRequest(object): return filename + def _keepAlive(self, sock, *readbuffer): fdset = select([sock], [], [], 0) if sock not in fdset[0]: @@ -124,21 +128,29 @@ class XDCCRequest(object): if first[0] == "PING": sock.send("PONG %s\r\n" % first[1]) + def abortDownloads(self): self.abort = True @property + + def size(self): return self.filesize @property + + def arrived(self): return self.recv @property + + def percent(self): if not self.filesize: return 0 return (self.recv * 100) / self.filesize + def close(self): pass |