From e00ef98491f79ae8aa972ae1473dae4a7b78c07e Mon Sep 17 00:00:00 2001
From: Walter Purcaro <vuolter@users.noreply.github.com>
Date: Mon, 13 Apr 2015 17:20:59 +0200
Subject: Cleanup
---
pyload/network/Browser.py | 23 +++++++++++++++++++++++
pyload/network/CookieJar.py | 1 +
pyload/network/HTTPChunk.py | 24 ++++++++++++++++++++++++
pyload/network/HTTPDownload.py | 13 +++++++++++++
pyload/network/HTTPRequest.py | 18 ++++++++++++++++++
pyload/network/JsEngine.py | 4 ++++
pyload/network/RequestFactory.py | 9 +++++++++
pyload/network/XDCCRequest.py | 12 ++++++++++++
8 files changed, 104 insertions(+)
(limited to 'pyload/network')
diff --git a/pyload/network/Browser.py b/pyload/network/Browser.py
index 89341ff25..1c0596ccc 100644
--- a/pyload/network/Browser.py
+++ b/pyload/network/Browser.py
@@ -9,6 +9,7 @@ from pyload.network.HTTPDownload import HTTPDownload
class Browser(object):
__slots__ = ("log", "options", "bucket", "cj", "_size", "http", "dl")
+
def __init__(self, bucket=None, options={}):
self.log = getLogger("log")
@@ -26,6 +27,7 @@ class Browser(object):
if hasattr(self, "http"): self.http.close()
self.http = HTTPRequest(self.cj, self.options)
+
def setLastURL(self, val):
self.http.lastURL = val
@@ -35,17 +37,22 @@ class Browser(object):
code = property(lambda self: self.http.code)
cookieJar = property(lambda self: self.cj)
+
def setCookieJar(self, cj):
self.cj = cj
self.http.cj = cj
@property
+
+
def speed(self):
if self.dl:
return self.dl.speed
return 0
@property
+
+
def size(self):
if self._size:
return self._size
@@ -54,30 +61,38 @@ class Browser(object):
return 0
@property
+
+
def arrived(self):
if self.dl:
return self.dl.arrived
return 0
@property
+
+
def percent(self):
if not self.size: return 0
return (self.arrived * 100) / self.size
+
def clearCookies(self):
if self.cj:
self.cj.clear()
self.http.clearCookies()
+
def clearReferer(self):
self.http.lastURL = None
+
def abortDownloads(self):
self.http.abort = True
if self.dl:
self._size = self.dl.size
self.dl.abort = True
+
def httpDownload(self, url, filename, get={}, post={}, ref=True, cookies=True, chunks=1, resume=False,
progressNotify=None, disposition=False):
""" this can also download ftp """
@@ -91,14 +106,17 @@ class Browser(object):
return name
+
def load(self, *args, **kwargs):
""" retrieves page """
return self.http.load(*args, **kwargs)
+
def putHeader(self, name, value):
""" add a header to the request """
self.http.putHeader(name, value)
+
def addAuth(self, pwd):
"""Adds user and pw for http auth
@@ -107,20 +125,25 @@ class Browser(object):
self.options["auth"] = pwd
self.renewHTTPRequest() #we need a new request
+
def removeAuth(self):
if "auth" in self.options: del self.options["auth"]
self.renewHTTPRequest()
+
def setOption(self, name, value):
"""Adds an option to the request, see HTTPRequest for existing ones"""
self.options[name] = value
+
def deleteOption(self, name):
if name in self.options: del self.options[name]
+
def clearHeaders(self):
self.http.clearHeaders()
+
def close(self):
""" cleanup """
if hasattr(self, "http"):
diff --git a/pyload/network/CookieJar.py b/pyload/network/CookieJar.py
index a2b302776..35d7fa6ef 100644
--- a/pyload/network/CookieJar.py
+++ b/pyload/network/CookieJar.py
@@ -21,6 +21,7 @@ class CookieJar(Cookie.SimpleCookie):
def getCookie(self, name):
return self[name].value
+
def setCookie(self, domain, name, value, path="/", exp=None, secure="FALSE"):
self[name] = value
self[name]["domain"] = domain
diff --git a/pyload/network/HTTPChunk.py b/pyload/network/HTTPChunk.py
index 41752b940..5395b7bda 100644
--- a/pyload/network/HTTPChunk.py
+++ b/pyload/network/HTTPChunk.py
@@ -17,12 +17,14 @@ class WrongFormat(Exception):
class ChunkInfo(object):
+
def __init__(self, name):
self.name = unicode(name)
self.size = 0
self.resume = False
self.chunks = []
+
def __repr__(self):
ret = "ChunkInfo: %s, %s\n" % (self.name, self.size)
for i, c in enumerate(self.chunks):
@@ -30,15 +32,19 @@ class ChunkInfo(object):
return ret
+
def setSize(self, size):
self.size = int(size)
+
def addChunk(self, name, range):
self.chunks.append((name, range))
+
def clear(self):
self.chunks = []
+
def createChunks(self, chunks):
self.clear()
chunk_size = self.size / chunks
@@ -62,6 +68,8 @@ class ChunkInfo(object):
fh.close()
@staticmethod
+
+
def load(name):
fs_name = fs_encode("%s.chunks" % name)
if not exists(fs_name):
@@ -93,21 +101,26 @@ class ChunkInfo(object):
fh.close()
return ci
+
def remove(self):
fs_name = fs_encode("%s.chunks" % self.name)
if exists(fs_name): remove(fs_name)
+
def getCount(self):
return len(self.chunks)
+
def getChunkName(self, index):
return self.chunks[index][0]
+
def getChunkRange(self, index):
return self.chunks[index][1]
class HTTPChunk(HTTPRequest):
+
def __init__(self, id, parent, range=None, resume=False):
self.id = id
self.p = parent # HTTPDownload instance
@@ -136,13 +149,17 @@ class HTTPChunk(HTTPRequest):
self.sleep = 0.000
self.lastSize = 0
+
def __repr__(self):
return "<HTTPChunk id=%d, size=%d, arrived=%d>" % (self.id, self.size, self.arrived)
@property
+
+
def cj(self):
return self.p.cj
+
def getHandle(self):
""" returns a Curl handle ready to use for perform/multiperform """
@@ -188,6 +205,7 @@ class HTTPChunk(HTTPRequest):
return self.c
+
def writeHeader(self, buf):
self.header += buf
#@TODO forward headers?, this is possibly unneeeded, when we just parse valid 200 headers
@@ -202,6 +220,7 @@ class HTTPChunk(HTTPRequest):
self.headerParsed = True
+
def writeBody(self, buf):
#ignore BOM, it confuses unrar
if not self.BOMChecked:
@@ -263,25 +282,30 @@ class HTTPChunk(HTTPRequest):
self.headerParsed = True
+
def stop(self):
"""The download will not proceed after next call of writeBody"""
self.range = [0, 0]
self.size = 0
+
def resetRange(self):
""" Reset the range, so the download will load all data available """
self.range = None
+
def setRange(self, range):
self.range = range
self.size = range[1] - range[0]
+
def flushFile(self):
""" flush and close file """
self.fp.flush()
fsync(self.fp.fileno()) #make sure everything was written to disk
self.fp.close() #needs to be closed, or merging chunks will fail
+
def close(self):
""" closes everything, unusable after this """
if self.fp: self.fp.close()
diff --git a/pyload/network/HTTPDownload.py b/pyload/network/HTTPDownload.py
index 0d331ca99..89e594640 100644
--- a/pyload/network/HTTPDownload.py
+++ b/pyload/network/HTTPDownload.py
@@ -59,19 +59,26 @@ class HTTPDownload(object):
self.progress = progress
@property
+
+
def speed(self):
last = [sum(x) for x in self.lastSpeeds if x]
return (sum(self.speeds) + sum(last)) / (1 + len(last))
@property
+
+
def arrived(self):
return sum([c.arrived for c in self.chunks])
@property
+
+
def percent(self):
if not self.size: return 0
return (self.arrived * 100) / self.size
+
def _copyChunks(self):
init = fs_encode(self.info.getChunkName(0)) #initial chunk name
@@ -104,6 +111,7 @@ class HTTPDownload(object):
move(init, fs_encode(self.filename))
self.info.remove() #remove info file
+
def download(self, chunks=1, resume=False):
""" returns new filename or None """
@@ -132,6 +140,7 @@ class HTTPDownload(object):
if self.nameDisposition and self.disposition: return self.nameDisposition
return None
+
def _download(self, chunks, resume):
if not resume:
self.info.clear()
@@ -280,15 +289,18 @@ class HTTPDownload(object):
self._copyChunks()
+
def updateProgress(self):
if self.progress:
self.progress(self.percent)
+
def findChunk(self, handle):
""" linear search to find a chunk (should be ok since chunk size is usually low) """
for chunk in self.chunks:
if chunk.c == handle: return chunk
+
def closeChunk(self, chunk):
try:
self.m.remove_handle(chunk.c)
@@ -297,6 +309,7 @@ class HTTPDownload(object):
finally:
chunk.close()
+
def close(self):
""" cleanup """
for chunk in self.chunks:
diff --git a/pyload/network/HTTPRequest.py b/pyload/network/HTTPRequest.py
index fe7e26c48..3e5903df3 100644
--- a/pyload/network/HTTPRequest.py
+++ b/pyload/network/HTTPRequest.py
@@ -28,6 +28,7 @@ bad_headers = range(400, 404) + range(405, 418) + range(500, 506)
class BadHeader(Exception):
+
def __init__(self, code, content=""):
Exception.__init__(self, "Bad server response: %s %s" % (code, responses[int(code)]))
self.code = code
@@ -35,6 +36,7 @@ class BadHeader(Exception):
class HTTPRequest(object):
+
def __init__(self, cookies=None, options=None):
self.c = pycurl.Curl()
self.rep = StringIO()
@@ -58,6 +60,7 @@ class HTTPRequest(object):
self.log = getLogger("log")
+
def initHandle(self):
""" sets common options to curl handle """
self.c.setopt(pycurl.FOLLOWLOCATION, 1)
@@ -87,6 +90,7 @@ class HTTPRequest(object):
"Keep-Alive: 300",
"Expect:"])
+
def setInterface(self, options):
interface, proxy, ipv6 = options["interface"], options["proxies"], options["ipv6"]
@@ -119,11 +123,13 @@ class HTTPRequest(object):
if "timeout" in options:
self.c.setopt(pycurl.LOW_SPEED_TIME, options["timeout"])
+
def addCookies(self):
""" put cookies from curl handle to cj """
if self.cj:
self.cj.addCookies(self.c.getinfo(pycurl.INFO_COOKIELIST))
+
def getCookies(self):
""" add cookies from cj to curl handle """
if self.cj:
@@ -131,9 +137,11 @@ class HTTPRequest(object):
self.c.setopt(pycurl.COOKIELIST, c)
return
+
def clearCookies(self):
self.c.setopt(pycurl.COOKIELIST, "")
+
def setRequestContext(self, url, get, post, referer, cookies, multipart=False):
""" sets everything needed for the request """
@@ -171,6 +179,7 @@ class HTTPRequest(object):
self.c.setopt(pycurl.COOKIEJAR, "")
self.getCookies()
+
def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False, follow_location=True, save_cookies=True):
""" load and returns a given page """
@@ -212,6 +221,7 @@ class HTTPRequest(object):
return rep
+
def verifyHeader(self):
""" raise an exceptions on bad headers """
code = int(self.c.getinfo(pycurl.RESPONSE_CODE))
@@ -220,10 +230,12 @@ class HTTPRequest(object):
raise BadHeader(code, self.getResponse())
return code
+
def checkHeader(self):
""" check if header indicates failure"""
return int(self.c.getinfo(pycurl.RESPONSE_CODE)) not in bad_headers
+
def getResponse(self):
""" retrieve response from string io """
if self.rep is None:
@@ -234,6 +246,7 @@ class HTTPRequest(object):
self.rep = StringIO()
return value
+
def decodeResponse(self, rep):
""" decode with correct encoding, relies on header """
header = self.header.splitlines()
@@ -269,6 +282,7 @@ class HTTPRequest(object):
return rep
+
def write(self, buf):
""" writes response """
if self.rep.tell() > 1000000 or self.abort:
@@ -283,16 +297,20 @@ class HTTPRequest(object):
else:
self.rep.write(buf)
+
def writeHeader(self, buf):
""" writes header """
self.header += buf
+
def putHeader(self, name, value):
self.headers.append("%s: %s" % (name, value))
+
def clearHeaders(self):
self.headers = []
+
def close(self):
""" cleanup, unusable after this """
self.rep.close()
diff --git a/pyload/network/JsEngine.py b/pyload/network/JsEngine.py
index c64e8c490..bcf6ae0e2 100644
--- a/pyload/network/JsEngine.py
+++ b/pyload/network/JsEngine.py
@@ -33,6 +33,8 @@ class JsEngine(object):
@classmethod
+
+
def find(cls):
""" Check if there is any engine available """
return [E for E in ENGINES if E.find()]
@@ -120,6 +122,8 @@ class AbstractEngine(object):
@classmethod
+
+
def find(cls):
""" Check if the engine is available """
try:
diff --git a/pyload/network/RequestFactory.py b/pyload/network/RequestFactory.py
index 579eafea7..5f8e7e206 100644
--- a/pyload/network/RequestFactory.py
+++ b/pyload/network/RequestFactory.py
@@ -11,6 +11,7 @@ from pyload.network.CookieJar import CookieJar
from pyload.network.XDCCRequest import XDCCRequest
class RequestFactory(object):
+
def __init__(self, core):
self.lock = Lock()
self.core = core
@@ -18,9 +19,11 @@ class RequestFactory(object):
self.updateBucket()
self.cookiejars = {}
+
def iface(self):
return self.core.config["download"]["interface"]
+
def getRequest(self, pluginName, account=None, type="HTTP"):
self.lock.acquire()
@@ -38,12 +41,14 @@ class RequestFactory(object):
self.lock.release()
return req
+
def getHTTPRequest(self, **kwargs):
""" returns a http request, dont forget to close it ! """
options = self.getOptions()
options.update(kwargs) # submit kwargs as additional options
return HTTPRequest(CookieJar(None), options)
+
def getURL(self, *args, **kwargs):
""" see HTTPRequest for argument list """
cj = None
@@ -65,6 +70,7 @@ class RequestFactory(object):
return rep
+
def getCookieJar(self, pluginName, account=None):
if (pluginName, account) in self.cookiejars:
return self.cookiejars[(pluginName, account)]
@@ -73,6 +79,7 @@ class RequestFactory(object):
self.cookiejars[(pluginName, account)] = cj
return cj
+
def getProxies(self):
""" returns a proxy list for the request classes """
if not self.core.config["proxy"]["proxy"]:
@@ -99,12 +106,14 @@ class RequestFactory(object):
"password": pw,
}
+
def getOptions(self):
"""returns options needed for pycurl"""
return {"interface": self.iface(),
"proxies": self.getProxies(),
"ipv6": self.core.config["download"]["ipv6"]}
+
def updateBucket(self):
""" set values in the bucket according to settings"""
if not self.core.config["download"]["limit_speed"]:
diff --git a/pyload/network/XDCCRequest.py b/pyload/network/XDCCRequest.py
index c49f418c4..a4813e039 100644
--- a/pyload/network/XDCCRequest.py
+++ b/pyload/network/XDCCRequest.py
@@ -16,6 +16,7 @@ from pyload.plugin.Plugin import Abort
class XDCCRequest(object):
+
def __init__(self, timeout=30, proxies={}):
self.proxies = proxies
@@ -27,6 +28,7 @@ class XDCCRequest(object):
self.abort = False
+
def createSocket(self):
# proxytype = None
# proxy = None
@@ -46,6 +48,7 @@ class XDCCRequest(object):
return socket.socket()
+
def download(self, ip, port, filename, irc, progress=None):
ircbuffer = ""
@@ -109,6 +112,7 @@ class XDCCRequest(object):
return filename
+
def _keepAlive(self, sock, *readbuffer):
fdset = select([sock], [], [], 0)
if sock not in fdset[0]:
@@ -124,21 +128,29 @@ class XDCCRequest(object):
if first[0] == "PING":
sock.send("PONG %s\r\n" % first[1])
+
def abortDownloads(self):
self.abort = True
@property
+
+
def size(self):
return self.filesize
@property
+
+
def arrived(self):
return self.recv
@property
+
+
def percent(self):
if not self.filesize: return 0
return (self.recv * 100) / self.filesize
+
def close(self):
pass
--
cgit v1.2.3