diff options
Diffstat (limited to 'module/network')
| -rw-r--r-- | module/network/Browser.py | 6 | ||||
| -rw-r--r-- | module/network/Bucket.py | 3 | ||||
| -rw-r--r-- | module/network/HTTPRequest.py | 16 | ||||
| -rw-r--r-- | module/network/RequestFactory.py | 42 |
4 files changed, 56 insertions, 11 deletions
diff --git a/module/network/Browser.py b/module/network/Browser.py index be8dee27d..0a45c1ef4 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -9,17 +9,17 @@ from HTTPDownload import HTTPDownload class Browser(object): - def __init__(self, interface=None, cj=None, bucket=None, proxies={}): + def __init__(self, interface=None, bucket=None, proxies={}): self.log = getLogger("log") self.interface = interface - self.cj = cj self.bucket = bucket self.proxies = proxies + self.cj = None # needs to be setted later self._size = 0 - self.http = HTTPRequest(cj, interface, proxies) + self.http = HTTPRequest(self.cj, interface, proxies) self.dl = None lastEffectiveURL = property(lambda self: self.http.lastEffectiveURL) diff --git a/module/network/Bucket.py b/module/network/Bucket.py index dc1280ede..60d8a757a 100644 --- a/module/network/Bucket.py +++ b/module/network/Bucket.py @@ -29,11 +29,12 @@ class Bucket: def setRate(self, rate): self.lock.acquire() - self.rate = rate + self.rate = int(rate) self.lock.release() def consumed(self, amount): """ return time the process have to sleep, after consumed specified amount """ + if self.rate < 0: return 0 self.lock.acquire() self.calc_tokens() diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 3a240b081..805305f80 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -51,7 +51,6 @@ class HTTPRequest(): self.c.setopt(pycurl.NOPROGRESS, 1) if hasattr(pycurl, "AUTOREFERER"): self.c.setopt(pycurl.AUTOREFERER, 1) - self.c.setopt(pycurl.BUFFERSIZE, 32 * 1024) self.c.setopt(pycurl.SSL_VERIFYPEER, 0) self.c.setopt(pycurl.LOW_SPEED_TIME, 30) self.c.setopt(pycurl.LOW_SPEED_LIMIT, 20) @@ -67,11 +66,22 @@ class HTTPRequest(): "Connection: keep-alive", "Keep-Alive: 300"]) - def setInterface(self, interface, proxies): + def setInterface(self, interface, proxy): if interface and interface.lower() != "none": self.c.setopt(pycurl.INTERFACE, interface) - #@TODO set proxies + if proxy: + if proxy["type"] == "socks4": + self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS4) + elif proxy["type"] == "socks5": + self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5) + else: + self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_HTTP) + + self.c.setopt(pycurl.PROXY, "%s:%s" % (proxy["address"], proxy["port"])) + + if proxy["username"]: + self.c.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (proxy["username"], proxy["password"])) def addCookies(self): if self.cj: diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 6ad64589a..6bc7e3fe7 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -20,6 +20,7 @@ from threading import Lock from Browser import Browser +from Bucket import Bucket from HTTPRequest import HTTPRequest from CookieJar import CookieJar @@ -27,13 +28,14 @@ class RequestFactory(): def __init__(self, core): self.lock = Lock() self.core = core + self.bucket = Bucket() + self.updateBucket() self.cookiejars = {} def getRequest(self, pluginName, account=None): self.lock.acquire() - req = Browser() - #@TODO proxy stuff, bucket + req = Browser(self.core.config["download"]["interface"], self.bucket, self.getProxies()) if account: cj = self.getCookieJar(pluginName, account) @@ -45,8 +47,7 @@ class RequestFactory(): return req def getURL(self, url, get={}, post={}): - #a bit to much overhead for single url - h = HTTPRequest() + h = HTTPRequest(None, self.core.config["download"]["interface"], self.getProxies()) rep = h.load(url, get, post) h.close() return rep @@ -59,6 +60,39 @@ class RequestFactory(): self.cookiejars[(pluginName, account)] = cj return cj + def getProxies(self): + """ returns a proxy list for the request classes """ + if not self.core.config["download"]["proxy"]: + return {} + else: + type = "http" + setting = self.core.config["proxy"]["type"].lower() + if setting == "socks4": type = "socks4" + if setting == "socks5": type = "socks5" + + username = None + if self.core.config["proxy"]["username"] and self.core.config["proxy"]["username"].lower() != "none": + username = self.core.config["proxy"]["username"] + + pw = None + if self.core.config["proxy"]["password"] and self.core.config["proxy"]["password"].lower() != "none": + pw = self.core.config["proxy"]["password"] + + return { + "type": type, + "address": self.core.config["proxy"]["address"], + "port": self.core.config["proxy"]["port"], + "username": username, + "password": pw, + } + + def updateBucket(self): + """ set values in the bucket according to settings""" + if not self.core.config["download"]["limit_speed"]: + self.bucket.setRate(-1) + else: + self.bucket.setRate(self.core.config["download"]["max_speed"] * 1024) + # needs pyreq in global namespace def getURL(url, get={}, post={}): return pyreq.getURL(url, get, post)
\ No newline at end of file |