diff options
Diffstat (limited to 'module/network')
| -rw-r--r-- | module/network/Browser.py | 91 | ||||
| -rw-r--r-- | module/network/HTTPDownload.py | 2 | ||||
| -rwxr-xr-x | module/network/Request.py | 18 |
3 files changed, 75 insertions, 36 deletions
diff --git a/module/network/Browser.py b/module/network/Browser.py index d70cb3aba..3db2299f0 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -1,51 +1,73 @@ -from HTTPBase import HTTPBase -from HTTPDownload import HTTPDownload -from XDCCBase import XDCCDownload - from random import randint - -import zlib +from helper import * +from os.path import join +from logging import getLogger from cookielib import CookieJar +import zlib + +from HTTPBase import HTTPBase +from HTTPDownload import HTTPDownload from FTPBase import FTPDownload +from XDCCBase import XDCCDownload -class Browser(): +class Browser(object): def __init__(self, interface=None, cookieJar=CookieJar(), bucket=None, proxies={}): + self.log = getLogger("log") + self.lastURL = None self.interface = interface self.cookieJar = cookieJar self.bucket = bucket - + self.http = HTTPBase(interface=interface, proxies=proxies) self.setCookieJar(cookieJar) self.proxies = proxies - + def setCookieJar(self, cookieJar): self.http.cookieJar = cookieJar - + + def clearCookies(self): + pass #@TODO + def clearReferer(self): self.lastURL = None - + def getPage(self, url, get={}, post={}, referer=None, cookies=True, customHeaders={}): if not referer: referer = self.lastURL self.http.followRedirect = True - resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders) + resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, + customHeaders=customHeaders) data = resp.read() try: if resp.info()["Content-Encoding"] == "gzip": - data = zlib.decompress(data, 16+zlib.MAX_WBITS) + data = zlib.decompress(data, 16 + zlib.MAX_WBITS) elif resp.info()["Content-Encoding"] == "deflate": data = zlib.decompress(data, -zlib.MAX_WBITS) except: pass + + try: + content_type = resp.info()["Content-Type"] + infos = [info.strip() for info in content_type.split(";")] + charset = None + for info in infos: + if info.startswith("charset"): + none, charset = info.split("=") + if charset: + data = data.decode(charset) + except Exception, e: + self.log.debug("Could not decode charset: %s" % e) + self.lastURL = resp.geturl() return data - + def getRedirectLocation(self, url, get={}, post={}, referer=None, cookies=True, customHeaders={}): if not referer: referer = self.lastURL self.http.followRedirect = False - resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders) + resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, + customHeaders=customHeaders) resp.close() self.lastURL = resp.geturl() location = None @@ -54,26 +76,40 @@ class Browser(): except: pass return location - - def httpDownload(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, chunks=1, resume=False): + + def download(self, url, file_name, folder, get={}, post={}, ref=True, cookies=True, no_post_encode=False): + #@TODO + + filename = join(folder, file_name) + d = self.httpDownload(url, filename, get, post) + waitFor(d) + + return filename + + def httpDownload(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, chunks=1, + resume=False): if not referer: referer = self.lastURL - - dwnld = HTTPDownload(url, filename, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders, bucket=self.bucket, interface=self.interface, proxies=self.proxies) + + dwnld = HTTPDownload(url, filename, get=get, post=post, referer=referer, cookies=cookies, + customHeaders=customHeaders, bucket=self.bucket, interface=self.interface, + proxies=self.proxies) dwnld.cookieJar = self.cookieJar - + d = dwnld.download(chunks=chunks, resume=resume) return d - + def ftpDownload(self, url, filename, resume=False): dwnld = FTPDownload(url, filename, bucket=self.bucket, interface=self.interface, proxies=self.proxies) - + d = dwnld.download(resume=resume) return d - - def xdccDownload(self, server, port, channel, bot, pack, filename, nick="pyload_%d" % randint(1000,9999), ident="pyload", real="pyloadreal"): - dwnld = XDCCDownload(server, port, channel, bot, pack, nick, ident, real, filename, bucket=self.bucket, interface=self.interface, proxies=self.proxies) - + + def xdccDownload(self, server, port, channel, bot, pack, filename, nick="pyload_%d" % randint(1000, 9999), + ident="pyload", real="pyloadreal"): + dwnld = XDCCDownload(server, port, channel, bot, pack, nick, ident, real, filename, bucket=self.bucket, + interface=self.interface, proxies=self.proxies) + d = dwnld.download() return d @@ -86,8 +122,9 @@ if __name__ == "__main__": #print browser.getRedirectLocation("http://google.com/") #browser.getPage("https://encrypted.google.com/") #browser.getPage("http://google.com/search?q=bar") - + browser.httpDownload("http://speedtest.netcologne.de/test_100mb.bin", "test_100mb.bin") from time import sleep + while True: sleep(1)
\ No newline at end of file diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index 2b8c44a87..ebc8c8e65 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -105,7 +105,7 @@ class ChunkInfo(): class WrappedHTTPDeferred(WrappedDeferred): pass -class HTTPDownload(): +class HTTPDownload(object): def __init__(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, bucket=None, interface=None, proxies={}): self.url = url self.filename = filename diff --git a/module/network/Request.py b/module/network/Request.py index 9ac7d54aa..b0a8063ef 100755 --- a/module/network/Request.py +++ b/module/network/Request.py @@ -29,7 +29,7 @@ from module.plugins.Plugin import Abort from module.network.Browser import Browser from module.network.helper import waitFor -class Request: +class Request(object): def __init__(self, interface=None): self.browser = Browser(interface=interface) self.d = None @@ -60,7 +60,8 @@ class Request: self.timeout = int(timeout) def setCookieJar(self, j): - self.cookieJar = j + #self.cookieJar = j + pass def addCookies(self): #@TODO @@ -109,12 +110,6 @@ class Request: self.browser.clearReferer() self.d = self.browser.httpDownload(url, file_temp, get=get, post=post, cookies=cookies, chunks=1, resume=self.canContinue) - self.dl_time = property(lambda: self.d.startTime) - self.dl_finished = property(lambda: self.d.endTime) - self.dl_speed = property(lambda: self.d.speed) - self.dl_size = property(lambda: self.d.size) - self.dl = property(lambda: True if self.d.startTime and not self.d.endTime else False) - self.abort = property(self.d.getAbort, self.d.setAbort) waitFor(self.d) @@ -132,6 +127,13 @@ class Request: return free_name + dl_time = property(lambda self: self.d.startTime) + dl_finished = property(lambda self: self.d.endTime) + dl_speed = property(lambda self: self.d.speed) + dl_size = property(lambda self: self.d.size) + dl = property(lambda self: True if self.d.startTime and not self.d.endTime else False) + abort = property(lambda self: self.d.getAbort, lambda self, value: self.d.setAbort(value)) + def get_speed(self): try: return self.dl_speed |