diff options
Diffstat (limited to 'module/network/Browser.py')
| -rw-r--r-- | module/network/Browser.py | 91 |
1 files changed, 64 insertions, 27 deletions
diff --git a/module/network/Browser.py b/module/network/Browser.py index d70cb3aba..3db2299f0 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -1,51 +1,73 @@ -from HTTPBase import HTTPBase -from HTTPDownload import HTTPDownload -from XDCCBase import XDCCDownload - from random import randint - -import zlib +from helper import * +from os.path import join +from logging import getLogger from cookielib import CookieJar +import zlib + +from HTTPBase import HTTPBase +from HTTPDownload import HTTPDownload from FTPBase import FTPDownload +from XDCCBase import XDCCDownload -class Browser(): +class Browser(object): def __init__(self, interface=None, cookieJar=CookieJar(), bucket=None, proxies={}): + self.log = getLogger("log") + self.lastURL = None self.interface = interface self.cookieJar = cookieJar self.bucket = bucket - + self.http = HTTPBase(interface=interface, proxies=proxies) self.setCookieJar(cookieJar) self.proxies = proxies - + def setCookieJar(self, cookieJar): self.http.cookieJar = cookieJar - + + def clearCookies(self): + pass #@TODO + def clearReferer(self): self.lastURL = None - + def getPage(self, url, get={}, post={}, referer=None, cookies=True, customHeaders={}): if not referer: referer = self.lastURL self.http.followRedirect = True - resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders) + resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, + customHeaders=customHeaders) data = resp.read() try: if resp.info()["Content-Encoding"] == "gzip": - data = zlib.decompress(data, 16+zlib.MAX_WBITS) + data = zlib.decompress(data, 16 + zlib.MAX_WBITS) elif resp.info()["Content-Encoding"] == "deflate": data = zlib.decompress(data, -zlib.MAX_WBITS) except: pass + + try: + content_type = resp.info()["Content-Type"] + infos = [info.strip() for info in content_type.split(";")] + charset = None + for info in infos: + if info.startswith("charset"): + none, charset = info.split("=") + if charset: + data = data.decode(charset) + except Exception, e: + self.log.debug("Could not decode charset: %s" % e) + self.lastURL = resp.geturl() return data - + def getRedirectLocation(self, url, get={}, post={}, referer=None, cookies=True, customHeaders={}): if not referer: referer = self.lastURL self.http.followRedirect = False - resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders) + resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, + customHeaders=customHeaders) resp.close() self.lastURL = resp.geturl() location = None @@ -54,26 +76,40 @@ class Browser(): except: pass return location - - def httpDownload(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, chunks=1, resume=False): + + def download(self, url, file_name, folder, get={}, post={}, ref=True, cookies=True, no_post_encode=False): + #@TODO + + filename = join(folder, file_name) + d = self.httpDownload(url, filename, get, post) + waitFor(d) + + return filename + + def httpDownload(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, chunks=1, + resume=False): if not referer: referer = self.lastURL - - dwnld = HTTPDownload(url, filename, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders, bucket=self.bucket, interface=self.interface, proxies=self.proxies) + + dwnld = HTTPDownload(url, filename, get=get, post=post, referer=referer, cookies=cookies, + customHeaders=customHeaders, bucket=self.bucket, interface=self.interface, + proxies=self.proxies) dwnld.cookieJar = self.cookieJar - + d = dwnld.download(chunks=chunks, resume=resume) return d - + def ftpDownload(self, url, filename, resume=False): dwnld = FTPDownload(url, filename, bucket=self.bucket, interface=self.interface, proxies=self.proxies) - + d = dwnld.download(resume=resume) return d - - def xdccDownload(self, server, port, channel, bot, pack, filename, nick="pyload_%d" % randint(1000,9999), ident="pyload", real="pyloadreal"): - dwnld = XDCCDownload(server, port, channel, bot, pack, nick, ident, real, filename, bucket=self.bucket, interface=self.interface, proxies=self.proxies) - + + def xdccDownload(self, server, port, channel, bot, pack, filename, nick="pyload_%d" % randint(1000, 9999), + ident="pyload", real="pyloadreal"): + dwnld = XDCCDownload(server, port, channel, bot, pack, nick, ident, real, filename, bucket=self.bucket, + interface=self.interface, proxies=self.proxies) + d = dwnld.download() return d @@ -86,8 +122,9 @@ if __name__ == "__main__": #print browser.getRedirectLocation("http://google.com/") #browser.getPage("https://encrypted.google.com/") #browser.getPage("http://google.com/search?q=bar") - + browser.httpDownload("http://speedtest.netcologne.de/test_100mb.bin", "test_100mb.bin") from time import sleep + while True: sleep(1)
\ No newline at end of file |