From d52bcc46770e2f932b12fcaac45d056846814ef1 Mon Sep 17 00:00:00 2001
From: RaNaN <Mast3rRaNaN@hotmail.de>
Date: Wed, 22 Dec 2010 12:53:38 +0100
Subject: at least its downloading now
---
module/network/Browser.py | 91 +++++++++++++++++++++++++++++-------------
module/network/HTTPDownload.py | 2 +-
module/network/Request.py | 18 +++++----
3 files changed, 75 insertions(+), 36 deletions(-)
(limited to 'module/network')
diff --git a/module/network/Browser.py b/module/network/Browser.py
index d70cb3aba..3db2299f0 100644
--- a/module/network/Browser.py
+++ b/module/network/Browser.py
@@ -1,51 +1,73 @@
-from HTTPBase import HTTPBase
-from HTTPDownload import HTTPDownload
-from XDCCBase import XDCCDownload
-
from random import randint
-
-import zlib
+from helper import *
+from os.path import join
+from logging import getLogger
from cookielib import CookieJar
+import zlib
+
+from HTTPBase import HTTPBase
+from HTTPDownload import HTTPDownload
from FTPBase import FTPDownload
+from XDCCBase import XDCCDownload
-class Browser():
+class Browser(object):
def __init__(self, interface=None, cookieJar=CookieJar(), bucket=None, proxies={}):
+ self.log = getLogger("log")
+
self.lastURL = None
self.interface = interface
self.cookieJar = cookieJar
self.bucket = bucket
-
+
self.http = HTTPBase(interface=interface, proxies=proxies)
self.setCookieJar(cookieJar)
self.proxies = proxies
-
+
def setCookieJar(self, cookieJar):
self.http.cookieJar = cookieJar
-
+
+ def clearCookies(self):
+ pass #@TODO
+
def clearReferer(self):
self.lastURL = None
-
+
def getPage(self, url, get={}, post={}, referer=None, cookies=True, customHeaders={}):
if not referer:
referer = self.lastURL
self.http.followRedirect = True
- resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders)
+ resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies,
+ customHeaders=customHeaders)
data = resp.read()
try:
if resp.info()["Content-Encoding"] == "gzip":
- data = zlib.decompress(data, 16+zlib.MAX_WBITS)
+ data = zlib.decompress(data, 16 + zlib.MAX_WBITS)
elif resp.info()["Content-Encoding"] == "deflate":
data = zlib.decompress(data, -zlib.MAX_WBITS)
except:
pass
+
+ try:
+ content_type = resp.info()["Content-Type"]
+ infos = [info.strip() for info in content_type.split(";")]
+ charset = None
+ for info in infos:
+ if info.startswith("charset"):
+ none, charset = info.split("=")
+ if charset:
+ data = data.decode(charset)
+ except Exception, e:
+ self.log.debug("Could not decode charset: %s" % e)
+
self.lastURL = resp.geturl()
return data
-
+
def getRedirectLocation(self, url, get={}, post={}, referer=None, cookies=True, customHeaders={}):
if not referer:
referer = self.lastURL
self.http.followRedirect = False
- resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders)
+ resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies,
+ customHeaders=customHeaders)
resp.close()
self.lastURL = resp.geturl()
location = None
@@ -54,26 +76,40 @@ class Browser():
except:
pass
return location
-
- def httpDownload(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, chunks=1, resume=False):
+
+ def download(self, url, file_name, folder, get={}, post={}, ref=True, cookies=True, no_post_encode=False):
+ #@TODO
+
+ filename = join(folder, file_name)
+ d = self.httpDownload(url, filename, get, post)
+ waitFor(d)
+
+ return filename
+
+ def httpDownload(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, chunks=1,
+ resume=False):
if not referer:
referer = self.lastURL
-
- dwnld = HTTPDownload(url, filename, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders, bucket=self.bucket, interface=self.interface, proxies=self.proxies)
+
+ dwnld = HTTPDownload(url, filename, get=get, post=post, referer=referer, cookies=cookies,
+ customHeaders=customHeaders, bucket=self.bucket, interface=self.interface,
+ proxies=self.proxies)
dwnld.cookieJar = self.cookieJar
-
+
d = dwnld.download(chunks=chunks, resume=resume)
return d
-
+
def ftpDownload(self, url, filename, resume=False):
dwnld = FTPDownload(url, filename, bucket=self.bucket, interface=self.interface, proxies=self.proxies)
-
+
d = dwnld.download(resume=resume)
return d
-
- def xdccDownload(self, server, port, channel, bot, pack, filename, nick="pyload_%d" % randint(1000,9999), ident="pyload", real="pyloadreal"):
- dwnld = XDCCDownload(server, port, channel, bot, pack, nick, ident, real, filename, bucket=self.bucket, interface=self.interface, proxies=self.proxies)
-
+
+ def xdccDownload(self, server, port, channel, bot, pack, filename, nick="pyload_%d" % randint(1000, 9999),
+ ident="pyload", real="pyloadreal"):
+ dwnld = XDCCDownload(server, port, channel, bot, pack, nick, ident, real, filename, bucket=self.bucket,
+ interface=self.interface, proxies=self.proxies)
+
d = dwnld.download()
return d
@@ -86,8 +122,9 @@ if __name__ == "__main__":
#print browser.getRedirectLocation("http://google.com/")
#browser.getPage("https://encrypted.google.com/")
#browser.getPage("http://google.com/search?q=bar")
-
+
browser.httpDownload("http://speedtest.netcologne.de/test_100mb.bin", "test_100mb.bin")
from time import sleep
+
while True:
sleep(1)
\ No newline at end of file
diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py
index 2b8c44a87..ebc8c8e65 100644
--- a/module/network/HTTPDownload.py
+++ b/module/network/HTTPDownload.py
@@ -105,7 +105,7 @@ class ChunkInfo():
class WrappedHTTPDeferred(WrappedDeferred):
pass
-class HTTPDownload():
+class HTTPDownload(object):
def __init__(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, bucket=None, interface=None, proxies={}):
self.url = url
self.filename = filename
diff --git a/module/network/Request.py b/module/network/Request.py
index 9ac7d54aa..b0a8063ef 100755
--- a/module/network/Request.py
+++ b/module/network/Request.py
@@ -29,7 +29,7 @@ from module.plugins.Plugin import Abort
from module.network.Browser import Browser
from module.network.helper import waitFor
-class Request:
+class Request(object):
def __init__(self, interface=None):
self.browser = Browser(interface=interface)
self.d = None
@@ -60,7 +60,8 @@ class Request:
self.timeout = int(timeout)
def setCookieJar(self, j):
- self.cookieJar = j
+ #self.cookieJar = j
+ pass
def addCookies(self):
#@TODO
@@ -109,12 +110,6 @@ class Request:
self.browser.clearReferer()
self.d = self.browser.httpDownload(url, file_temp, get=get, post=post, cookies=cookies, chunks=1, resume=self.canContinue)
- self.dl_time = property(lambda: self.d.startTime)
- self.dl_finished = property(lambda: self.d.endTime)
- self.dl_speed = property(lambda: self.d.speed)
- self.dl_size = property(lambda: self.d.size)
- self.dl = property(lambda: True if self.d.startTime and not self.d.endTime else False)
- self.abort = property(self.d.getAbort, self.d.setAbort)
waitFor(self.d)
@@ -132,6 +127,13 @@ class Request:
return free_name
+ dl_time = property(lambda self: self.d.startTime)
+ dl_finished = property(lambda self: self.d.endTime)
+ dl_speed = property(lambda self: self.d.speed)
+ dl_size = property(lambda self: self.d.size)
+ dl = property(lambda self: True if self.d.startTime and not self.d.endTime else False)
+ abort = property(lambda self: self.d.getAbort, lambda self, value: self.d.setAbort(value))
+
def get_speed(self):
try:
return self.dl_speed
--
cgit v1.2.3