new download backend integrated so far, downloading works, but still big todo list

10 files changed, 94 insertions, 23 deletions

diff --git a/module/PyFile.py b/module/PyFile.py
index 62588e116..1723808c7 100644
--- a/module/PyFile.py
+++ b/module/PyFile.py
@@ -161,11 +161,13 @@ class PyFile():
         """abort pyfile if possible"""
         while self.id in self.m.core.threadManager.processingIds():
             self.abort = True
-            if self.plugin and self.plugin.req: self.plugin.req.abort = True
+            if self.plugin and self.plugin.req:
+                self.plugin.req.abortDownloads()
             sleep(0.1)
         
         self.abort = False
-        if hasattr(self, "plugin") and self.plugin and self.plugin.req: self.plugin.req.abort = False
+        if hasattr(self, "plugin") and self.plugin and self.plugin.req:
+            self.plugin.req.abortDownloads()
         self.release()
         
     def finishIfDone(self):
@@ -205,7 +207,7 @@ class PyFile():
     def getSpeed(self):
         """ calculates speed """
         try:
-            return self.plugin.req.get_speed()
+            return self.plugin.req.speed/1024 #kb/s
         except:
             return 0
         
diff --git a/module/config/default.conf b/module/config/default.conf
index cd23b3ca1..884891375 100644
--- a/module/config/default.conf
+++ b/module/config/default.conf
@@ -32,6 +32,7 @@ general - "General":
 	bool skip_existing : "Skip already existing files" = False

 	ip download_interface : "Outgoing IP address for downloads" = None

 	int renice : "CPU Priority" = 0

+	int chunks : "Max connections for one download" = 4

 permission - "Permissions":

     bool change_user : "Change user of running process" = False

     str user : "Username" = user

diff --git a/module/gui/Queue.py b/module/gui/Queue.py
index 7c5c59f15..aa1afbd3c 100644
--- a/module/gui/Queue.py
+++ b/module/gui/Queue.py
@@ -88,6 +88,7 @@ class QueueModel(CollectorModel):
                 child = pack.getChild(d["id"])
                 if child:
                     child.data["downloading"] = d
+                    child.data["progress"] = child.data["downloading"]["percent"]
                     k = pack.getChildKey(d["id"])
                     self.emit(SIGNAL("dataChanged(const QModelIndex &, const QModelIndex &)"), self.index(k, 0, self.index(p, 0)), self.index(k, self.cols, self.index(p, self.cols)))
                     
diff --git a/module/network/Browser.py b/module/network/Browser.py
index b68c39c53..28edf8e9d 100644
--- a/module/network/Browser.py
+++ b/module/network/Browser.py
@@ -23,6 +23,9 @@ class Browser(object):
         self.http = HTTPBase(interface=interface, proxies=proxies)
         self.setCookieJar(cookieJar)
         self.proxies = proxies
+        self.abort = property(lambda: False, lambda val: self.abortDownloads())
+        
+        self.downloadConnections = []
 
     def setCookieJar(self, cookieJar):
         self.cookieJar = cookieJar
@@ -78,7 +81,22 @@ class Browser(object):
         except:
             pass
         return location
-
+    
+    def _removeConnection(self, d):
+        i = self.downloadConnections.index(d)
+        del self.downloadConnections[i]
+    
+    def abortDownloads(self):
+        for d in self.downloadConnections:
+            d.abort = True
+    
+    @property
+    def speed(self):
+        speed = 0
+        for d in self.downloadConnections:
+            speed += d.speed
+        return speed
+    
     def httpDownload(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, chunks=1,
                      resume=False):
         if not referer:
@@ -90,12 +108,16 @@ class Browser(object):
         dwnld.cookieJar = self.cookieJar
 
         d = dwnld.download(chunks=chunks, resume=resume)
+        self.downloadConnections.append(d)
+        d.addCallback(self._removeConnection, d)
         return d
 
     def ftpDownload(self, url, filename, resume=False):
         dwnld = FTPDownload(url, filename, bucket=self.bucket, interface=self.interface, proxies=self.proxies)
 
         d = dwnld.download(resume=resume)
+        self.downloadConnections.append(d)
+        d.addCallback(self._removeConnection, d)
         return d
 
     def xdccDownload(self, server, port, channel, bot, pack, filename, nick="pyload_%d" % randint(1000, 9999),
@@ -104,6 +126,8 @@ class Browser(object):
                              interface=self.interface, proxies=self.proxies)
 
         d = dwnld.download()
+        self.downloadConnections.append(d)
+        d.addCallback(self._removeConnection, d)
         return d
     
     #compatibility wrapper
diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py
index 509ff3983..f26e681b6 100644
--- a/module/network/HTTPChunk.py
+++ b/module/network/HTTPChunk.py
@@ -22,6 +22,8 @@ from urllib2 import HTTPError
 from helper import *
 from time import sleep
 from traceback import print_exc
+from module.plugins.Plugin import Abort
+from module.plugins.Plugin import Fail
 
 class HTTPChunk(HTTPBase):
     def __init__(self, url, fh, get={}, post={}, referer=None, cookies=True, customHeaders={}, range=None, bucket=None, interface=None, proxies={}):
@@ -109,11 +111,11 @@ class HTTPChunk(HTTPBase):
         self.fh.close()
         
         if self.abort:
-            self.deferred.error("abort")
+            self.deferred.error(Abort)
         elif self.size == self.arrived:
             self.deferred.callback(resp)
         else:
-            self.deferred.error("wrong content lenght")
+            self.deferred.error(Fail)
     
     def getEncoding(self):
         try:
@@ -127,7 +129,6 @@ class HTTPChunk(HTTPBase):
         if self.range:
             self.customHeaders["Range"] = "bytes=%i-%i" % self.range
         try:
-            print "req"
             resp = self.getResponse(self.url, self.get, self.post, self.referer, self.cookies, self.customHeaders)
             self.resp = resp
         except HTTPError, e:
diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py
index ebc8c8e65..e38e1ba1b 100644
--- a/module/network/HTTPDownload.py
+++ b/module/network/HTTPDownload.py
@@ -274,7 +274,7 @@ class HTTPDownload(object):
             dg.addCallback(self._copyChunks)
             if not len(self.chunks):
                 dg.callback()
-            return WrappedHTTPDeferred(self, self.deferred)
+            return WrappedHTTPDeferred(self, dg)
         else:
             raise Exception("no chunks")
 
diff --git a/module/network/helper.py b/module/network/helper.py
index 8cc61d3ff..ed9a88798 100644
--- a/module/network/helper.py
+++ b/module/network/helper.py
@@ -24,6 +24,7 @@ class Deferred():
     def __init__(self):
         self.call = []
         self.err = []
+        self.prgr = {}
         self.result = ()
         self.errresult = ()
     
@@ -35,6 +36,13 @@ class Deferred():
             kwargs.update(ckwargs)
             callInThread(f, *args, **kwargs)
     
+    #test not in use
+    def addProgress(self, chain, f):
+        if self.prgr.has_key(chain):
+            self.prgr[chain].append(f)
+        else:
+            self.prgr[chain] = [f]
+    
     def addErrback(self, f, *cargs, **ckwargs):
         self.err.append((f, cargs, ckwargs))
         if self.errresult:
@@ -58,6 +66,13 @@ class Deferred():
             args+=tuple(cargs)
             kwargs.update(ckwargs)
             callInThread(f, *args, **kwargs)
+    
+    #test not in use
+    def progress(self, chain, *args, **kwargs):
+        if not self.prgr.has_key(chain):
+            return
+        for f in self.prgr[chain]:
+            callInThread(f, *args, **kwargs)
 
 #decorator
 def threaded(f):
@@ -71,12 +86,17 @@ def waitFor(d):
         args = ()
         err = None
         
+        def __init__(self, d):
+            self.d = d
+        
         def wait(self):
-            d.addCallback(self.callb)
-            d.addErrback(self.errb)
+            self.d.addCallback(self.callb)
+            self.d.addErrback(self.errb)
             while self.waiting:
                 sleep(0.5)
             if self.err:
+                if issubclass(self.err[0][0], Exception):
+                    raise self.err[0][0]()
                 raise Exception(self.err)
             return self.args
         
@@ -87,7 +107,7 @@ def waitFor(d):
         def errb(self, *args, **kwargs):
             self.waiting = False
             self.err = (args, kwargs)
-    w = Waiter()
+    w = Waiter(d)
     return w.wait()
 
 class DeferredGroup(Deferred):
@@ -110,16 +130,25 @@ class DeferredGroup(Deferred):
         if len(self.group) == self.done:
             self.callback()
 
-class WrappedDeferred():
+class WrappedDeferred(object):
     def __init__(self, download, d):
-        self.download = download
-        self.d = d
+        self.__dict__["download"] = download
+        self.__dict__["d"] = d
     
     def addCallback(self, *args, **kwargs):
-        self.d.addCallback(*args, **kwargs)
+        self.__dict__["d"].addCallback(*args, **kwargs)
     
     def addErrback(self, *args, **kwargs):
-        self.d.addErrback(*args, **kwargs)
+        self.__dict__["d"].addErrback(*args, **kwargs)
+    
+    def callback(self, *args, **kwargs):
+        self.__dict__["d"].callback(*args, **kwargs)
+    
+    def error(self, *args, **kwargs):
+        self.__dict__["d"].error(*args, **kwargs)
     
     def __getattr__(self, attr):
         return getattr(self.download, attr)
+    
+    def __setattr__(self, attr, val):
+        return setattr(self.download, attr, val)
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index f2c3b18eb..4af3d0d8b 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -42,6 +42,8 @@ from mimetypes import guess_type
 
 from itertools import islice
 
+from module.network.helper import waitFor
+
 def chunks(iterable, size):
   it = iter(iterable)
   item = list(islice(it, size))
@@ -86,6 +88,8 @@ class Plugin(object):
         self.wantReconnect = False
         self.multiDL = True
         self.limitDL = 0
+        self.chunkLimit = 1
+        self.resumeDownload = False
 
         self.waitUntil = 0 # time() + wait in seconds
         self.waiting = False
@@ -113,7 +117,12 @@ class Plugin(object):
         self.js = self.core.js  # js engine
 
         #self.setup()
-
+    
+    def getChunkCount(self):
+        if self.chunkLimit <= 0:
+            return self.config["general"]["chunks"]
+        return min(self.config["general"]["chunks"], self.chunkLimit)
+    
     def __call__(self):
         return self.__name__
 
@@ -283,7 +292,7 @@ class Plugin(object):
         """ returns the content loaded """
         if self.pyfile.abort: raise Abort
 
-        res  = self.req.load(url, get, post, ref, cookies, just_header, no_post_encode, raw_cookies)
+        res = self.req.getPage(url, get=get, post=post, cookies=cookies)
         if self.core.debug:
             from inspect import currentframe
             frame = currentframe()
@@ -320,8 +329,10 @@ class Plugin(object):
                     self.log.warning(_("Setting User and Group failed: %s") % str(e))
 
         name = self.pyfile.name.encode(sys.getfilesystemencoding(), "replace")
-        newname = self.req.download(url, name, location, get, post, ref, cookies)
-        newname = basename(newname)
+        filename = join(location, name)
+        d = self.req.httpDownload(url, filename, get=get, post=post, chunks=self.getChunkCount(), resume=self.resumeDownload)
+        waitFor(d)
+        newname = basename(filename)
         
         self.pyfile.size = self.req.dl_size
 
diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py
index 8370aebb7..8163ffc44 100644
--- a/module/plugins/hoster/RapidshareCom.py
+++ b/module/plugins/hoster/RapidshareCom.py
@@ -66,7 +66,8 @@ class RapidshareCom(Hoster):
 
         if self.account:
             self.multiDL = True
-            self.req.canContinue = True
+            self.chunkLimit = -1
+            self.resumeDownload = True
 
     def process(self, pyfile):
         self.url = self.pyfile.url        
diff --git a/module/plugins/hoster/UploadedTo.py b/module/plugins/hoster/UploadedTo.py
index 5d4798d4e..1bfafc9c6 100644
--- a/module/plugins/hoster/UploadedTo.py
+++ b/module/plugins/hoster/UploadedTo.py
@@ -38,7 +38,8 @@ class UploadedTo(Hoster):
         self.url = False
         if self.account:
             self.multiDL = True
-            self.req.canContinue = True
+            self.chunkLimit = -1
+            self.resumeDownload = True
         
 
     def process(self, pyfile):
@@ -157,4 +158,4 @@ class UploadedTo(Hoster):
         url = url.replace("/?id=", "/file/")
         url = url.replace("?id=", "file/")
         url = re.sub("/\?(.*?)&id=", "/file/", url, 1)
-        return url
\ No newline at end of file
+        return url