fixed weird issues, comments

7 files changed, 101 insertions, 79 deletions

diff --git a/module/network/Browser.py b/module/network/Browser.py
index 28edf8e9d..512a437ed 100644
--- a/module/network/Browser.py
+++ b/module/network/Browser.py
@@ -82,8 +82,8 @@ class Browser(object):
             pass
         return location
     
-    def _removeConnection(self, d):
-        i = self.downloadConnections.index(d)
+    def _removeConnection(self, *args, **kwargs):
+        i = self.downloadConnections.index(args[-1])
         del self.downloadConnections[i]
     
     def abortDownloads(self):
@@ -110,6 +110,7 @@ class Browser(object):
         d = dwnld.download(chunks=chunks, resume=resume)
         self.downloadConnections.append(d)
         d.addCallback(self._removeConnection, d)
+        d.addErrback(self._removeConnection, d)
         return d
 
     def ftpDownload(self, url, filename, resume=False):
diff --git a/module/network/HTTPBase.py b/module/network/HTTPBase.py
index 2737fd33c..7627c8499 100644
--- a/module/network/HTTPBase.py
+++ b/module/network/HTTPBase.py
@@ -333,8 +333,9 @@ class HTTPBase():
             print "[HTTP] creating request"
             print "[HTTP] URL:", url
             print "[HTTP] GET"
-            for key, value in get.iteritems(): 
-                print "[HTTP] \t", key, ":", value
+            if get:
+                for key, value in get.iteritems(): 
+                    print "[HTTP] \t", key, ":", value
             if post:
                 print "[HTTP] POST"
                 for key, value in post.iteritems(): 
diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py
index 48ebc13f3..02134ca63 100644
--- a/module/network/HTTPChunk.py
+++ b/module/network/HTTPChunk.py
@@ -85,7 +85,7 @@ class HTTPChunk(HTTPBase):
             try:
                 data = resp.read(count)
             except:
-                self.deferred.error("timeout")
+                self.deferred.error(Fail, "timeout")
                 break
             
             if self.speedCalcTime < inttime():
@@ -104,7 +104,7 @@ class HTTPChunk(HTTPBase):
             if self.noRangeHeader and self.arrived == self.range[1]:
                 running = False
 
-            if data:
+            if size:
                 self.fh.write(data)
             else:
                 break
@@ -117,9 +117,10 @@ class HTTPChunk(HTTPBase):
         if self.abort:
             self.deferred.error(Abort)
         elif self.size == self.arrived:
-            self.deferred.callback(resp)
+            self.deferred.callback()
         else:
-            self.deferred.error(Fail)
+            print self.arrived, self.size
+            self.deferred.error(Fail, "wrong content-length")
     
     def getEncoding(self):
         try:
@@ -140,7 +141,7 @@ class HTTPChunk(HTTPBase):
             self.deferred.error(e)
             return self.deferred
         
-        if (self.range and resp.getcode() == 206) or (not self.range and resp.getcode() == 200):
+        if resp.getcode() in (200, 206):
             self._download(resp)
         else:
             self.deferred.error(resp.getcode(), resp)
diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py
index 1baf0ccad..b81008cc5 100644
--- a/module/network/HTTPDownload.py
+++ b/module/network/HTTPDownload.py
@@ -105,7 +105,7 @@ class ChunkInfo():
 class WrappedHTTPDeferred(WrappedDeferred):
     pass
     
-class HTTPDownload(object):
+class HTTPDownload():
     def __init__(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, bucket=None, interface=None, proxies={}):
         self.url = url
         self.filename = filename
@@ -131,6 +131,7 @@ class HTTPDownload(object):
         self.cookieJar = CookieJar()
         
         self.chunks = []
+        self.chunksDone = 0
         try:
             self.info = ChunkInfo.load(filename)
         except IOError:
@@ -170,11 +171,18 @@ class HTTPDownload(object):
     def calcProgress(self, p):
         self.deferred.progress("percent", 100-int((self.size - self.arrived)/float(self.size)*100))
     
+    def _chunkDone(self):
+        self.chunksDone += 1
+        print self.chunksDone, "/", len(self.chunks)
+        if self.chunksDone == len(self.chunks):
+            self._copyChunks()
+    
     def _copyChunks(self):
-        fo = open(self.filename, "wb")
+        fo = open(self.filename, "wb") #out file
         for i in range(self.info.getCount()):
             encoding = self.info.getChunkEncoding(i)
             
+            #decompress method, if any
             decompress = lambda data: data
             if encoding == "gzip":
                 gz = decompressobj(16+MAX_WBITS)
@@ -183,18 +191,19 @@ class HTTPDownload(object):
                 df = decompressobj(-MAX_WBITS)
                 decompress = lambda data: df.decompress(data)
             
+            #input file
             fname = "%s.chunk%d" % (self.filename, i)
             fi = open(fname, "rb")
-            while True:
+            while True: #copy in chunks, consumes less memory
                 data = fi.read(512*1024)
                 if not data:
                     break
-                fo.write(decompress(data))
+                fo.write(decompress(data)) #decompressing
             fi.close()
-            remove(fname)
+            remove(fname) #remove
         fo.close()
-        self.info.removeInfo()
-        self.deferred.callback()
+        self.info.removeInfo() #remove info file
+        self.deferred.callback() #done, emit callbacks
     
     def _createChunk(self, fh, range=None):
         chunk = HTTPChunk(self.url, fh, get=self.get, post=self.post,
@@ -205,81 +214,92 @@ class HTTPDownload(object):
         chunk.cookieJar = self.cookieJar
         return chunk
     
+    def _addChunk(self, chunk, d):
+        self.chunks.append(chunk)
+        d.addProgress("percent", self.calcProgress)
+        d.addCallback(self._chunkDone)
+        d.addErrback(lambda *args, **kwargs: self.setAbort(True))
+        d.addErrback(self.deferred.error)
+    
     def download(self, chunks=1, resume=False):
+        self.chunksDone = 0
         if chunks > 0:
-            dg = DeferredGroup()
+            #diffentent chunk count in info, resetting
             if self.info.loaded and not self.info.getCount() == chunks:
                 self.info.clear()
+            
+            #if resuming, calculate range with offset
             crange = None
             if resume:
                 if self.info.getCount() == chunks and exists("%s.chunk0" % (self.filename, )):
                     crange = self.info.getChunkRange(0)
                     crange = (crange[0]+getsize("%s.chunk0" % (self.filename, )), crange[1]-1)
             
+            #if firstpart not done
             if crange is None or crange[1]-crange[0] > 0:
                 fh = open("%s.chunk0" % (self.filename, ), "ab" if crange else "wb")
+                
                 chunk = self._createChunk(fh, range=crange)
-                self.chunks.append(chunk)
-                d = chunk.download()
-                dg.addDeferred(d)
-                d.addProgress("percent", self.calcProgress)
                 
+                d = chunk.download() #start downloading
+                self._addChunk(chunk, d)
+                
+                #no info file, need to calculate ranges
                 if not self.info.loaded:
-                    size = chunk.size
-                    chunksize = size/chunks
-                    lastchunk = chunksize
+                    size = chunk.size #overall size
+                    chunksize = size/chunks #chunk size
                     
-                    chunk.range = (0, chunksize)
+                    chunk.range = (0, chunksize) #setting range for first part
                     chunk.noRangeHeader = True
-                    self.size = chunk.size
-                    self.info.setSize(self.size)
-                    chunk.size = chunksize
-                    self.info.addChunk("%s.chunk0" % (self.filename, ), chunk.range, chunk.getEncoding())
+                    chunk.size = chunksize #setting size for first chunk
                     
-                    lastchunk = size - chunksize*(chunks-1)
-                else:
-                    self.size = self.info.size
-                self.firstchunk = chunk
+                    self.size = size #setting overall size
+                    self.info.setSize(self.size) #saving overall size
+                    self.info.addChunk("%s.chunk0" % (self.filename, ), chunk.range, chunk.getEncoding()) #add chunk to infofile
+                    
+                    lastchunk = size - chunksize*(chunks-1) #calculating size for last chunk
+                self.firstchunk = chunk #remeber first chunk
+            
+            if self.info.loaded and not self.size:
+                self.size = self.info.size #setting overall size
             
-            for i in range(1, chunks):
+            for i in range(1, chunks): #other chunks
                 cont = False
                 if not self.info.loaded: #first time load
-                    if i+1 == chunks:
-                        rng = (i*chunksize, i*chunksize+lastchunk)
+                    if i+1 == chunks: #last chunk?
+                        rng = (i*chunksize, i*chunksize+lastchunk-1)
                     else:
-                        rng = (i*chunksize, (i+1)*chunksize-1)
-                else: #not finished
-                    rng = self.info.getChunkRange(i)
+                        rng = (i*chunksize, (i+1)*chunksize-1) #adjusting range
+                else: #info available
+                    rng = self.info.getChunkRange(i) #loading previous range
                     if resume and exists("%s.chunk%d" % (self.filename, i)): #continue chunk
-                        rng = (rng[0]+getsize("%s.chunk%d" % (self.filename, i)), rng[1])
-                        cont = True
+                        rng = (rng[0]+getsize("%s.chunk%d" % (self.filename, i)), rng[1]) #adjusting offset
+                        cont = True #set append mode
                 
                 if rng[1]-rng[0] <= 0: #chunk done
                     continue
                 
                 fh = open("%s.chunk%d" % (self.filename, i), "ab" if cont else "wb")
                 chunk = self._createChunk(fh, range=rng)
-                d = chunk.download()
-                if not chunk.resp.getcode() == 206 and i == 1: #no range supported, tell chunk0 to download everything
+                d = chunk.download() #try
+                
+                if not chunk.resp.getcode() == 206 and i == 1: #no range supported, tell first chunk to download everything
                     chunk.abort = True
                     self.noChunkSupport = True
                     self.firstchunk.size = self.size
                     self.firstchunk.range = None
-                    self.info.clear()
-                    self.info.addChunk("%s.chunk0" % (self.filename, ), (0, self.firstchunk.size), chunk.getEncoding())
+                    self.info.clear() #clear info
+                    self.info.addChunk("%s.chunk0" % (self.filename, ), (0, self.firstchunk.size), chunk.getEncoding()) #re-adding info with correct ranges
                     break
-                self.chunks.append(chunk)
-                dg.addDeferred(d)
-                d.addProgress("percent", self.calcProgress)
                 
-                if not self.info.loaded:
+                self._addChunk(chunk, d)
+                
+                if not self.info.loaded: #adding info
                     self.info.addChunk("%s.chunk%d" % (self.filename, i), chunk.range, chunk.getEncoding())
             
-            self.info.save()
-            dg.addCallback(self._copyChunks)
+            self.info.save() #saving info
             if not len(self.chunks):
-                dg.callback()
-            dg.addErrback(self.deferred.error)
+                self._copyChunks()
             return WrappedHTTPDeferred(self, self.deferred)
         else:
             raise Exception("no chunks")
diff --git a/module/network/helper.py b/module/network/helper.py
index ebe0fa8f5..dac0185ce 100644
--- a/module/network/helper.py
+++ b/module/network/helper.py
@@ -51,13 +51,13 @@ class Deferred():
             callInThread(f, *args, **kwargs)
     
     def callback(self, *args, **kwargs):
-        if self.result:
-            raise AlreadyCalled
         self.result = (args, kwargs)
         for f, cargs, ckwargs in self.call:
             args+=tuple(cargs)
             kwargs.update(ckwargs)
             callInThread(f, *args, **kwargs)
+        self.call = []
+        self.result = ()
     
     def error(self, *args, **kwargs):
         self.errresult = (args, kwargs)
@@ -65,6 +65,8 @@ class Deferred():
             args+=tuple(cargs)
             kwargs.update(ckwargs)
             callInThread(f, *args, **kwargs)
+        self.err = []
+        self.errresult = ()
     
     def progress(self, chain, *args, **kwargs):
         if not self.prgr.has_key(chain):
@@ -93,9 +95,12 @@ def waitFor(d):
             while self.waiting:
                 sleep(0.5)
             if self.err:
+                #try:
                 if issubclass(self.err[0][0], Exception):
-                    raise self.err[0][0]()
-                raise Exception(self.err)
+                    raise self.err[0][0](*self.err[0][1:], **self.err[1])
+                #except:
+                #    pass
+                raise Exception(*self.err[0], **self.err[1])
             return self.args
         
         def callb(self, *args, **kwargs):
@@ -127,32 +132,25 @@ class DeferredGroup(Deferred):
         self.done += 1
         if len(self.group) == self.done:
             self.callback()
+            self.group = []
+            self.done = 0
+    
+    def error(self, *args, **kwargs):
+        Deferred.error(self, *args, **kwargs)
+        self.group = []
+        self.done = 0
 
 class WrappedDeferred(object):
     def __init__(self, download, d):
         self.__dict__["download"] = download
         self.__dict__["d"] = d
     
-    def addCallback(self, *args, **kwargs):
-        self.__dict__["d"].addCallback(*args, **kwargs)
-    
-    def addErrback(self, *args, **kwargs):
-        self.__dict__["d"].addErrback(*args, **kwargs)
-    
-    def addProgress(self, *args, **kwargs):
-        self.__dict__["d"].addProgress(*args, **kwargs)
-    
-    def callback(self, *args, **kwargs):
-        self.__dict__["d"].callback(*args, **kwargs)
-    
-    def error(self, *args, **kwargs):
-        self.__dict__["d"].error(*args, **kwargs)
-    
-    def progress(self, *args, **kwargs):
-        self.__dict__["d"].progress(*args, **kwargs)
-    
     def __getattr__(self, attr):
-        return getattr(self.download, attr)
+        if attr in ("addCallback", "addErrback", "addProgress", "callback", "error", "progress"):
+            return getattr(self.__dict__["d"], attr)
+        return getattr(self.__dict__["download"], attr)
     
     def __setattr__(self, attr, val):
-        return setattr(self.download, attr, val)
+        if attr in ("addCallback", "addErrback", "addProgress", "callback", "error", "progress"):
+            return setattr(self.__dict__["d"], attr, val)
+        return setattr(self.__dict__["download"], attr, val)
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index 6a7ec5b14..95e937d1c 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -336,7 +336,7 @@ class Plugin(object):
         waitFor(d)
         self.pyfile.download = None
         newname = basename(filename)
-        
+
         self.pyfile.size = d.size
 
         if newname and newname != name:
diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py
index a0f50cfeb..d7aee2c27 100644
--- a/module/plugins/hoster/NetloadIn.py
+++ b/module/plugins/hoster/NetloadIn.py
@@ -65,7 +65,8 @@ class NetloadIn(Hoster):
         self.multiDL = False
         if self.account:
             self.multiDL = True
-            self.req.canContinue = True
+            self.chunkLimit = -1
+            self.resumeDownload = True
     
     def process(self, pyfile):
         self.url = pyfile.url