remove links from txt + several improvments

4 files changed, 90 insertions, 96 deletions

diff --git a/module/Py_Load_File.py b/module/Py_Load_File.py
index 97479384c..9fcd554a5 100644
--- a/module/Py_Load_File.py
+++ b/module/Py_Load_File.py
@@ -11,7 +11,7 @@ class PyLoadFile:
         self.url = url
         self.filename = "filename"
         self.download_folder = ""
-        self.modul = __import__(self._get_my_plugin())  #maybe replace to prepare download
+        self.modul = __import__(self._get_my_plugin())
         pluginClass = getattr(self.modul, self.modul.__name__)
         self.plugin = pluginClass(self)
         self.status = Status(self)
@@ -22,7 +22,7 @@ class PyLoadFile:
         for plugin, plugin_pattern in self.parent.plugins_avaible.items():
             if re.match(plugin_pattern, self.url) != None:
                 return plugin
-        #logger: kein plugin gefunden # was soll passieren wenn nichts gefunden wird?!?
+
         return "Plugin"
     
     def prepareDownload(self):
diff --git a/module/download_thread.py b/module/download_thread.py
index d5e4f4edc..5313a7d34 100644
--- a/module/download_thread.py
+++ b/module/download_thread.py
@@ -19,7 +19,6 @@
 ###
 
 import threading
-import random
 from time import time, sleep
 
 
@@ -30,35 +29,12 @@ class Status(object):
         self.pyfile = pyfile
         self.type = None
         self.status_queue = None
-        self.total_kb = 0
-        self.downloaded_kb = 0
-        self.rate = 0
-        self.expected_time = 0
         self.filename = None
         self.url = None
         self.exists = False
         self.waituntil = None
         self.want_reconnect = False
     
-#    def __call__(self, blocks_read, block_size, total_size):
-#        if self.status_queue == None:
-#            return False
-#        self.start = time()
-#        self.last_status = time()
-#        self.total_kb = total_size / 1024
-#        self.downloaded_kb = (blocks_read * block_size) / 1024
-#        elapsed_time = time() - self.start
-#        if elapsed_time != 0:
-#            self.rate = self.downloaded_kb / elapsed_time
-#            if self.rate != 0:
-#                self.expected_time = self.downloaded_kb / self.rate
-#        if self.last_status+0.2 < time():
-#            self.status_queue.put(copy(self))
-#            self.last_status = time()
-#
-    def set_status_queue(self, queue):
-        self.status_queue = queue
-
     def get_ETA(self):
         return self.pyfile.plugin.req.get_ETA()
     def get_speed(self):
@@ -86,6 +62,7 @@ class Download_Thread(threading.Thread):
                         self.download(self.loadedPyFile)
                     except Exception, e:
                         print "Error:", e #catch up all error here
+                        self.loadedPyFile.status.type = "failed"
                     finally:
                         self.parent.job_finished(self.loadedPyFile)
             sleep(0.5)
@@ -96,7 +73,6 @@ class Download_Thread(threading.Thread):
     def download(self, pyfile):
         status = pyfile.status
         pyfile.prepareDownload()
-    	print "dl prepared", status.filename
 
         if not status.exists:
             raise "FileDontExists" #i know its deprecated, who cares^^
@@ -104,7 +80,7 @@ class Download_Thread(threading.Thread):
     	status.type = "waiting"
 
         while (time() < status.waituntil):
-            if self.parent.init_reconnect(pyfile) or self.parent.reconnecting:
+            if self.parent.init_reconnect() or self.parent.reconnecting:
                 status.type = "reconnected"
             	status.want_reconnect = False
                 return False
diff --git a/module/network/Request.py b/module/network/Request.py
index ce6dc6c8c..8725d0607 100755
--- a/module/network/Request.py
+++ b/module/network/Request.py
@@ -3,15 +3,15 @@
 """
 authored by: RaNaN
 """
-import urllib
-import urllib2
-import cookielib
 import base64
+import cookielib
 import time
+import urllib
+import urllib2
+from gzip import GzipFile
 
 from Keepalive import HTTPHandler
 from cStringIO import StringIO
-from gzip import GzipFile
 
 """
     handles all outgoing HTTP-Requests of the Server
@@ -31,64 +31,64 @@ class Request:
         self.dl_arrived = 0
         self.dl = False
 	
-	self.lastURL = None
-	self.cj = cookielib.CookieJar()
+        self.lastURL = None
+        self.cj = cookielib.CookieJar()
         handler = HTTPHandler()
-	self.opener = urllib2.build_opener(handler, urllib2.HTTPCookieProcessor(self.cj))
-	self.downloader = urllib2.build_opener()
-	#self.opener.add_handler()
+        self.opener = urllib2.build_opener(handler, urllib2.HTTPCookieProcessor(self.cj))
+        self.downloader = urllib2.build_opener()
+        #self.opener.add_handler()
 	
-	self.opener.addheaders = [
-        ("User-Agent","Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.10"),
-        ("Accept-Encoding","gzip,deflate"),
-        ("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
-        ("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7"),
-	("Connection","keep-alive"),
-        ("Keep-Alive","300")]
+        self.opener.addheaders = [
+        ("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.10"),
+        ("Accept-Encoding", "gzip,deflate"),
+        ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
+        ("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7"),
+	("Connection", "keep-alive"),
+        ("Keep-Alive", "300")]
 
-	self.downloader.addheaders = [
-        ("User-Agent","Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.10"),
-        ("Accept-Encoding","gzip,deflate"),
-        ("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
-        ("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7")]
+        self.downloader.addheaders = [
+        ("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.10"),
+        ("Accept-Encoding", "gzip,deflate"),
+        ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
+        ("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7")]
 	
     
-    def load(self, url, get = {}, post = {}, ref = True):
+    def load(self, url, get={}, post={}, ref=True):
 	
-	if post:
-	    post = urllib.urlencode(post)
-	else:
-	   post = None
+        if post:
+            post = urllib.urlencode(post)
+        else:
+          post = None
 	
-	if get:
-	    get = urllib.urlencode(get)
-	else:
-	   get = ""
+        if get:
+          get = urllib.urlencode(get)
+        else:
+          get = ""
 	
-	url = url + get
-        req = urllib2.Request(url, data = post)
-		
-	if ref and self.lastURL is not None:
-	    req.add_header("Referer",self.lastURL)
+        url = url + get
+        req = urllib2.Request(url, data=post)
+
+        if ref and self.lastURL is not None:
+            req.add_header("Referer", self.lastURL)
 	
-	rep = self.opener.open(req)
+        rep = self.opener.open(req)
 	
-	output = rep.read()
+        output = rep.read()
 				
-	if rep.headers.has_key("content-encoding") :
-	    if rep.headers["content-encoding"] == "gzip" :
-		output = GzipFile('','r',0,StringIO(output)).read()
+        if rep.headers.has_key("content-encoding"):
+            if rep.headers["content-encoding"] == "gzip":
+                output = GzipFile('', 'r', 0, StringIO(output)).read()
 	
-	self.lastURL = url
+        self.lastURL = url
 		
-	return output
+        return output
 
     def add_auth(self, user, pw):
-        self.downloader.addheaders.append(['Authorization','Basic ' + base64.encodestring(user + ':' + pw)[:-1]])
+        self.downloader.addheaders.append(['Authorization', 'Basic ' + base64.encodestring(user + ':' + pw)[:-1]])
 
 
     #def download(url, filename, reporthook = None, data = None): #default von urlretrieve auch None?
-     #  return self.downloader.urlretrieve(url, filename, reporthook, data)
+        #  return self.downloader.urlretrieve(url, filename, reporthook, data)
 
     def download(self, url, filename, post={}):
         
@@ -115,7 +115,7 @@ class Request:
      
     def get_speed(self):
         try:
-            return (self.dl_arrived / ((time.time() if self.dl else self.dl_finished)  - self.dl_time )) / 1024
+            return (self.dl_arrived / ((time.time() if self.dl else self.dl_finished)  - self.dl_time)) / 1024
         except:
             return 0
 
@@ -128,6 +128,6 @@ class Request:
     def kB_left(self):
         return (self.dl_size - self.dl_arrived) / 1024
 
-if __name__ == "__main__" :
+if __name__ == "__main__":
     import doctest
     doctest.testmod()
diff --git a/module/thread_list.py b/module/thread_list.py
index a8224ad52..531bc7401 100644
--- a/module/thread_list.py
+++ b/module/thread_list.py
@@ -56,24 +56,20 @@ class Thread_List(object):
     def remove_thread(self, thread):
         self.threads.remove(thread)
     
-#    def status(self):
-#        if not self.status_queue.empty():
-#            while not self.status_queue.empty():
-#                status = self.status_queue.get()
-#                self.py_load_files[status.id].status = status
 
     def get_job(self):
-        # return job if suitable, otherwise send thread idle
-        
+        """return job if suitable, otherwise send thread idle"""
 
-        if self.reconnecting:
+        if not self.parent.is_dltime():
             return None
 
-        self.init_reconnect()
-
         if self.pause:
             return None
+        
+        if self.reconnecting:
+            return None
 
+        self.init_reconnect()
 
         self.lock.acquire()
 
@@ -87,7 +83,7 @@ class Thread_List(object):
             self.py_downloading.append(pyfile)	
             if not pyfile.plugin.multi_dl:
                 self.occ_plugins.append(pyfile.modul.__name__)
-            self.parent.logger.info('start downloading ' + pyfile.url)
+            self.parent.logger.info('Download starts: ' + pyfile.url)
         
         self.lock.release()
         return pyfile
@@ -102,16 +98,34 @@ class Thread_List(object):
         self.py_downloading.remove(pyfile)	
 
         if pyfile.status.type == "finished":
-            self.parent.logger.info('finished downloading ' + pyfile.url + ' @' + str(pyfile.status.get_speed()) + 'kb/s')
+            self.parent.logger.info('Download finished: ' + pyfile.url + ' @' + str(pyfile.status.get_speed()) + 'kb/s')
 
-            #remove from txt
+            with open(self.parent.config['link_file'], 'r') as f:
+                data = f.read()
 
             if pyfile.plugin.props['type'] == "container":
+                links = ""
+
+                for link in pyfile.plugin.links:
+                    links += link+"\n"
+
                 self.parent.extend_links(pyfile.plugin.links)
+                data = links + data # put the links into text file
+
+            data = data.replace(pyfile.url+'\n', "")
+
+            with open(self.parent.config['link_file'], 'w') as f:
+                f.write(data)
 
         if pyfile.status.type == "reconnected":#put it back in queque
             self.py_load_files.insert(0, pyfile)
 
+        if pyfile.status.type == "failed":
+            self.parent.logger.warning("Download failed: " + pyfile.url)
+            with open(self.parent.config['failed_file'], 'a') as f:
+                f.write(pyfile.url+"\n")
+
+
         self.lock.release()
         return True
 
@@ -130,12 +144,15 @@ class Thread_List(object):
         self.f_relation[1] += 1
         self.select_thread()
 
-    def init_reconnect(self, pyfile=None):
+    def init_reconnect(self):
         """initialise a reonnect"""
-	if self.reconnecting:
-	    return False
+        if not self.parent.config['use_reconnect']:
+            return False
+
+        if self.reconnecting:
+            return False
 		
-	self.lock.acquire()
+        self.lock.acquire()
 
         if self.check_reconnect():
 
@@ -143,13 +160,14 @@ class Thread_List(object):
 
             self.reconnect()
 	
-	    time.sleep(1)		
+            time.sleep(1.1)
 
             self.reconnecting = False
-	    self.lock.release()
-	    return True
-	self.lock.release()
-	return False
+            self.lock.release()
+            return True
+        
+        self.lock.release()
+        return False
     
     def check_reconnect(self):
         """checks if all files want reconnect"""