show progress of decrypting and link checking, added indicator in link grabber

5 files changed, 60 insertions, 10 deletions

diff --git a/pyload/threads/AddonThread.py b/pyload/threads/AddonThread.py
index d8d84cbfa..c82045803 100644
--- a/pyload/threads/AddonThread.py
+++ b/pyload/threads/AddonThread.py
@@ -21,7 +21,7 @@ class AddonThread(BaseThread):
         self.active = []
         self.progress = 0
 
-        m.localThreads.append(self)
+        m.addThread(self)
 
         self.start()
 
@@ -76,4 +76,4 @@ class AddonThread(BaseThread):
             for x in local:
                 self.finishFile(x)
 
-            self.m.localThreads.remove(self)
\ No newline at end of file
+            self.finished()
\ No newline at end of file
diff --git a/pyload/threads/BaseThread.py b/pyload/threads/BaseThread.py
index 9b4e5af67..a370bd661 100644
--- a/pyload/threads/BaseThread.py
+++ b/pyload/threads/BaseThread.py
@@ -30,6 +30,10 @@ class BaseThread(Thread):
     def user(self):
         return primary_uid(self.owner)
 
+    def finished(self):
+        """ Remove thread from list  """
+        self.m.removeThread(self)
+
     def getProgress(self):
         """ retrieves progress information about the current running task
 
diff --git a/pyload/threads/DecrypterThread.py b/pyload/threads/DecrypterThread.py
index a0bebdfbf..c7a01683a 100644
--- a/pyload/threads/DecrypterThread.py
+++ b/pyload/threads/DecrypterThread.py
@@ -3,13 +3,14 @@
 
 from time import sleep
 
-from pyload.Api import LinkStatus, DownloadStatus as DS
+from pyload.Api import LinkStatus, DownloadStatus as DS, ProgressInfo, ProgressType
 from pyload.utils import uniqify, accumulate
 from pyload.plugins.Base import Abort, Retry
 from pyload.plugins.Crypter import Package
 
 from BaseThread import BaseThread
 
+
 class DecrypterThread(BaseThread):
     """thread for decrypting"""
 
@@ -19,15 +20,22 @@ class DecrypterThread(BaseThread):
         # [... (plugin, url) ...]
         self.data = data
         self.pid = pid
+        # holds the progress, while running
+        self.progress = None
 
+        self.m.addThread(self)
         self.start()
 
+    def getProgress(self):
+        return self.progress
+
     def run(self):
         pack = self.m.core.files.getPackage(self.pid)
         links, packages = self.decrypt(accumulate(self.data), pack.password)
 
         if links:
-            self.log.info(_("Decrypted %(count)d links into package %(name)s") % {"count": len(links), "name": pack.name})
+            self.log.info(
+                _("Decrypted %(count)d links into package %(name)s") % {"count": len(links), "name": pack.name})
             self.m.core.api.addFiles(self.pid, [l.url for l in links])
 
         # TODO: add single package into this one and rename it?
@@ -35,17 +43,25 @@ class DecrypterThread(BaseThread):
         for p in packages:
             self.m.core.api.addPackage(p.name, p.getURLs(), pack.password)
 
+        self.finished()
+
     def decrypt(self, plugin_map, password=None, err=False):
         result = []
 
+        self.progress = ProgressInfo("BasePlugin", "",  _("decrypting"),
+                                         0, 0, len(self.data), self.owner, ProgressType.Decrypting)
         # TODO QUEUE_DECRYPT
-
         for name, urls in plugin_map.iteritems():
             klass = self.m.core.pluginManager.loadClass("crypter", name)
             plugin = None
             plugin_result = []
 
+            # updating progress
+            self.progress.plugin = name
+            self.progress.name = _("Decrypting %s links") % len(urls) if len(urls) > 1 else urls[0]
+
             #TODO: dependency check, there is a new error code for this
+            # TODO: decrypting with result yielding
             if not klass:
                 plugin_result.extend(LinkStatus(url, url, -1, DS.NotPossible, name) for url in urls)
                 self.log.debug("Plugin for decrypting was not loaded")
@@ -77,8 +93,12 @@ class DecrypterThread(BaseThread):
                     if plugin:
                         plugin.clean()
 
+            self.progress.done += len(urls)
             result.extend(plugin_result)
 
+        # clear the progress
+        self.progress = None
+
         # generated packages
         packs = {}
         # urls without package
diff --git a/pyload/threads/InfoThread.py b/pyload/threads/InfoThread.py
index 8d33df705..a91ca679e 100644
--- a/pyload/threads/InfoThread.py
+++ b/pyload/threads/InfoThread.py
@@ -3,7 +3,7 @@
 
 from time import time
 
-from pyload.Api import LinkStatus, DownloadStatus
+from pyload.Api import LinkStatus, DownloadStatus, ProgressInfo, ProgressType
 from pyload.utils.packagetools import parseNames
 from pyload.utils import has_method, accumulate
 
@@ -22,6 +22,7 @@ class InfoThread(DecrypterThread):
         # urls that already have a package name
         self.names = {}
 
+        self.m.addThread(self)
         self.start()
 
     def run(self):
@@ -54,6 +55,9 @@ class InfoThread(DecrypterThread):
         # db or info result
         cb = self.updateDB if self.pid > 1 else self.updateResult
 
+        self.progress = ProgressInfo("BasePlugin", "", _("online check"), 0, 0, sum(len(urls) for urls in plugins.itervalues()),
+                                     self.owner, ProgressType.LinkCheck)
+
         for pluginname, urls in plugins.iteritems():
             plugin = self.m.core.pluginManager.loadModule("hoster", pluginname)
             klass = self.m.core.pluginManager.getPluginClass("hoster", pluginname, overwrite=False)
@@ -69,6 +73,8 @@ class InfoThread(DecrypterThread):
 
         self.names.clear()
         self.m.timestamp = time() + 5 * 60
+        self.progress = None
+        self.finished()
 
     def updateDB(self, result):
         # writes results to db
@@ -100,6 +106,12 @@ class InfoThread(DecrypterThread):
         """executes info fetching for given plugin and urls"""
         # also works on module names
         pluginname = plugin.__name__.split(".")[-1]
+
+        self.progress.plugin = pluginname
+        self.progress.name = _("Checking %d links") % len(urls)
+
+        # final number of links to be checked
+        done = self.progress.done + len(urls)
         try:
             cached = [] #results loaded from cache
             process = [] #urls to process
@@ -111,6 +123,7 @@ class InfoThread(DecrypterThread):
 
             if cached:
                 self.m.log.debug("Fetched %d links from cache for %s" % (len(cached), pluginname))
+                self.progress.done += len(cached)
                 cb(cached)
 
             if process:
@@ -135,10 +148,13 @@ class InfoThread(DecrypterThread):
                     for link in links:
                         self.m.infoCache[link.url] = link
 
+                    self.progress.done += len(links)
                     cb(links)
 
             self.m.log.debug("Finished Info Fetching for %s" % pluginname)
         except Exception, e:
             self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") %
                                {"name": pluginname, "err": str(e)})
-            self.core.print_exc()
\ No newline at end of file
+            self.core.print_exc()
+        finally:
+            self.progress.done = done
\ No newline at end of file
diff --git a/pyload/threads/ThreadManager.py b/pyload/threads/ThreadManager.py
index 3132e98e2..0e0b6320d 100644
--- a/pyload/threads/ThreadManager.py
+++ b/pyload/threads/ThreadManager.py
@@ -19,7 +19,7 @@
 from os.path import exists, join
 import re
 from subprocess import Popen
-from threading import Event, Lock
+from threading import Event, RLock
 from time import sleep, time
 from traceback import print_exc
 from random import choice
@@ -53,7 +53,7 @@ class ThreadManager:
         self.reconnecting.clear()
         self.downloaded = 0 #number of files downloaded since last cleanup
 
-        self.lock = Lock()
+        self.lock = RLock()
 
         # some operations require to fetch url info from hoster, so we caching them so it wont be done twice
         # contains a timestamp and will be purged after timeout
@@ -71,13 +71,23 @@ class ThreadManager:
         for i in range(self.core.config.get("download", "max_downloads")):
             self.createThread()
 
-
     def createThread(self):
         """create a download thread"""
 
         thread = DownloadThread(self)
         self.threads.append(thread)
 
+    @lock
+    def addThread(self, thread):
+        self.localThreads.append(thread)
+
+    @lock
+    def removeThread(self, thread):
+        """ Remove a thread from the local list """
+        if thread in self.localThreads:
+            self.localThreads.remove(thread)
+
+    @lock
     def createInfoThread(self, data, pid):
         """ start a thread which fetches online status and other info's """
         self.timestamp = time() + 5 * 60