Extend ExtractArchive events + code optimizations

https://github.com/pyload/pyload/pull/622
https://github.com/pyload/pyload/pull/726

3 files changed, 148 insertions, 76 deletions

diff --git a/module/plugins/hooks/ExternalScripts.py b/module/plugins/hooks/ExternalScripts.py
index 8d03d27d4..bdde9cd62 100644
--- a/module/plugins/hooks/ExternalScripts.py
+++ b/module/plugins/hooks/ExternalScripts.py
@@ -2,6 +2,7 @@
 
 import subprocess
 
+from itertools import chain
 from os import listdir, access, X_OK, makedirs
 from os.path import join, exists, basename, abspath
 
@@ -12,23 +13,27 @@ from module.utils import save_join
 class ExternalScripts(Hook):
     __name__ = "ExternalScripts"
     __type__ = "hook"
-    __version__ = "0.23"
+    __version__ = "0.24"
 
     __config__ = [("activated", "bool", "Activated", True)]
 
     __description__ = """Run external scripts"""
-    __author_name__ = ("mkaay", "RaNaN", "spoob")
-    __author_mail__ = ("mkaay@mkaay.de", "ranan@pyload.org", "spoob@pyload.org")
+    __author_name__ = ("mkaay", "RaNaN", "spoob", "Walter Purcaro")
+    __author_mail__ = ("mkaay@mkaay.de", "ranan@pyload.org", "spoob@pyload.org", "vuolter@gmail.com")
 
-    event_list = ["unrarFinished", "allDownloadsFinished", "allDownloadsProcessed"]
+    event_list = ["archive_extracted", "package_extracted", "all_archives_extracted", "all_archives_processed",
+                  "allDownloadsFinished", "allDownloadsProcessed"]
 
 
     def setup(self):
         self.scripts = {}
 
-        folders = ["download_preparing", "download_finished", "package_finished",
-                   "before_reconnect", "after_reconnect", "unrar_finished",
-                   "all_dls_finished", "all_dls_processed"]
+        folders = ["download_preparing", "download_finished", "all_downloads_finished", "all_downloads_processed",
+                   "before_reconnect", "after_reconnect",
+                   "package_finished", "package_extracted",
+                   "archive_extracted", "all_archives_extracted", "all_archives_processed",
+                   # deprecated folders
+                   "unrar_finished", "all_dls_finished", "all_dls_processed"]
 
         for folder in folders:
             self.scripts[folder] = []
@@ -40,6 +45,7 @@ class ExternalScripts(Hook):
             if names:
                 self.logInfo((_("Installed scripts for %s: ") % script_type) + ", ".join([basename(x) for x in names]))
 
+
     def initPluginType(self, folder, path):
         if not exists(path):
             try:
@@ -57,6 +63,7 @@ class ExternalScripts(Hook):
 
             self.scripts[folder].append(join(path, f))
 
+
     def callScript(self, script, *args):
         try:
             cmd = [script] + [str(x) if not isinstance(x, basestring) else x for x in args]
@@ -66,39 +73,65 @@ class ExternalScripts(Hook):
         except Exception, e:
             self.logError(_("Error in %(script)s: %(error)s") % {"script": basename(script), "error": str(e)})
 
+
     def downloadPreparing(self, pyfile):
         for script in self.scripts['download_preparing']:
             self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.id)
 
+
     def downloadFinished(self, pyfile):
+        download_folder = self.config['general']['download_folder']
         for script in self.scripts['download_finished']:
-            self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.name,
-                            save_join(self.config['general']['download_folder'],
-                                      pyfile.package().folder, pyfile.name), pyfile.id)
+            filename = save_join(download_folder, pyfile.package().folder, pyfile.name)
+            self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.name, filename, pyfile.id)
+
 
     def packageFinished(self, pypack):
+        download_folder = self.config['general']['download_folder']
         for script in self.scripts['package_finished']:
-            folder = self.config['general']['download_folder']
-            folder = save_join(folder, pypack.folder)
-
+            folder = save_join(download_folder, pypack.folder)
             self.callScript(script, pypack.name, folder, pypack.password, pypack.id)
 
+
     def beforeReconnecting(self, ip):
         for script in self.scripts['before_reconnect']:
             self.callScript(script, ip)
 
+
     def afterReconnecting(self, ip):
         for script in self.scripts['after_reconnect']:
             self.callScript(script, ip)
 
-    def unrarFinished(self, folder, fname):
-        for script in self.scripts['unrar_finished']:
-            self.callScript(script, folder, fname)
+
+    def archive_extracted(self, pyfile, folder, filename, files):
+        for script in self.scripts['archive_extracted']:
+            self.callScript(script, folder, filename, files)
+        for script in self.scripts['unrar_finished']:  #: deprecated
+            self.callScript(script, folder, filename)
+
+
+    def package_extracted(self, pypack):
+        download_folder = self.config['general']['download_folder']
+        for script in self.scripts['package_extracted']:
+            folder = save_join(download_folder, pypack.folder)
+            self.callScript(script, pypack.name, folder, pypack.password, pypack.id)
+
+
+    def all_archives_extracted(self):
+        for script in self.scripts['all_archives_extracted']:
+            self.callScript(script)
+
+
+    def all_archives_processed(self):
+        for script in self.scripts['all_archives_processed']:
+            self.callScript(script)
+
 
     def allDownloadsFinished(self):
-        for script in self.scripts['all_dls_finished']:
+        for script in chain(self.scripts['all_downloads_finished'], self.scripts['all_dls_finished']):
             self.callScript(script)
 
+
     def allDownloadsProcessed(self):
-        for script in self.scripts['all_dls_processed']:
+        for script in chain(self.scripts['all_downloads_processed'], self.scripts['all_dls_processed']):
             self.callScript(script)
diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py
index 144829459..84f6eab25 100644
--- a/module/plugins/hooks/ExtractArchive.py
+++ b/module/plugins/hooks/ExtractArchive.py
@@ -53,17 +53,14 @@ from module.utils import save_join, fs_encode
 
 
 class ExtractArchive(Hook):
-    """
-    Provides: unrarFinished (folder, filename)
-    """
     __name__ = "ExtractArchive"
     __type__ = "hook"
-    __version__ = "0.16"
+    __version__ = "0.17"
 
     __config__ = [("activated", "bool", "Activated", True),
                   ("fullpath", "bool", "Extract full path", True),
                   ("overwrite", "bool", "Overwrite files", True),
-                  ("passwordfile", "file", "password file", "unrar_passwords.txt"),
+                  ("passwordfile", "file", "password file", "archive_password.txt"),
                   ("deletearchive", "bool", "Delete archives when done", False),
                   ("subfolder", "bool", "Create subfolder for each package", False),
                   ("destination", "folder", "Extract files to", ""),
@@ -73,8 +70,8 @@ class ExtractArchive(Hook):
                   ("renice", "int", "CPU Priority", 0)]
 
     __description__ = """Extract different kind of archives"""
-    __author_name__ = ("pyLoad Team", "AndroKev")
-    __author_mail__ = ("admin@pyload.org", "@pyloadforum")
+    __author_name__ = ("RaNaN", "AndroKev", "Walter Purcaro")
+    __author_mail__ = ("ranan@pyload.org", "@pyloadforum", "vuolter@gmail.com")
 
     event_list = ["allDownloadsProcessed"]
 
@@ -113,33 +110,50 @@ class ExtractArchive(Hook):
         # queue with package ids
         self.queue = []
 
+
     @Expose
     def extractPackage(self, id):
         """ Extract package with given id"""
         self.manager.startThread(self.extract, [id])
 
+
     def packageFinished(self, pypack):
+        pid = pypack.id
         if self.getConfig("queue"):
             self.logInfo(_("Package %s queued for later extracting") % pypack.name)
-            self.queue.append(pypack.id)
+            self.queue.append(pid)
         else:
-            self.manager.startThread(self.extract, [pypack.id])
+            self.manager.startThread(self.extract, [pid])
+
 
     @threaded
     def allDownloadsProcessed(self, thread):
         local = copy(self.queue)
         del self.queue[:]
-        self.extract(local, thread)
+        if self.extract(local, thread):  #: check only if all gone fine, no failed reporting for now
+            self.manager.dispatchEvent("all_archives_extracted")
+        self.manager.dispatchEvent("all_archives_processed")
+
 
     def extract(self, ids, thread=None):
+        processed = []
+        extracted = []
+        failed = []
+
+        destination = self.getConfig("destination")
+        subfolder = self.getConfig("subfolder")
+        fullpath = self.getConfig("fullpath")
+        overwrite = self.getConfig("overwrite")
+        excludefiles = self.getConfig("excludefiles")
+        renice = self.getConfig("renice")
+        recursive = self.getConfig("recursive")
+
         # reload from txt file
         self.reloadPasswords()
 
         # dl folder
         dl = self.config['general']['download_folder']
 
-        extracted = []
-
         #iterate packages -> plugins -> targets
         for pid in ids:
             p = self.core.files.getPackage(pid)
@@ -148,22 +162,17 @@ class ExtractArchive(Hook):
                 continue
 
             # determine output folder
-            out = save_join(dl, p.folder, "")
-            # force trailing slash
-
-            if self.getConfig("destination") and self.getConfig("destination").lower() != "none":
-
-                out = save_join(dl, p.folder, self.getConfig("destination"), "")
-                #relative to package folder if destination is relative, otherwise absolute path overwrites them
+            out = save_join(dl, p.folder, destination, "")  #: force trailing slash
 
-                if self.getConfig("subfolder"):
-                    out = save_join(out, fs_encode(p.folder))
+            if subfolder:
+                out = save_join(out, fs_encode(p.folder))
 
-                if not exists(out):
-                    makedirs(out)
+            if not exists(out):
+                makedirs(out)
 
             files_ids = [(save_join(dl, p.folder, x['name']), x['id']) for x in p.getChildren().itervalues()]
             matched = False
+            success = True
 
             # check as long there are unseen files
             while files_ids:
@@ -175,17 +184,23 @@ class ExtractArchive(Hook):
                         self.logDebug("Targets for %s: %s" % (plugin.__name__, targets))
                         matched = True
                     for target, fid in targets:
-                        if target in extracted:
+                        if target in processed:
                             self.logDebug(basename(target), "skipped")
                             continue
-                        extracted.append(target)  # prevent extracting same file twice
 
-                        klass = plugin(self, target, out, self.getConfig("fullpath"), self.getConfig("overwrite"), self.getConfig("excludefiles"),
-                                       self.getConfig("renice"))
-                        klass.init()
+                        processed.append(target)  # prevent extracting same file twice
 
                         self.logInfo(basename(target), _("Extract to %s") % out)
-                        new_files = self.startExtracting(klass, fid, p.password.strip().splitlines(), thread)
+                        try:
+                            klass = plugin(self, target, out, fullpath, overwrite, excludefiles, renice)
+                            klass.init()
+                            password = p.password.strip().splitlines()
+                            new_files = self._extract(klass, fid, password, thread)
+                        except Exception, e:
+                            self.logError(basename(target), str(e))
+                            success = False
+                            continue
+
                         self.logDebug("Extracted: %s" % new_files)
                         self.setPermissions(new_files)
 
@@ -193,18 +208,27 @@ class ExtractArchive(Hook):
                             if not exists(file):
                                 self.logDebug("new file %s does not exists" % file)
                                 continue
-                            if self.getConfig("recursive") and isfile(file):
+                            if recursive and isfile(file):
                                 new_files_ids.append((file, fid))  # append as new target
 
                 files_ids = new_files_ids  # also check extracted files
 
-            if not matched:
+            if matched:
+                if success:
+                    extracted.append(pid)
+                    self.manager.dispatchEvent("package_extracted", p)
+                else:
+                    failed.append(pid)
+                    self.manager.dispatchEvent("package_extract_failed", p)
+            else:
                 self.logInfo(_("No files found to extract"))
 
-    def startExtracting(self, plugin, fid, passwords, thread):
+        return True if not failed else False
+
+
+    def _extract(self, plugin, fid, passwords, thread):
         pyfile = self.core.files.getFile(fid)
-        if not pyfile:
-            return []
+        deletearchive = self.getConfig("deletearchive")
 
         pyfile.setCustomStatus(_("extracting"))
         thread.addActive(pyfile)  # keep this file until everything is done
@@ -221,7 +245,7 @@ class ExtractArchive(Hook):
                 self.logDebug("Passwords: %s" % str(passwords))
 
                 pwlist = copy(self.getPasswords())
-                #remove already supplied pws from list (only local)
+                # remove already supplied pws from list (only local)
                 for pw in passwords:
                     if pw in pwlist:
                         pwlist.remove(pw)
@@ -238,13 +262,12 @@ class ExtractArchive(Hook):
                         self.logDebug("Password was wrong")
 
             if not success:
-                self.logError(basename(plugin.file), _("Wrong password"))
-                return []
+                raise Exception(_("Wrong password"))
 
             if self.core.debug:
                 self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles()))
 
-            if self.getConfig("deletearchive"):
+            if deletearchive:
                 files = plugin.getDeleteFiles()
                 self.logInfo(_("Deleting %s files") % len(files))
                 for f in files:
@@ -254,9 +277,11 @@ class ExtractArchive(Hook):
                         self.logDebug("%s does not exists" % f)
 
             self.logInfo(basename(plugin.file), _("Extracting finished"))
-            self.manager.dispatchEvent("unrarFinished", plugin.out, plugin.file)
 
-            return plugin.getExtractedFiles()
+            extracted_files = plugin.getExtractedFiles()
+            self.manager.dispatchEvent("archive_extracted", pyfile, plugin.out, plugin.file, extracted_files)
+
+            return extracted_files
 
         except ArchiveError, e:
             self.logError(basename(plugin.file), _("Archive Error"), str(e))
@@ -267,40 +292,45 @@ class ExtractArchive(Hook):
                 print_exc()
             self.logError(basename(plugin.file), _("Unknown Error"), str(e))
 
-        return []
+        self.manager.dispatchEvent("archive_extract_failed", pyfile)
+        raise Exception(_("Extract failed"))
+
 
     @Expose
     def getPasswords(self):
         """ List of saved passwords """
         return self.passwords
 
+
     def reloadPasswords(self):
-        pwfile = self.getConfig("passwordfile")
-        if not exists(pwfile):
-            open(pwfile, "wb").close()
+        passwordfile = self.getConfig("passwordfile")
+        if not exists(passwordfile):
+            open(passwordfile, "wb").close()
 
         passwords = []
-        f = open(pwfile, "rb")
+        f = open(passwordfile, "rb")
         for pw in f.read().splitlines():
             passwords.append(pw)
         f.close()
 
         self.passwords = passwords
 
+
     @Expose
     def addPassword(self, pw):
         """  Adds a password to saved list"""
-        pwfile = self.getConfig("passwordfile")
+        passwordfile = self.getConfig("passwordfile")
 
         if pw in self.passwords:
             self.passwords.remove(pw)
         self.passwords.insert(0, pw)
 
-        f = open(pwfile, "wb")
+        f = open(passwordfile, "wb")
         for pw in self.passwords:
             f.write(pw + "\n")
         f.close()
 
+
     def setPermissions(self, files):
         for f in files:
             if not exists(f):
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
index 19c278735..91be69433 100644
--- a/module/plugins/internal/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -4,7 +4,7 @@ import os
 import re
 
 from glob import glob
-from os.path import join
+from os import basename, join
 from string import digits
 from subprocess import Popen, PIPE
 
@@ -12,14 +12,23 @@ from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPas
 from module.utils import save_join, decode
 
 
+def renice(pid, value):
+    if os.name != "nt" and value:
+        try:
+            Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1)
+        except:
+            print "Renice failed"
+
+
 class UnRar(AbtractExtractor):
     __name__ = "UnRar"
-    __version__ = "0.16"
+    __version__ = "0.17"
 
     __description__ = """Rar extractor plugin"""
     __author_name__ = "RaNaN"
     __author_mail__ = "RaNaN@pyload.org"
 
+
     CMD = "unrar"
 
     # there are some more uncovered rar formats
@@ -50,6 +59,7 @@ class UnRar(AbtractExtractor):
 
         return True
 
+
     @staticmethod
     def getTargets(files_ids):
         result = []
@@ -68,12 +78,14 @@ class UnRar(AbtractExtractor):
 
         return result
 
+
     def init(self):
         self.passwordProtected = False
         self.headerProtected = False  #: list files will not work without password
         self.smallestFile = None  #: small file to test passwords
         self.password = ""  #: save the correct password
 
+
     def checkArchive(self):
         p = self.call_unrar("l", "-v", self.file)
         out, err = p.communicate()
@@ -100,6 +112,7 @@ class UnRar(AbtractExtractor):
 
         return False
 
+
     def checkPassword(self, password):
         # at this point we can only verify header protected files
         if self.headerProtected:
@@ -110,6 +123,7 @@ class UnRar(AbtractExtractor):
 
         return True
 
+
     def extract(self, progress, password=None):
         command = "x" if self.fullpath else "e"
 
@@ -151,13 +165,15 @@ class UnRar(AbtractExtractor):
             self.password = password
             self.listContent()
 
+
     def getDeleteFiles(self):
-        if ".part" in self.file:
+        if ".part" in basename(self.file):
             return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.IGNORECASE))
         # get files which matches .r* and filter unsuited files out
         parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.IGNORECASE))
         return filter(lambda x: self.re_partfiles.match(x), parts)
 
+
     def listContent(self):
         command = "vb" if self.fullpath else "lb"
         p = self.call_unrar(command, "-v", self.file, password=self.password)
@@ -177,6 +193,7 @@ class UnRar(AbtractExtractor):
 
         self.files = result
 
+
     def call_unrar(self, command, *xargs, **kwargs):
         args = []
         # overwrite flag
@@ -202,11 +219,3 @@ class UnRar(AbtractExtractor):
         p = Popen(call, stdout=PIPE, stderr=PIPE)
 
         return p
-
-
-def renice(pid, value):
-    if os.name != "nt" and value:
-        try:
-            Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1)
-        except:
-            print "Renice failed"