Update Extractor (3)

5 files changed, 128 insertions, 116 deletions

diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py
index dd1a82a82..74dded9b7 100644
--- a/module/plugins/hooks/ExtractArchive.py
+++ b/module/plugins/hooks/ExtractArchive.py
@@ -51,6 +51,7 @@ if os.name != "nt":
 
 from module.plugins.Hook import Hook, threaded, Expose
 from module.plugins.internal.Extractor import ArchiveError, CRCError, PasswordError
+from module.plugins.internal.SimpleHoster import replace_patterns
 from module.utils import fs_encode, save_join, uniqify
 
 
@@ -94,7 +95,7 @@ class ArchiveQueue(object):
         try:
             queue.remove(item)
         except ValueError:
-            pass   
+            pass
         return self.set(queue)
 
 
@@ -102,7 +103,7 @@ class ArchiveQueue(object):
 class ExtractArchive(Hook):
     __name__    = "ExtractArchive"
     __type__    = "hook"
-    __version__ = "1.22"
+    __version__ = "1.23"
 
     __config__ = [("activated"       , "bool"  , "Activated"                                 , True                                                                     ),
                   ("fullpath"        , "bool"  , "Extract with full paths"                   , True                                                                     ),
@@ -127,48 +128,55 @@ class ExtractArchive(Hook):
 
     event_list = ["allDownloadsProcessed"]
 
+    NAME_REPLACEMENTS = [(r'\.part\d+\.rar$', ".part.rar")]
+
 
     #@TODO: Remove in 0.4.10
     def initPeriodical(self):
         pass
 
 
-    def coreReady(self):
-        self.extracting = False
-
-
     def setup(self):
         self.queue  = ArchiveQueue(self, "Queue")
         self.failed = ArchiveQueue(self, "Failed")
 
         self.interval   = 300
+        self.extracting = False
         self.extractors = []
         self.passwords  = []
 
-        names = []
+
+    def coreReady(self):
+        # self.extracting = False
+
         for p in ("UnRar", "SevenZip", "UnZip"):
             try:
                 module = self.core.pluginManager.loadModule("internal", p)
                 klass  = getattr(module, p)
                 if klass.isUsable():
-                    names.append(p)
                     self.extractors.append(klass)
 
             except OSError, e:
                 if e.errno == 2:
                     self.logInfo(_("No %s installed") % p)
                 else:
-                    self.logWarning(_("Could not activate %s") % p, e)
+                    self.logWarning(_("Could not activate: %s") % p, e)
                     if self.core.debug:
                         print_exc()
 
             except Exception, e:
-                self.logWarning(_("Could not activate %s") % p, e)
+                self.logWarning(_("Could not activate: %s") % p, e)
                 if self.core.debug:
                     print_exc()
 
-        if names:
-            self.logInfo(_("Activated") + " " + " ".join(names))
+        if self.extractors:
+            self.logInfo(_("Activated") + " " + " ".join(Extractor.__name__ for Extractor in self.extractors))
+
+            if self.getConfig("waitall"):
+                self.extractPackage(*self.queue.get())  #: Resume unfinished extractions
+            else:
+                super(ExtractArchive, self).initPeriodical()
+
         else:
             self.logInfo(_("No Extract plugins activated"))
 
@@ -185,11 +193,7 @@ class ExtractArchive(Hook):
 
 
     def packageFinished(self, pypack):
-        if self.extracting or self.getConfig("waitall"):
-            self.logInfo(_("Package %s queued for later extracting") % pypack.name)
-            self.queue.add(pypack.id)
-        else:
-            self.extractPackage(pypack.id)
+        self.queue.add(pypack.id)
 
 
     @threaded
@@ -201,6 +205,8 @@ class ExtractArchive(Hook):
 
 
     def extract(self, ids):
+        self.extracting = True
+
         processed = []
         extracted = []
         failed    = []
@@ -267,16 +273,19 @@ class ExtractArchive(Hook):
                     for fname, fid in targets:
                         name = os.path.basename(fname)
 
-                        if fname in processed:
-                            self.logDebug(name, "Skipped")
+                        if not os.path.exists(fname):
+                            self.logDebug(name, "File not found")
                             continue
 
-                        processed.append(fname)  # prevent extracting same file twice
+                        pname = replace_patterns(fname, self.NAME_REPLACEMENTS)
+                        if pname not in processed:
+                            processed.append(pname)  #: prevent extracting same file twice
+                        else:
+                            self.logDebug(name, "Skipped")
+                            continue
 
                         self.logInfo(name, _("Extract to: %s") % out)
                         try:
-                            self.extracting = True
-
                             archive = Extractor(self,
                                                 fname,
                                                 out,
@@ -339,11 +348,9 @@ class ExtractArchive(Hook):
         pyfile = self.core.files.getFile(fid)
         name   = os.path.basename(archive.filename)
 
-        pyfile.setCustomStatus(_("extracting"))
-        pyfile.setProgress(0)
+        pyfile.setStatus("processing")
 
         encrypted = False
-
         try:
             try:
                 archive.check()
@@ -354,8 +361,14 @@ class ExtractArchive(Hook):
 
                 if self.getConfig("repair"):
                     self.logWarning(name, _("Repairing..."))
+
+                    pyfile.setCustomStatus(_("repairing"))
+                    pyfile.setProgress(0)
+
                     repaired = archive.repair()
 
+                    pyfile.setProgress(100)
+
                     if not repaired and not self.getConfig("keepbroken"):
                         raise CRCError("Archive damaged")
 
@@ -368,13 +381,18 @@ class ExtractArchive(Hook):
 
             self.logDebug("Password: %s" % (password or "No provided"))
 
+            pyfile.setCustomStatus(_("extracting"))
+            pyfile.setProgress(0)
+
             if not encrypted or not self.getConfig("usepasswordfile"):
                 archive.extract(password)
             else:
                 for pw in set(self.getPasswords(False) + [password]):
                     try:
                         self.logDebug("Try password: %s" % pw)
-                        if archive.isPassword(pw):
+
+                        ispw = archive.isPassword(pw)
+                        if ispw or ispw is None:
                             archive.extract(pw)
                             self.addPassword(pw)
                             break
@@ -385,7 +403,7 @@ class ExtractArchive(Hook):
                     raise PasswordError
 
             pyfile.setProgress(100)
-            pyfile.setStatus("processing")
+            pyfile.setCustomStatus(_("finalizing"))
 
             if self.core.debug:
                 self.logDebug("Would delete: %s" % ", ".join(archive.getDeleteFiles()))
@@ -402,7 +420,7 @@ class ExtractArchive(Hook):
 
             self.logInfo(name, _("Extracting finished"))
 
-            extracted_files = archive.getExtractedFiles()
+            extracted_files = archive.files or archive.list()
             self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, extracted_files)
 
             return extracted_files
diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py
index 719dc613c..45c13c159 100644
--- a/module/plugins/internal/Extractor.py
+++ b/module/plugins/internal/Extractor.py
@@ -19,7 +19,7 @@ class PasswordError(Exception):
 
 class Extractor:
     __name__    = "Extractor"
-    __version__ = "0.17"
+    __version__ = "0.18"
 
     __description__ = """Base extractor plugin"""
     __license__     = "GPLv3"
@@ -41,7 +41,7 @@ class Extractor:
         """ Check if system statisfy dependencies
         :return: boolean
         """
-        return True
+        return None
 
 
     @classmethod
@@ -99,11 +99,11 @@ class Extractor:
         :param password:
         :return: boolean
         """
-        return True
+        return None
 
 
     def repair(self):
-        return False
+        return None
 
 
     def extract(self, password=None):
@@ -127,6 +127,6 @@ class Extractor:
         return [self.filename]
 
 
-    def getExtractedFiles(self):
+    def list(self, password=None):
         """Populate self.files at some point while extracting"""
         return self.files
diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py
index 126958829..2f4dc5565 100644
--- a/module/plugins/internal/SevenZip.py
+++ b/module/plugins/internal/SevenZip.py
@@ -11,7 +11,7 @@ from module.utils import fs_encode, save_join
 
 class SevenZip(UnRar):
     __name__    = "SevenZip"
-    __version__ = "0.05"
+    __version__ = "0.06"
 
     __description__ = """7-Zip extractor plugin"""
     __license__     = "GPLv3"
@@ -89,40 +89,24 @@ class SevenZip(UnRar):
 
         renice(p.pid, self.renice)
 
-        progressstring = ""
-        while True:
-            c = p.stdout.read(1)
-            # quit loop on eof
-            if not c:
-                break
-            # reading a percentage sign -> set progress and restart
-            if c == '%':
-                self.notifyProgress(int(progressstring))
-                progressstring = ""
-            # not reading a digit -> therefore restart
-            elif c not in digits:
-                progressstring = ""
-            # add digit to progressstring
-            else:
-                progressstring += c
-
-        # retrieve stderr
-        err = p.stderr.read()
-
-        if self.re_wrongpwd.search(err):
-            raise PasswordError
+        # communicate and retrieve stderr
+        self._progress(p)
+        err = p.stderr.read().strip()
 
-        elif self.re_wrongcrc.search(err):
-            raise CRCError(err)
+        if err:
+            if self.re_wrongpwd.search(err):
+                raise PasswordError
+
+            elif self.re_wrongcrc.search(err):
+                raise CRCError(err)
 
-        elif err.strip():  #: raise error if anything is on stderr
-            raise ArchiveError(err)
+            else:  #: raise error if anything is on stderr
+                raise ArchiveError(err)
 
         if p.returncode > 1:
             raise ArchiveError(_("Process return code: %d") % p.returncode)
 
-        if not self.files:
-            self.files = self.list(password)
+        self.files = self.list(password)
 
 
     def list(self, password=None):
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
index 1b6816a9f..d378bf167 100644
--- a/module/plugins/internal/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -22,7 +22,7 @@ def renice(pid, value):
 
 class UnRar(Extractor):
     __name__    = "UnRar"
-    __version__ = "1.07"
+    __version__ = "1.08"
 
     __description__ = """Rar extractor plugin"""
     __license__     = "GPLv3"
@@ -39,7 +39,9 @@ class UnRar(Extractor):
     re_rarpart1 = re.compile(r'\.part(\d+)\.rar$', re.I)
     re_rarpart2 = re.compile(r'\.r(\d+)$', re.I)
 
+    re_filefixed = re.compile(r'Building (.+)')
     re_filelist  = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)')
+
     re_wrongpwd  = re.compile(r'password', re.I)
     re_wrongcrc  = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I)
 
@@ -63,21 +65,6 @@ class UnRar(Extractor):
         return True
 
 
-    @classmethod
-    def getTargets(cls, files_ids):
-        targets = []
-
-        for fname, id in files_ids:
-            if not cls.isArchive(fname):
-                continue
-
-            m = cls.re_rarpart1.search(fname)
-            if not m or int(m.group(1)) == 1:  #@NOTE: only add first part file
-                targets.append((fname, id))
-
-        return targets
-
-
     def check(self):
         p = self.call_cmd("l", "-v", fs_encode(self.filename))
         out, err = p.communicate()
@@ -103,67 +90,73 @@ class UnRar(Extractor):
 
     def repair(self):
         p = self.call_cmd("rc", fs_encode(self.filename))
-        out, err = p.communicate()
 
-        if p.returncode or err.strip():
+        # communicate and retrieve stderr
+        self._progress(p)
+        err = p.stderr.read().strip()
+
+        if err or p.returncode:
             p = self.call_cmd("r", fs_encode(self.filename))
-            out, err = p.communicate()
 
-            if p.returncode or err.strip():
+            # communicate and retrieve stderr
+            self._progress(p)
+            err = p.stderr.read().strip()
+
+            if err or p.returncode:
                 return False
             else:
-                dir, name = os.path.split(filename)
-
-                if 'fixed' in out:
-                    self.filename = os.path.join(dir, 'fixed.' + name)
+                dir  = os.path.dirname(filename)
+                name = re_filefixed.search(out).group(1)
 
-                elif 'rebuild' in out:
-                    self.filename = os.path.join(dir, 'rebuild.' + name)
+                self.filename = os.path.join(dir, name)
 
         return True
 
 
-    def extract(self, password=None):
-        command = "x" if self.fullpath else "e"
-
-        p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password)
-
-        renice(p.pid, self.renice)
-
-        progressstring = ""
+    def _progress(self, process):
+        s = ""
         while True:
-            c = p.stdout.read(1)
+            c = process.stdout.read(1)
             # quit loop on eof
             if not c:
                 break
             # reading a percentage sign -> set progress and restart
             if c == '%':
-                self.notifyProgress(int(progressstring))
-                progressstring = ""
+                self.notifyProgress(int(s))
+                s = ""
             # not reading a digit -> therefore restart
             elif c not in digits:
-                progressstring = ""
+                s = ""
             # add digit to progressstring
             else:
-                progressstring += c
+                s += c
 
-        # retrieve stderr
-        err = p.stderr.read()
 
-        if self.re_wrongpwd.search(err):
-            raise PasswordError
+    def extract(self, password=None):
+        command = "x" if self.fullpath else "e"
 
-        elif self.re_wrongcrc.search(err):
-            raise CRCError(err)
+        p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password)
+
+        renice(p.pid, self.renice)
+
+        # communicate and retrieve stderr
+        self._progress(p)
+        err = p.stderr.read().strip()
+
+        if err:
+            if self.re_wrongpwd.search(err):
+                raise PasswordError
+
+            elif self.re_wrongcrc.search(err):
+                raise CRCError(err)
 
-        elif err.strip():  #: raise error if anything is on stderr
-            raise ArchiveError(err)
+            else:  #: raise error if anything is on stderr
+                raise ArchiveError(err)
 
         if p.returncode:
             raise ArchiveError(_("Process return code: %d") % p.returncode)
 
-        if not self.files:
-            self.files = self.list(password)
+        self.files = self.list(password)
 
 
     def getDeleteFiles(self):
diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py
index eb8259e47..781c47d7b 100644
--- a/module/plugins/internal/UnZip.py
+++ b/module/plugins/internal/UnZip.py
@@ -4,15 +4,15 @@ from __future__ import with_statement
 
 import os
 import sys
+import zipfile
 
-from zipfile import ZipFile, BadZipfile, LargeZipFile
 from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError
 from module.utils import fs_encode
 
 
 class UnZip(Extractor):
     __name__    = "UnZip"
-    __version__ = "1.07"
+    __version__ = "1.08"
 
     __description__ = """Zip extractor plugin"""
     __license__     = "GPLv3"
@@ -27,20 +27,35 @@ class UnZip(Extractor):
         return sys.version_info[:2] >= (2, 6)
 
 
+    def list(self, password=None):
+        with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z:
+            z.setpassword(password)
+            return z.namelist()
+
+
+    def check(self):
+        with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z:
+            badfile = z.testzip()
+
+            if badfile:
+                raise CRCError(badfile)
+            else:
+                raise PasswordError
+
+
     def extract(self, password=None):
         try:
-            with ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z:
+            with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z:
                 z.setpassword(password)
 
                 badfile = z.testzip()
 
                 if not badfile:
                     z.extractall(self.out)
-                    self.files = z.namelist()
                 else:
                     raise CRCError(badfile)
 
-        except (BadZipfile, LargeZipFile), e:
+        except (zipfile.BadZipfile, zipfile.LargeZipFile), e:
             raise ArchiveError(e)
 
         except RuntimeError, e:
@@ -48,3 +63,5 @@ class UnZip(Extractor):
                 raise PasswordError
             else:
                 raise ArchiveError(e)
+        else:
+            self.files = z.namelist()