Merge pull request #1194 from immenz/dev_extract

[ExtractArchive] Some more changes

5 files changed, 170 insertions, 138 deletions

diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py
index 3ea8839dc..a1e85ba57 100644
--- a/module/plugins/hooks/ExtractArchive.py
+++ b/module/plugins/hooks/ExtractArchive.py
@@ -104,13 +104,14 @@ class ArchiveQueue(object):
 class ExtractArchive(Hook):
     __name__    = "ExtractArchive"
     __type__    = "hook"
-    __version__ = "1.29"
+    __version__ = "1.30"
 
     __config__ = [("activated"       , "bool"  , "Activated"                                 , True                                                                     ),
                   ("fullpath"        , "bool"  , "Extract with full paths"                   , True                                                                     ),
                   ("overwrite"       , "bool"  , "Overwrite files"                           , False                                                                    ),
                   ("keepbroken"      , "bool"  , "Try to extract broken archives"            , False                                                                    ),
-                  ("repair"          , "bool"  , "Repair broken archives"                    , True                                                                     ),
+                  ("repair"          , "bool"  , "Repair broken archives (rar required)"     , False                                                                    ),
+                  ("test"            , "bool"  , "Test archive before extracting"            , False                                                                    ),
                   ("usepasswordfile" , "bool"  , "Use password file"                         , True                                                                     ),
                   ("passwordfile"    , "file"  , "Password file"                             , "archive_password.txt"                                                   ),
                   ("delete"          , "bool"  , "Delete archive when successfully extracted", False                                                                    ),
@@ -128,35 +129,32 @@ class ExtractArchive(Hook):
                        ("Immenz"        , "immenz@gmx.net"   )]
 
 
-    event_list = ["allDownloadsProcessed"]
+    event_list = ["allDownloadsProcessed","packageDeleted"]
 
     NAME_REPLACEMENTS = [(r'\.part\d+\.rar$', ".part.rar")]
 
 
-    #@TODO: Remove in 0.4.10
-    def initPeriodical(self):
-        pass
-
-
     def setup(self):
         self.queue  = ArchiveQueue(self, "Queue")
         self.failed = ArchiveQueue(self, "Failed")
 
-        self.interval   = 60
-        self.extracting = False
-        self.extractors = []
-        self.passwords  = []
+        self.interval    = 60
+        self.extracting  = False
+        self.lastPackage = False
+        self.extractors  = []
+        self.passwords   = []
+        self.repair      = False
 
 
     def coreReady(self):
-        # self.extracting = False
-
         for p in ("UnRar", "SevenZip", "UnZip"):
             try:
                 module = self.core.pluginManager.loadModule("internal", p)
                 klass  = getattr(module, p)
                 if klass.isUsable():
                     self.extractors.append(klass)
+                if klass.REPAIR:
+                    self.repair = self.getConfig("repair")
 
             except OSError, e:
                 if e.errno == 2:
@@ -173,37 +171,49 @@ class ExtractArchive(Hook):
 
         if self.extractors:
             self.logInfo(_("Activated") + " " + "|".join("%s %s" % (Extractor.__name__,Extractor.VERSION) for Extractor in self.extractors))
-
-            if self.getConfig("waitall"):
-                self.extractPackage(*self.queue.get())  #: Resume unfinished extractions
-            else:
-                super(ExtractArchive, self).initPeriodical()
-
+            self.extractQueued()  #: Resume unfinished extractions
         else:
             self.logInfo(_("No Extract plugins activated"))
 
+    @threaded
+    def extractQueued(self,thread):
+        packages = self.queue.get()
+        while packages:
+            if self.lastPackage: # called from allDownloadsProcessed
+                self.lastPackage = False
+                if self.extract(packages, thread):  #@NOTE: check only if all gone fine, no failed reporting for now
+                    self.manager.dispatchEvent("all_archives_extracted")
+                self.manager.dispatchEvent("all_archives_processed")
+            else:
+                if self.extract(packages, thread):  #@NOTE: check only if all gone fine, no failed reporting for now
+                    pass
 
-    def periodical(self):
-        if not self.extracting:
-            self.extractPackage(*self.queue.get())
+            packages = self.queue.get() # check for packages added during extraction
 
 
     @Expose
     def extractPackage(self, *ids):
         """ Extract packages with given id"""
-        self.manager.startThread(self.extract, ids)
+        for id in ids:
+            self.queue.add(id)
+        if not self.getConfig("waitall") and not self.extracting:
+            self.extractQueued()
+
+
+    def packageDeleted(self, pid):
+        self.queue.remove(pid)
 
 
     def packageFinished(self, pypack):
         self.queue.add(pypack.id)
+        if not self.getConfig("waitall") and not self.extracting:
+            self.extractQueued()
 
 
-    @threaded
-    def allDownloadsProcessed(self, thread):
-        if self.extract(self.queue.get(), thread):  #@NOTE: check only if all gone fine, no failed reporting for now
-            self.manager.dispatchEvent("all_archives_extracted")
-
-        self.manager.dispatchEvent("all_archives_processed")
+    def allDownloadsProcessed(self):
+        self.lastPackage = True
+        if not self.extracting:
+            self.extractQueued()
 
 
     def extract(self, ids, thread=None):
@@ -315,13 +325,17 @@ class ExtractArchive(Hook):
 
                             if recursive and os.path.isfile(file):
                                 new_files_ids.append((filename, fid, os.path.dirname(filename)))  # append as new target
-
+                        
+                        pyfile = self.core.files.getFile(fid)
+                        self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, new_files)
+                        
                 files_ids = new_files_ids  # also check extracted files
 
             if matched:
                 if success:
                     extracted.append(pid)
                     self.manager.dispatchEvent("package_extracted", pypack)
+
                 else:
                     failed.append(pid)
                     self.manager.dispatchEvent("package_extract_failed", pypack)
@@ -352,50 +366,65 @@ class ExtractArchive(Hook):
 
         encrypted = False
         try:
-            try:
-                archive.check()
-
-            except CRCError, e:
-                self.logDebug(name, e)
-                self.logInfo(name, _("Header protected"))
-
-                if self.getConfig("repair"):
-                    self.logWarning(name, _("Repairing..."))
-
-                    pyfile.setCustomStatus(_("repairing"))
-                    pyfile.setProgress(0)
-
-                    repaired = archive.repair()
-
-                    pyfile.setProgress(100)
-
-                    if not repaired and not self.getConfig("keepbroken"):
-                        raise CRCError("Archive damaged")
-
-            except PasswordError:
-                self.logInfo(name, _("Password protected"))
-                encrypted = True
-
-            except ArchiveError, e:
-                raise ArchiveError(e)
-
-            self.logDebug("Password: %s" % (password or "No provided"))
+            self.logDebug("Password: %s" % (password or "None provided"))
+            passwords = uniqify([password] + self.getPasswords(False)) if self.getConfig("usepasswordfile") else [password]
+            for pw in passwords:
+                try:
+                    if self.getConfig("test") or self.repair:
+                        pyfile.setCustomStatus(_("testing"))
+                        if pw:
+                            self.logDebug("Testing with password: %s" % pw)
+                        pyfile.setProgress(0)
+                        archive.test(pw)
+                        pyfile.setProgress(100)
+                    else:
+                        archive.check(pw)
+                    
+                    self.addPassword(pw)
+                    break
+
+                except PasswordError:
+                    if not encrypted:
+                        self.logInfo(name, _("Password protected"))
+                        encrypted = True
+                    
+                except CRCError, e:
+                    self.logDebug(name, e)
+                    self.logInfo(name, _("CRC Error"))
+
+                    if self.repair:
+                        self.logWarning(name, _("Repairing..."))
+
+                        pyfile.setCustomStatus(_("repairing"))
+                        pyfile.setProgress(0)
+                        repaired = archive.repair()
+                        pyfile.setProgress(100)
+
+                        if not repaired and not self.getConfig("keepbroken"):
+                            raise CRCError("Archive damaged")
+
+                        self.addPassword(pw)
+                        break
+                    
+                    raise CRCError("Archive damaged")
+                        
+                except ArchiveError, e:
+                    raise ArchiveError(e)
 
             pyfile.setCustomStatus(_("extracting"))
             pyfile.setProgress(0)
 
             if not encrypted or not self.getConfig("usepasswordfile"):
+                self.logDebug("Extracting using password: %s" % (password or "None"))
                 archive.extract(password)
             else:
                 for pw in filter(None, uniqify([password] + self.getPasswords(False))):
                     try:
-                        self.logDebug("Try password: %s" % pw)
+                        self.logDebug("Extracting using password: %s" % pw)
 
-                        ispw = archive.isPassword(pw)
-                        if ispw or ispw is None:
-                            archive.extract(pw)
-                            self.addPassword(pw)
-                            break
+                        archive.extract(pw)
+                        self.addPassword(pw)
+                        break
 
                     except PasswordError:
                         self.logDebug("Password was wrong")
@@ -419,9 +448,7 @@ class ExtractArchive(Hook):
                         self.logDebug("%s does not exists" % f)
 
             self.logInfo(name, _("Extracting finished"))
-
             extracted_files = archive.files or archive.list()
-            self.manager.dispatchEvent("archive_extracted", pyfile, archive.out, archive.filename, extracted_files)
 
             return extracted_files
 
diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py
index b445f1497..8bf1875cf 100644
--- a/module/plugins/internal/Extractor.py
+++ b/module/plugins/internal/Extractor.py
@@ -19,7 +19,7 @@ class PasswordError(Exception):
 
 class Extractor:
     __name__    = "Extractor"
-    __version__ = "0.20"
+    __version__ = "0.21"
 
     __description__ = """Base extractor plugin"""
     __license__     = "GPLv3"
@@ -30,6 +30,7 @@ class Extractor:
 
     EXTENSIONS = []
     VERSION    = ""
+    REPAIR     = False
 
 
     @classmethod
@@ -90,23 +91,24 @@ class Extractor:
 
 
     def check(self):
-        """Check if password if needed. Raise ArchiveError if integrity is
-        questionable.
+        """Quick Check by listing content of archive.
+        Raises error if password is needed, integrity is questionable or else.
 
-        :return: boolean
+        :raises PasswordError
+        :raises CRCError
         :raises ArchiveError
         """
-        raise PasswordError
-
-
-    def isPassword(self, password):
-        """ Check if the given password is/might be correct.
-        If it can not be decided at this point return true.
-
-        :param password:
-        :return: boolean
+        raise NotImplementedError
+        
+    def test(self):
+        """Testing with Extractors buildt-in method
+        Raises error if password is needed, integrity is questionable or else.
+        
+        :raises PasswordError
+        :raises CRCError
+        :raises ArchiveError
         """
-        return None
+        raise NotImplementedError
 
 
     def repair(self):
diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py
index 7ad6b0d7a..bfa7f3943 100644
--- a/module/plugins/internal/SevenZip.py
+++ b/module/plugins/internal/SevenZip.py
@@ -11,7 +11,7 @@ from module.utils import fs_encode, save_join
 
 class SevenZip(UnRar):
     __name__    = "SevenZip"
-    __version__ = "0.08"
+    __version__ = "0.09"
 
     __description__ = """7-Zip extractor plugin"""
     __license__     = "GPLv3"
@@ -32,9 +32,9 @@ class SevenZip(UnRar):
 
     #@NOTE: there are some more uncovered 7z formats
     re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)')
-    re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password)', re.I)
-    re_wrongcrc = re.compile(r'Encrypted\s+\=\s+\+', re.I)
-    re_version   = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I)
+    re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password|Encrypted\s+\=\s+\+)', re.I)
+    re_wrongcrc = re.compile(r'CRC Failed|Can not open file', re.I)
+    re_version  = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I)
 
 
     @classmethod
@@ -52,36 +52,38 @@ class SevenZip(UnRar):
         return True
 
 
-    def check(self):
+    def test(self, password):
         file = fs_encode(self.filename)
 
-        p = self.call_cmd("t", file)
+        # 7z can't distinguish crc and pw error in test
+        p = self.call_cmd("l", "-slt", file)
         out, err = p.communicate()
 
-        if p.returncode > 1:
+        if self.re_wrongpwd.search(out):
+            raise PasswordError
+
+        if self.re_wrongpwd.search(err):
+            raise PasswordError
+
+        if self.re_wrongcrc.search(err):
             raise CRCError(err)
 
+
+
+    def check(self, password):
+        file = fs_encode(self.filename)
+
         p = self.call_cmd("l", "-slt", file)
         out, err = p.communicate()
 
-        if p.returncode > 1:
-            raise ArchiveError(_("Process return code: %d") % p.returncode)
-
         # check if output or error macthes the 'wrong password'-Regexp
         if self.re_wrongpwd.search(out):
             raise PasswordError
 
-        # check if output matches 'Encrypted = +'
         if self.re_wrongcrc.search(out):
             raise CRCError(_("Header protected"))
 
 
-    def isPassword(self, password):
-        p = self.call_cmd("l", fs_encode(self.filename), password=password)
-        p.communicate()
-        return p.returncode == 0
-
-
     def repair(self):
         return False
 
@@ -142,7 +144,7 @@ class SevenZip(UnRar):
 
         #set a password
         if "password" in kwargs and kwargs["password"]:
-            args.append("-p'%s'" % kwargs["password"])
+            args.append("-p%s" % kwargs["password"])
         else:
             args.append("-p-")
 
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
index 54d64c430..ca8fdd326 100644
--- a/module/plugins/internal/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -22,7 +22,7 @@ def renice(pid, value):
 
 class UnRar(Extractor):
     __name__    = "UnRar"
-    __version__ = "1.13"
+    __version__ = "1.14"
 
     __description__ = """Rar extractor plugin"""
     __license__     = "GPLv3"
@@ -33,34 +33,40 @@ class UnRar(Extractor):
 
     CMD = "unrar"
     VERSION = ""
-
     EXTENSIONS = [".rar"]
 
 
-    re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?',re.I)
+    re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?(\.rev|\.bad)?',re.I)
 
     re_filefixed = re.compile(r'Building (.+)')
     re_filelist  = re.compile(r'^(.)(\s*[\w\.\-]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I)
 
     re_wrongpwd  = re.compile(r'password', re.I)
-    re_wrongcrc  = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I)
+    re_wrongcrc  = re.compile(r'encrypted|damaged|CRC failed|checksum error|corrupt', re.I)
 
-    re_version   = re.compile(r'UNRAR\s(\d+\.\d+)', re.I)
+    re_version   = re.compile(r'(?:UN)?RAR\s(\d+\.\d+)', re.I)
 
 
     @classmethod
     def isUsable(cls):
         if os.name == "nt":
-            cls.CMD = os.path.join(pypath, "UnRAR.exe")
-            p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
-            out, err = p.communicate()
-        else:
             try:
+                cls.CMD = os.path.join(pypath, "RAR.exe")
                 p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
                 out, err = p.communicate()
-
-            except OSError:  #: fallback to rar
-                cls.CMD = "rar"
+                cls.__name__ = "RAR"
+                cls.REPAIR = True
+            except OSError:
+                cls.CMD = os.path.join(pypath, "UnRAR.exe")
+                p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
+                out, err = p.communicate()
+        else:
+            try:
+                p = Popen(["rar"], stdout=PIPE, stderr=PIPE)
+                out, err = p.communicate()
+                cls.__name__ = "RAR"
+                cls.REPAIR = True
+            except OSError:  #: fallback to unrar
                 p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
                 out, err = p.communicate()
 
@@ -74,13 +80,25 @@ class UnRar(Extractor):
         multipart = cls.re_multipart.search(filename)
         if multipart:
             # First Multipart file (part1.rar for *.part1-9.rar format or *.rar for .r1-9 format) handled as normal Archive
-            return False if (multipart.group(1) == "part" and int(multipart.group(2)) == 1) else True
+            return False if (multipart.group(1) == "part" and int(multipart.group(2)) == 1 and not multipart.group(3)) else True
 
         return False
 
 
-    def check(self):
-        p = self.call_cmd("l", "-v", fs_encode(self.filename))
+    def test(self, password):
+        p = self.call_cmd("t", "-v", fs_encode(self.filename), password=password)
+        self._progress(p)
+        err = p.stderr.read().strip()
+        
+        if self.re_wrongpwd.search(err):
+            raise PasswordError
+
+        if self.re_wrongcrc.search(err):
+            raise CRCError(err)
+
+
+    def check(self, password):
+        p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password)
         out, err = p.communicate()
 
         if self.re_wrongpwd.search(err):
@@ -95,35 +113,14 @@ class UnRar(Extractor):
                 raise PasswordError
 
 
-    def isPassword(self, password):
-        # at this point we can only verify header protected files
-        p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password)
-        out, err = p.communicate()
-        return False if self.re_wrongpwd.search(err) else True
-
-
     def repair(self):
         p = self.call_cmd("rc", fs_encode(self.filename))
 
         # communicate and retrieve stderr
         self._progress(p)
         err = p.stderr.read().strip()
-
         if err or p.returncode:
-            p = self.call_cmd("r", fs_encode(self.filename))
-
-            # communicate and retrieve stderr
-            self._progress(p)
-            err = p.stderr.read().strip()
-
-            if err or p.returncode:
-                return False
-            else:
-                dir  = os.path.dirname(filename)
-                name = re_filefixed.search(out).group(1)
-
-                self.filename = os.path.join(dir, name)
-
+            return False
         return True
 
 
diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py
index 704b49ca0..d95afbc70 100644
--- a/module/plugins/internal/UnZip.py
+++ b/module/plugins/internal/UnZip.py
@@ -12,7 +12,7 @@ from module.utils import fs_encode
 
 class UnZip(Extractor):
     __name__    = "UnZip"
-    __version__ = "1.10"
+    __version__ = "1.11"
 
     __description__ = """Zip extractor plugin"""
     __license__     = "GPLv3"
@@ -34,7 +34,11 @@ class UnZip(Extractor):
             return z.namelist()
 
 
-    def check(self):
+    def check(self, password):
+        pass
+
+
+    def test(self):
         with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z:
             badfile = z.testzip()