Merge branch 'pr/n1154_immenz' into stable

5 files changed, 82 insertions, 63 deletions

diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py
index 88036da39..3ea8839dc 100644
--- a/module/plugins/hooks/ExtractArchive.py
+++ b/module/plugins/hooks/ExtractArchive.py
@@ -104,7 +104,7 @@ class ArchiveQueue(object):
 class ExtractArchive(Hook):
     __name__    = "ExtractArchive"
     __type__    = "hook"
-    __version__ = "1.26"
+    __version__ = "1.29"
 
     __config__ = [("activated"       , "bool"  , "Activated"                                 , True                                                                     ),
                   ("fullpath"        , "bool"  , "Extract with full paths"                   , True                                                                     ),
@@ -124,7 +124,8 @@ class ExtractArchive(Hook):
 
     __description__ = """Extract different kind of archives"""
     __license__     = "GPLv3"
-    __authors__     = [("Walter Purcaro", "vuolter@gmail.com")]
+    __authors__     = [("Walter Purcaro", "vuolter@gmail.com"),
+                       ("Immenz"        , "immenz@gmx.net"   )]
 
 
     event_list = ["allDownloadsProcessed"]
@@ -171,7 +172,7 @@ class ExtractArchive(Hook):
                     print_exc()
 
         if self.extractors:
-            self.logInfo(_("Activated") + " " + " ".join(Extractor.__name__ for Extractor in self.extractors))
+            self.logInfo(_("Activated") + " " + "|".join("%s %s" % (Extractor.__name__,Extractor.VERSION) for Extractor in self.extractors))
 
             if self.getConfig("waitall"):
                 self.extractPackage(*self.queue.get())  #: Resume unfinished extractions
@@ -198,14 +199,14 @@ class ExtractArchive(Hook):
 
 
     @threaded
-    def allDownloadsProcessed(self):
-        if self.extract(self.queue.get()):  #@NOTE: check only if all gone fine, no failed reporting for now
+    def allDownloadsProcessed(self, thread):
+        if self.extract(self.queue.get(), thread):  #@NOTE: check only if all gone fine, no failed reporting for now
             self.manager.dispatchEvent("all_archives_extracted")
 
         self.manager.dispatchEvent("all_archives_processed")
 
 
-    def extract(self, ids):
+    def extract(self, ids, thread=None):
         if not ids:
             return False
 
@@ -258,14 +259,14 @@ class ExtractArchive(Hook):
 
             matched   = False
             success   = True
-            files_ids = [(save_join(dl, pypack.folder, pylink['name']), pylink['id']) for pylink in pypack.getChildren().itervalues()]
+            files_ids = [(save_join(dl, pypack.folder, pylink['name']), pylink['id'], out) for pylink in pypack.getChildren().itervalues()]
 
             # check as long there are unseen files
             while files_ids:
                 new_files_ids = []
 
                 if extensions:
-                    files_ids = [(fname, fid) for fname, fid in files_ids \
+                    files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids \
                                  if filter(lambda ext: fname.lower().endswith(ext), extensions)]
 
                 for Extractor in self.extractors:
@@ -274,25 +275,18 @@ class ExtractArchive(Hook):
                         self.logDebug("Targets for %s: %s" % (Extractor.__name__, targets))
                         matched = True
 
-                    for fname, fid in targets:
+                    for fname, fid, fout in targets:
                         name = os.path.basename(fname)
 
-                        pname = replace_patterns(fname, self.NAME_REPLACEMENTS)
-                        if pname not in processed:
-                            processed.append(pname)  #: prevent extracting same file twice
-                        else:
-                            self.logDebug(name, "Skipped")
-                            continue
-                            
                         if not os.path.exists(fname):
                             self.logDebug(name, "File not found")
                             continue
 
-                        self.logInfo(name, _("Extract to: %s") % out)
+                        self.logInfo(name, _("Extract to: %s") % fout)
                         try:
                             archive = Extractor(self,
                                                 fname,
-                                                out,
+                                                fout,
                                                 fullpath,
                                                 overwrite,
                                                 excludefiles,
@@ -302,13 +296,14 @@ class ExtractArchive(Hook):
                                                 fid)
                             archive.init()
 
-                            new_files = self._extract(archive, fid, pypack.password)
+                            new_files = self._extract(archive, fid, pypack.password, thread)
 
                         except Exception, e:
                             self.logError(name, e)
                             success = False
                             continue
 
+                        files_ids.remove((fname, fid, fout)) # don't let other extractors spam log
                         self.logDebug("Extracted files: %s" % new_files)
                         self.setPermissions(new_files)
 
@@ -319,7 +314,7 @@ class ExtractArchive(Hook):
                                 continue
 
                             if recursive and os.path.isfile(file):
-                                new_files_ids.append((filename, fid))  # append as new target
+                                new_files_ids.append((filename, fid, os.path.dirname(filename)))  # append as new target
 
                 files_ids = new_files_ids  # also check extracted files
 
@@ -348,10 +343,11 @@ class ExtractArchive(Hook):
         return True if not failed else False
 
 
-    def _extract(self, archive, fid, password):
+    def _extract(self, archive, fid, password, thread):
         pyfile = self.core.files.getFile(fid)
         name   = os.path.basename(archive.filename)
 
+        thread.addActive(pyfile)
         pyfile.setStatus("processing")
 
         encrypted = False
@@ -391,7 +387,7 @@ class ExtractArchive(Hook):
             if not encrypted or not self.getConfig("usepasswordfile"):
                 archive.extract(password)
             else:
-                for pw in uniqify([password] + self.getPasswords(False)):
+                for pw in filter(None, uniqify([password] + self.getPasswords(False))):
                     try:
                         self.logDebug("Try password: %s" % pw)
 
diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py
index 45c13c159..b445f1497 100644
--- a/module/plugins/internal/Extractor.py
+++ b/module/plugins/internal/Extractor.py
@@ -19,21 +19,28 @@ class PasswordError(Exception):
 
 class Extractor:
     __name__    = "Extractor"
-    __version__ = "0.18"
+    __version__ = "0.20"
 
     __description__ = """Base extractor plugin"""
     __license__     = "GPLv3"
     __authors__     = [("RaNaN", "ranan@pyload.org"),
-                       ("Walter Purcaro", "vuolter@gmail.com")]
+                       ("Walter Purcaro", "vuolter@gmail.com"),
+                       ("Immenz", "immenz@gmx.net")]
 
 
     EXTENSIONS = []
+    VERSION    = ""
 
 
     @classmethod
     def isArchive(cls, filename):
         name = os.path.basename(filename).lower()
-        return any(name.endswith(ext) for ext in cls.EXTENSIONS)
+        return any(name.endswith(ext) for ext in cls.EXTENSIONS) and not cls.isMultipart(filename)
+
+
+    @classmethod
+    def isMultipart(cls,filename):
+        return False
 
 
     @classmethod
@@ -50,7 +57,7 @@ class Extractor:
         :param files_ids: List of filepathes
         :return: List of targets, id tuple list
         """
-        return [(fname, id) for fname, id in files_ids if cls.isArchive(fname)]
+        return [(fname, id, fout) for fname, id, fout in files_ids if cls.isArchive(fname)]
 
 
     def __init__(self, manager, filename, out,
diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py
index 96e664573..7ad6b0d7a 100644
--- a/module/plugins/internal/SevenZip.py
+++ b/module/plugins/internal/SevenZip.py
@@ -11,7 +11,7 @@ from module.utils import fs_encode, save_join
 
 class SevenZip(UnRar):
     __name__    = "SevenZip"
-    __version__ = "0.07"
+    __version__ = "0.08"
 
     __description__ = """7-Zip extractor plugin"""
     __license__     = "GPLv3"
@@ -19,7 +19,8 @@ class SevenZip(UnRar):
                        ("Walter Purcaro", "vuolter@gmail.com")]
 
 
-    CMD = "7z"
+    CMD     = "7z"
+    VERSION = ""
 
     EXTENSIONS = [".7z", ".xz", ".zip", ".gz", ".gzip", ".tgz", ".bz2", ".bzip2",
                   ".tbz2", ".tbz", ".tar", ".wim", ".swm", ".lzma", ".rar", ".cab",
@@ -33,6 +34,7 @@ class SevenZip(UnRar):
     re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)')
     re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password)', re.I)
     re_wrongcrc = re.compile(r'Encrypted\s+\=\s+\+', re.I)
+    re_version   = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I)
 
 
     @classmethod
@@ -40,10 +42,12 @@ class SevenZip(UnRar):
         if os.name == "nt":
             cls.CMD = os.path.join(pypath, "7z.exe")
             p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
-            p.communicate()
+            out,err = p.communicate()
         else:
             p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
-            p.communicate()
+            out, err = p.communicate()
+        
+        cls.VERSION = cls.re_version.search(out).group(1)
 
         return True
 
@@ -143,9 +147,9 @@ class SevenZip(UnRar):
             args.append("-p-")
 
         #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue
-        call = [self.cmd, command] + args + list(xargs)
+        call = [self.CMD, command] + args + list(xargs)
 
-        self.manager.logDebug(" ".join(map(decode, call)))
+        self.manager.logDebug(" ".join(call))
 
         p = Popen(call, stdout=PIPE, stderr=PIPE)
         return p
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
index 81cfb38a7..54d64c430 100644
--- a/module/plugins/internal/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -22,50 +22,63 @@ def renice(pid, value):
 
 class UnRar(Extractor):
     __name__    = "UnRar"
-    __version__ = "1.10"
+    __version__ = "1.13"
 
     __description__ = """Rar extractor plugin"""
     __license__     = "GPLv3"
     __authors__     = [("RaNaN", "RaNaN@pyload.org"),
-                       ("Walter Purcaro", "vuolter@gmail.com")]
+                       ("Walter Purcaro", "vuolter@gmail.com"),
+                       ("Immenz", "immenz@gmx.net"),]
 
 
     CMD = "unrar"
+    VERSION = ""
 
-    # TODO: Find out what Filetypes Unrar supports exactly
-    EXTENSIONS = [".rar", ".cab", ".arj", ".lzh", ".tar", ".gz", ".bz2",
-                  ".ace", ".uue", ".jar", ".iso", ".7z", ".xz", ".z"]
+    EXTENSIONS = [".rar"]
 
-    #@NOTE: there are some more uncovered rar formats
-    re_rarpart1 = re.compile(r'\.part(\d+)\.rar$', re.I)
-    re_rarpart2 = re.compile(r'\.r(\d+)$', re.I)
+
+    re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?',re.I)
 
     re_filefixed = re.compile(r'Building (.+)')
-    re_filelist  = re.compile(r'(.+)\s+(\D+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)')
+    re_filelist  = re.compile(r'^(.)(\s*[\w\.\-]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I)
 
     re_wrongpwd  = re.compile(r'password', re.I)
     re_wrongcrc  = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I)
 
+    re_version   = re.compile(r'UNRAR\s(\d+\.\d+)', re.I)
+
 
     @classmethod
     def isUsable(cls):
         if os.name == "nt":
             cls.CMD = os.path.join(pypath, "UnRAR.exe")
             p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
-            p.communicate()
+            out, err = p.communicate()
         else:
             try:
                 p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
-                p.communicate()
+                out, err = p.communicate()
 
             except OSError:  #: fallback to rar
                 cls.CMD = "rar"
                 p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
-                p.communicate()
+                out, err = p.communicate()
+
+        cls.VERSION = cls.re_version.search(out).group(1)
 
         return True
 
 
+    @classmethod
+    def isMultipart(cls,filename):
+        multipart = cls.re_multipart.search(filename)
+        if multipart:
+            # First Multipart file (part1.rar for *.part1-9.rar format or *.rar for .r1-9 format) handled as normal Archive
+            return False if (multipart.group(1) == "part" and int(multipart.group(2)) == 1) else True
+
+        return False
+
+
     def check(self):
         p = self.call_cmd("l", "-v", fs_encode(self.filename))
         out, err = p.communicate()
@@ -161,23 +174,14 @@ class UnRar(Extractor):
 
 
     def getDeleteFiles(self):
-        files = []
-
-        for i in (1, 2):
-            try:
-                dir, name = os.path.split(self.filename)
+        dir, name = os.path.split(self.filename)
 
-                part     = getattr(self, "re_rarpart%d" % i).search(name).group(1)
-                new_name = name[::-1].replace((".part%s.rar" % part)[::-1], ".part*.rar"[::-1], 1)[::-1]
-                file     = fs_encode(os.path.join(dir, new_name))
+        # actually extracted file
+        files = [self.filename]
 
-                files.extend(glob(file))
-
-            except Exception:
-                continue
-
-        if self.filename not in files:
-            files.insert(0, self.filename)
+        # eventually Multipart Files
+        files.extend(save_join(dir, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir))
+                     if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file))
 
         return files
 
@@ -195,9 +199,16 @@ class UnRar(Extractor):
             self.manager.logError(err.strip())
 
         result = set()
-        for f in decode(out).splitlines():
-            f = f.strip()
-            result.add(save_join(self.out, f))
+        if not self.fullpath and self.VERSION.startswith('5'):
+            # NOTE: Unrar 5 always list full path
+            for f in decode(out).splitlines():
+                f = save_join(self.out, os.path.basename(f.strip()))
+                if os.path.isfile(f):
+                    result.add(save_join(self.out, os.path.basename(f)))
+        else:
+            for f in decode(out).splitlines():
+                f = f.strip()
+                result.add(save_join(self.out, f))
 
         return list(result)
 
diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py
index f81c235c1..caa0ecc0c 100644
--- a/module/plugins/internal/UnZip.py
+++ b/module/plugins/internal/UnZip.py
@@ -12,7 +12,7 @@ from module.utils import fs_encode
 
 class UnZip(Extractor):
     __name__    = "UnZip"
-    __version__ = "1.08"
+    __version__ = "1.09"
 
     __description__ = """Zip extractor plugin"""
     __license__     = "GPLv3"
@@ -20,6 +20,7 @@ class UnZip(Extractor):
 
 
     EXTENSIONS = [".zip", ".zip64"]
+    VERSION ="(python %s.%s.%s)" % (sys.version_info.major, sys.version_info.minor, sys.version_info.micro)
 
 
     @classmethod