Update extractors (1)

4 files changed, 115 insertions, 122 deletions

diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py
index 41ba4d429..1d035f3e6 100644
--- a/module/plugins/internal/Extractor.py
+++ b/module/plugins/internal/Extractor.py
@@ -5,7 +5,7 @@ import re
 
 from module.PyFile import PyFile
 from module.plugins.internal.Plugin import Plugin
-from module.plugins.internal.utils import encode
+from module.plugins.internal.misc import encode
 
 
 class ArchiveError(Exception):
@@ -40,7 +40,7 @@ class Extractor(Plugin):
     @classmethod
     def isarchive(cls, filename):
         name = os.path.basename(filename).lower()
-        return any(name.endswith(ext) for ext in cls.EXTENSIONS)
+        return any(name.endswith('.' + ext) for ext in cls.EXTENSIONS)
 
 
     @classmethod
@@ -63,43 +63,48 @@ class Extractor(Plugin):
         :param files_ids: List of filepathes
         :return: List of targets, id tuple list
         """
-        targets = []
+        targets   = []
         processed = []
 
-        for fname, id, fout in files_ids:
-            if cls.isarchive(fname):
-                pname = re.sub(cls.re_multipart, "", fname) if cls.ismultipart(fname) else os.path.splitext(fname)[0]
-                if pname not in processed:
-                    processed.append(pname)
-                    targets.append((fname, id, fout))
+        for id, fname, fout in files_ids:
+            if not cls.isarchive(fname):
+                continue
+
+            if cls.ismultipart(fname):
+                pname = re.sub(cls._RE_PART, "", fname)
+            else:
+                pname = os.path.splitext(fname)[0]
+
+            if pname in processed:
+                continue
+
+            processed.append(pname)
+            targets.append((id, fname, fout))
 
         return targets
 
 
-    def __init__(self, plugin, filename, out,
+    def __init__(self, pyfile, filename, out,
                  fullpath=True,
                  overwrite=False,
                  excludefiles=[],
                  priority=0,
-                 keepbroken=False,
-                 fid=None):
+                 keepbroken=False):
         """
         Initialize extractor for specific file
         """
-        self._init(plugin.pyload)
+        self._init(pyfile.m.core)
 
-        self.plugin       = plugin
+        self.pyfile       = pyfile
         self.filename     = filename
+        self.name         = os.path.basename(filename)
         self.out          = out
         self.fullpath     = fullpath
         self.overwrite    = overwrite
         self.excludefiles = excludefiles
         self.priority     = priority
         self.keepbroken   = keepbroken
-        self.files        = []  #: Store extracted files here
-
-        pyfile = self.pyload.files.getFile(fid) if fid else None
-        self.notify_progress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None
+        self.progress     = lambda x: pyfile.setProgress(int(x))
 
         self.init()
 
@@ -109,9 +114,14 @@ class Extractor(Plugin):
         return encode(self.filename)
 
 
+    @property
+    def dest(self):
+        return encode(self.out)
+
+
     def _log(self, level, plugintype, pluginname, messages):
         messages = (self.__name__,) + messages
-        return self.plugin._log(level, plugintype, self.plugin.__name__, messages)
+        return self.pyfile.plugin._log(level, plugintype, self.pyfile.plugin.__name__, messages)
 
 
     def verify(self, password=None):
@@ -134,7 +144,7 @@ class Extractor(Plugin):
         raise NotImplementedError
 
 
-    def items(self):
+    def chunks(self):
         """
         Return list of archive parts
         """
@@ -143,6 +153,6 @@ class Extractor(Plugin):
 
     def list(self, password=None):
         """
-        Populate self.files at some point while extracting
+        Return list of archive files
         """
-        return self.files
+        raise NotImplementedError
diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py
index bf33332ea..87a2fa433 100644
--- a/module/plugins/internal/SevenZip.py
+++ b/module/plugins/internal/SevenZip.py
@@ -5,13 +5,13 @@ import re
 import subprocess
 
 from module.plugins.internal.UnRar import UnRar, ArchiveError, CRCError, PasswordError
-from module.plugins.internal.utils import fs_join, renice
+from module.plugins.internal.misc import encode, fsjoin, renice
 
 
 class SevenZip(UnRar):
     __name__    = "SevenZip"
     __type__    = "extractor"
-    __version__ = "0.18"
+    __version__ = "0.19"
     __status__  = "testing"
 
     __description__ = """7-Zip extractor plugin"""
@@ -20,19 +20,18 @@ class SevenZip(UnRar):
                        ("Michael Nowak" , None               )]
 
 
-    CMD         = "7z"
-    EXTENSIONS  = [".7z", ".xz", ".zip", ".gz", ".gzip", ".tgz", ".bz2", ".bzip2",
-                   ".tbz2", ".tbz", ".tar", ".wim", ".swm", ".lzma", ".rar", ".cab",
-                   ".arj", ".z", ".taz", ".cpio", ".rpm", ".deb", ".lzh", ".lha",
-                   ".chm", ".chw", ".hxs", ".iso", ".msi", ".doc", ".xls", ".ppt",
-                   ".dmg", ".xar", ".hfs", ".exe", ".ntfs", ".fat", ".vhd", ".mbr",
-                   ".squashfs", ".cramfs", ".scap"]
+    CMD        = "7z"
+    EXTENSIONS = ["7z", "xz", "zip", "gz", "gzip", "tgz", "bz2", "bzip2", "tbz2",
+                  "tbz", "tar", "wim", "swm", "lzma", "rar", "cab", "arj", "z",
+                  "taz", "cpio", "rpm", "deb", "lzh", "lha", "chm", "chw", "hxs",
+                  "iso", "msi", "doc", "xls", "ppt", "dmg", "xar", "hfs", "exe",
+                  "ntfs", "fat", "vhd", "mbr", "squashfs", "cramfs", "scap"]
 
     #@NOTE: there are some more uncovered 7z formats
-    re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)')
-    re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password|Encrypted\s+\=\s+\+)', re.I)
-    re_wrongcrc = re.compile(r'CRC Failed|Can not open file', re.I)
-    re_version  = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I)
+    _RE_FILES   = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)')
+    _RE_BADPWD  = re.compile(r'(Can not open encrypted archive|Wrong password|Encrypted\s+\=\s+\+)', re.I)
+    _RE_BADCRC  = re.compile(r'CRC Failed|Can not open file', re.I)
+    _RE_VERSION = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I)
 
 
     @classmethod
@@ -48,7 +47,7 @@ class SevenZip(UnRar):
             return False
 
         else:
-            m = cls.re_version.search(out)
+            m = cls._RE_VERSION.search(out)
             if m is not None:
                 cls.VERSION = m.group(1)
 
@@ -60,33 +59,33 @@ class SevenZip(UnRar):
         p = self.call_cmd("l", "-slt", self.target)
         out, err = p.communicate()
 
-        if self.re_wrongpwd.search(out):
+        if self._RE_BADPWD.search(out):
             raise PasswordError
 
-        elif self.re_wrongpwd.search(err):
+        elif self._RE_BADPWD.search(err):
             raise PasswordError
 
-        elif self.re_wrongcrc.search(out):
+        elif self._RE_BADCRC.search(out):
             raise CRCError(_("Header protected"))
 
-        elif self.re_wrongcrc.search(err):
+        elif self._RE_BADCRC.search(err):
             raise CRCError(err)
 
 
     def extract(self, password=None):
         command = "x" if self.fullpath else "e"
 
-        p = self.call_cmd(command, '-o' + self.out, self.target, password=password)
+        p = self.call_cmd(command, '-o' + self.dest, self.target, password=password)
 
         #: Communicate and retrieve stderr
-        self._progress(p)
+        self.progress(p)
         err = p.stderr.read().strip()
 
         if err:
-            if self.re_wrongpwd.search(err):
+            if self._RE_BADPWD.search(err):
                 raise PasswordError
 
-            elif self.re_wrongcrc.search(err):
+            elif self._RE_BADCRC.search(err):
                 raise CRCError(err)
 
             else:  #: Raise error if anything is on stderr
@@ -95,8 +94,6 @@ class SevenZip(UnRar):
         if p.returncode > 1:
             raise ArchiveError(_("Process return code: %d") % p.returncode)
 
-        self.files = self.list(password)
-
 
     def list(self, password=None):
         command = "l" if self.fullpath else "l"
@@ -111,9 +108,9 @@ class SevenZip(UnRar):
             raise ArchiveError(_("Process return code: %d") % p.returncode)
 
         result = set()
-        for groups in self.re_filelist.findall(out):
+        for groups in self._RE_FILES.findall(out):
             f = groups[-1].strip()
-            result.add(fs_join(self.out, f))
+            result.add(fsjoin(self.dest, f))
 
         return list(result)
 
@@ -133,8 +130,8 @@ class SevenZip(UnRar):
 
         #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue
         call = [self.CMD, command] + args + list(xargs)
-
         self.log_debug(" ".join(call))
+        call = map(encode, call)
 
         p = subprocess.Popen(call, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
index 963ca2a2e..93fc04c32 100644
--- a/module/plugins/internal/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -6,34 +6,32 @@ import string
 import subprocess
 
 from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError
-from module.plugins.internal.utils import decode, fs_join, renice
+from module.plugins.internal.misc import decode, encode, fsjoin, renice
 
 
 class UnRar(Extractor):
     __name__    = "UnRar"
     __type__    = "extractor"
-    __version__ = "1.29"
+    __version__ = "1.30"
     __status__  = "testing"
 
-    __description__ = """Rar extractor plugin"""
+    __description__ = """RAR extractor plugin"""
     __license__     = "GPLv3"
     __authors__     = [("RaNaN"         , "RaNaN@pyload.org" ),
                        ("Walter Purcaro", "vuolter@gmail.com"),
                        ("Immenz"        , "immenz@gmx.net"   )]
 
 
-    CMD          = "unrar"
-    EXTENSIONS   = [".rar"]
+    CMD        = "unrar"
+    EXTENSIONS = ["rar", "zip", "cab", "arj", "lzh", "tar", "gz", "ace", "uue",
+                  "bz2", "jar", "iso", "7z", "xz", "z"]
 
-    re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?(\.rev|\.bad)?', re.I)
-
-    re_filefixed = re.compile(r'Building (.+)')
-    re_filelist  = re.compile(r'^(.)(\s*[\w\-.]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.I | re.M)
-
-    re_wrongpwd  = re.compile(r'password', re.I)
-    re_wrongcrc  = re.compile(r'encrypted|damaged|CRC failed|checksum error|corrupt', re.I)
-
-    re_version   = re.compile(r'(?:UN)?RAR\s(\d+\.\d+)', re.I)
+    _RE_PART    = re.compile(r'\.(part|r)\d+(\.rar|\.rev)?(\.bad)?', re.I)
+    _RE_FIXNAME = re.compile(r'Building (.+)')
+    _RE_FILES   = re.compile(r'^(.)(\s*[\w\-.]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.I | re.M)
+    _RE_BADPWD  = re.compile(r'password', re.I)
+    _RE_BADCRC  = re.compile(r'encrypted|damaged|CRC failed|checksum error|corrupt', re.I)
+    _RE_VERSION = re.compile(r'(?:UN)?RAR\s(\d+\.\d+)', re.I)
 
 
     @classmethod
@@ -62,7 +60,7 @@ class UnRar(Extractor):
             except OSError:
                 return False
 
-        m = cls.re_version.search(out)
+        m = cls._RE_VERSION.search(out)
         if m is not None:
             cls.VERSION = m.group(1)
 
@@ -71,21 +69,21 @@ class UnRar(Extractor):
 
     @classmethod
     def ismultipart(cls, filename):
-        return True if cls.re_multipart.search(filename) else False
+        return True if cls._RE_PART.search(filename) else False
 
 
     def verify(self, password=None):
         p = self.call_cmd("l", "-v", self.target, password=password)
         out, err = p.communicate()
 
-        if self.re_wrongpwd.search(err):
+        if self._RE_BADPWD.search(err):
             raise PasswordError
 
-        if self.re_wrongcrc.search(err):
+        if self._RE_BADCRC.search(err):
             raise CRCError(err)
 
         #: Output only used to check if passworded files are present
-        for attr in self.re_filelist.findall(out):
+        for attr in self._RE_FILES.findall(out):
             if attr[0].startswith("*"):
                 raise PasswordError
 
@@ -94,14 +92,14 @@ class UnRar(Extractor):
         p = self.call_cmd("rc", self.target)
 
         #: Communicate and retrieve stderr
-        self._progress(p)
+        self.progress(p)
         err = p.stderr.read().strip()
 
         if err or p.returncode:
             p = self.call_cmd("r", self.target)
 
             # communicate and retrieve stderr
-            self._progress(p)
+            self.progress(p)
             err = p.stderr.read().strip()
 
             if err or p.returncode:
@@ -109,14 +107,14 @@ class UnRar(Extractor):
 
             else:
                 dir  = os.path.dirname(filename)
-                name = re_filefixed.search(out).group(1)
+                name = _RE_FIXNAME.search(out).group(1)
 
                 self.filename = os.path.join(dir, name)
 
         return True
 
 
-    def _progress(self, process):
+    def progress(self, process):
         s = ""
         while True:
             c = process.stdout.read(1)
@@ -125,7 +123,7 @@ class UnRar(Extractor):
                 break
             #: Reading a percentage sign -> set progress and restart
             if c == "%":
-                self.notify_progress(int(s))
+                self.notifyprogress(int(s))
                 s = ""
             #: Not reading a digit -> therefore restart
             elif c not in string.digits:
@@ -138,17 +136,17 @@ class UnRar(Extractor):
     def extract(self, password=None):
         command = "x" if self.fullpath else "e"
 
-        p = self.call_cmd(command, self.target, self.out, password=password)
+        p = self.call_cmd(command, self.target, self.dest, password=password)
 
         #: Communicate and retrieve stderr
-        self._progress(p)
+        self.progress(p)
         err = p.stderr.read().strip()
 
         if err:
-            if self.re_wrongpwd.search(err):
+            if self._RE_BADPWD.search(err):
                 raise PasswordError
 
-            elif self.re_wrongcrc.search(err):
+            elif self._RE_BADCRC.search(err):
                 raise CRCError(err)
 
             else:  #: Raise error if anything is on stderr
@@ -157,18 +155,16 @@ class UnRar(Extractor):
         if p.returncode:
             raise ArchiveError(_("Process return code: %d") % p.returncode)
 
-        self.files = self.list(password)
 
-
-    def items(self):
+    def chunks(self):
         dir, name = os.path.split(self.filename)
 
         #: Actually extracted file
         files = [self.filename]
 
         #: eventually Multipart Files
-        files.extend(fs_join(dir, os.path.basename(file)) for file in filter(self.ismultipart, os.listdir(dir))
-                     if re.sub(self.re_multipart, ".rar", name) == re.sub(self.re_multipart, ".rar", file))
+        files.extend(fsjoin(dir, os.path.basename(file)) for file in filter(self.ismultipart, os.listdir(dir))
+                     if re.sub(self._RE_PART, "", name) == re.sub(self._RE_PART, "", file))
 
         return files
 
@@ -189,12 +185,12 @@ class UnRar(Extractor):
         if not self.fullpath and self.VERSION.startswith('5'):
             #@NOTE: Unrar 5 always list full path
             for f in decode(out).splitlines():
-                f = fs_join(self.out, os.path.basename(f.strip()))
+                f = fsjoin(self.dest, os.path.basename(f.strip()))
                 if os.path.isfile(f):
-                    result.add(fs_join(self.out, os.path.basename(f)))
+                    result.add(fsjoin(self.dest, os.path.basename(f)))
         else:
             for f in decode(out).splitlines():
-                result.add(fs_join(self.out, f.strip()))
+                result.add(fsjoin(self.dest, f.strip()))
 
         return list(result)
 
@@ -226,8 +222,8 @@ class UnRar(Extractor):
 
         #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue
         call = [self.CMD, command] + args + list(xargs)
-
         self.log_debug(" ".join(call))
+        call = map(encode, call)
 
         p = subprocess.Popen(call, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py
index ff929ae00..50ab80da3 100644
--- a/module/plugins/internal/UnZip.py
+++ b/module/plugins/internal/UnZip.py
@@ -2,26 +2,30 @@
 
 from __future__ import with_statement
 
-import os
 import sys
 import zipfile
 
 from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError
+from module.plugins.internal.misc import encode
 
 
 class UnZip(Extractor):
     __name__    = "UnZip"
     __type__    = "extractor"
-    __version__ = "1.20"
+    __version__ = "1.21"
     __status__  = "stable"
 
-    __description__ = """Zip extractor plugin"""
+    __description__ = """ZIP extractor plugin"""
     __license__     = "GPLv3"
     __authors__     = [("Walter Purcaro", "vuolter@gmail.com")]
 
 
-    VERSION    = "%s.%s.%s" % (sys.version_info[0], sys.version_info[1], sys.version_info[2])
-    EXTENSIONS = [".zip", ".zip64"]
+    VERSION = "%s.%s.%s" % (sys.version_info[0], sys.version_info[1], sys.version_info[2])
+
+
+    @classmethod
+    def isarchive(cls, filename):
+        return zipfile.is_zipfile(encode(filename))
 
 
     @classmethod
@@ -30,49 +34,35 @@ class UnZip(Extractor):
 
 
     def list(self, password=None):
-        with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z:
+        with zipfile.ZipFile(self.target, 'r') as z:
             z.setpassword(password)
             return z.namelist()
 
 
     def verify(self, password=None):
-        with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z:
-            z.setpassword(password)
-
-            try:
-                badfile = z.testzip()
-
-            except RuntimeError, e:
-                if "encrypted" in e.args[0] or "Bad password" in e.args[0]:
-                    raise PasswordError
-                else:
-                    raise CRCError("Archive damaged")
-
-            else:
-                if badfile:
-                    raise CRCError(badfile)
-
-
-
-    def extract(self, password=None):
         try:
-            with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z:
+            with zipfile.ZipFile(self.target, 'r') as z:
                 z.setpassword(password)
-
-                badfile = z.testzip()
-
-                if badfile:
+                if z.testzip():
                     raise CRCError(badfile)
-                else:
-                    z.extractall(self.out)
 
         except (zipfile.BadZipfile, zipfile.LargeZipFile), e:
             raise ArchiveError(e)
 
         except RuntimeError, e:
             if "encrypted" in e.args[0] or "Bad password" in e.args[0]:
-                raise PasswordError
+                raise PasswordError(e)
             else:
-                raise ArchiveError(e)
-        else:
-            self.files = z.namelist()
+                raise CRCError(e)
+
+
+    def extract(self, password=None):
+        self.verify(password)
+
+        try:
+            with zipfile.ZipFile(self.target, 'r') as z:
+                z.setpassword(password)
+                z.extractall(self.dest)
+
+        except RuntimeError, e:
+            raise ArchiveError(e)