first version of new extract plugin

5 files changed, 513 insertions, 139 deletions

diff --git a/module/PluginThread.py b/module/PluginThread.py
index aeb2ac2aa..37176574e 100644
--- a/module/PluginThread.py
+++ b/module/PluginThread.py
@@ -443,7 +443,8 @@ class HookThread(PluginThread):
 
     def addActive(self, pyfile):
         """ Adds a pyfile to active list and thus will be displayed on overview"""
-        self.active.append(pyfile)
+        if pyfile not in self.active:
+            self.active.append(pyfile)
 
     def finishFile(self, pyfile):
         if pyfile in self.active:
@@ -456,7 +457,12 @@ class HookThread(PluginThread):
             try:
                 self.kwargs["thread"] = self
                 self.f(*self.args, **self.kwargs)
-            except TypeError:
+            except TypeError, e:
+                #dirty method to filter out exceptions
+                if "unexpected keyword argument 'thread'" not in e.message:
+                    print_exc()
+                    raise e
+                
                 del self.kwargs["thread"]
                 self.f(*self.args, **self.kwargs)
         finally:
diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py
index 0f2abdcf9..814a70949 100644
--- a/module/plugins/Hoster.py
+++ b/module/plugins/Hoster.py
@@ -19,6 +19,10 @@
 
 from module.plugins.Plugin import Plugin
 
+def getInfo(self):
+        #result = [ .. (name, size, status, url) .. ]
+        return
+
 class Hoster(Plugin):
     __name__ = "Hoster"
     __version__ = "0.1"
@@ -27,8 +31,3 @@ class Hoster(Plugin):
     __description__ = """Base hoster plugin"""
     __author_name__ = ("mkaay")
     __author_mail__ = ("mkaay@mkaay.de")
-    
-    def getInfo(self):
-        #result = [ .. (name, size, status, url) .. ]
-        return
-        
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index 7c9e7c15c..e503cd6ed 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -82,16 +82,16 @@ class Base(object):
 
     #log functions
     def logInfo(self, *args):
-        self.log.info("%s: %s" % (self.__name__, " | ".join(args)))
+        self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
 
     def logWarning(self, *args):
-        self.log.warning("%s: %s" % (self.__name__, " | ".join(args)))
+        self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
 
     def logError(self, *args):
-        self.log.error("%s: %s" % (self.__name__, " | ".join(args)))
+        self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
 
     def logDebug(self, *args):
-        self.log.debug("%s: %s" % (self.__name__, "| ".join(args)))
+        self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
 
 
     def setConf(self, option, value):
@@ -313,7 +313,7 @@ class Plugin(Base):
         :param wait_time: time to wait in seconds
         :param reason: reason for retrying, will be passed to fail if max_tries reached
         """
-        if max_tries > 0 and self.retries >= max_tries:
+        if 0 < max_tries <= self.retries:
             if not reason: reason = "Max retries reached"
             raise Fail(reason)
 
diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py
new file mode 100644
index 000000000..90ee2298d
--- /dev/null
+++ b/module/plugins/hooks/ExtractArchive.py
@@ -0,0 +1,344 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+from os import remove, chmod
+from os.path import exists, basename, isfile, isdir
+from traceback import print_exc
+from copy import copy
+
+if os.name != "nt":
+    from os import chown
+    from pwd import getpwnam
+    from grp import getgrnam
+
+from module.plugins.Hook import Hook, threaded, Expose
+from module.utils import save_join
+
+
+class ArchiveError(Exception):
+    pass
+
+
+class CRCError(Exception):
+    pass
+
+
+class WrongPassword(Exception):
+    pass
+
+
+class ExtractArchive(Hook):
+    __name__ = "ExtractArchive"
+    __version__ = "0.1"
+    __description__ = "Extract different kind of archives"
+    __config__ = [("activated", "bool", "Activated", True),
+        ("fullpath", "bool", "Extract full path", True),
+        ("overwrite", "bool", "Overwrite files", True),
+        ("passwordfile", "file", "password file", "unrar_passwords.txt"),
+        ("deletearchive", "bool", "Delete archives when done", False),
+        ("destination", "folder", "Extract files to", ""),
+        ("queue", "bool", "Wait for all downloads to be fninished", True),
+        ("renice", "int", "CPU Priority", 0), ]
+    __author_name__ = ("pyload Team")
+    __author_mail__ = ("admin<at>pyload.org")
+
+    event_list = ["allDownloadsProcessed"]
+
+    def setup(self):
+        self.plugins = []
+        names = []
+
+        for p in ("UnRar",):
+            try:
+                module = self.core.pluginManager.getInternalModule(p)
+                klass = getattr(module, p)
+                if klass.checkDeps():
+                    names.append(p)
+                    self.plugins.append(klass)
+
+            except Exception, e:
+                self.logWarning(_("Could not activate %s") % p, str(e))
+                if self.core.debug:
+                    print_exc()
+
+        if names:
+            self.logInfo(_("Activated") + " " + " ".join(names))
+        else:
+            self.logInfo(_("No Extract plugins activated"))
+
+        # queue with package ids
+        self.queue = []
+
+    @Expose
+    def extractPackage(self, id):
+        """ Extract package with given id"""
+        self.manager.startThread(self.extract, [id])
+
+    def packageFinished(self, pypack):
+        if self.getConfig("queue"):
+            self.logInfo(_("Package %s queued for later extracting") % pypack.name)
+            self.queue.append(pypack.id)
+        else:
+            self.manager.startThread(self.extract, [pypack.id])
+
+
+    @threaded
+    def allDownloadsProcessed(self, thread):
+        local = copy(self.queue)
+        del self.queue[:]
+        self.extract(local, thread)
+
+
+    def extract(self, ids, thread=None):
+        # dl folder
+        dl = self.config['general']['download_folder']
+
+        extracted = []
+
+        #iterate packages -> plugins -> targets
+        for pid in ids:
+            p = self.core.files.getPackage(pid)
+            if not p: continue
+
+            # determine output folder
+            out = save_join(dl, p.folder, "")
+            # force trailing slash
+
+            if self.getConfig("destination") and self.getConfig("destination").lower() != "none":
+                if exists(self.getConfig("destination")):
+                    out = save_join(self.getConfig("destination"), "")
+
+            files_ids = [(save_join(dl, p.folder, x["name"]), x["id"]) for x in p.getChildren().itervalues()]
+
+            # check as long there are unseen files
+            while files_ids:
+                new_files_ids = []
+
+                for plugin in self.plugins:
+                    targets = plugin.getTargets(files_ids)
+                    self.logDebug("Targets: %s" % targets)
+                    for target, fid in targets:
+                        if target in extracted:
+                            self.logDebug(basename(target), "skipped")
+                            continue
+                        extracted.append(target) #prevent extracting same file twice
+
+                        klass = plugin(self, target, out, self.getConfig("fullpath"), self.getConfig("overwrite"),
+                            self.getConfig("renice"))
+                        klass.init()
+
+                        self.logInfo(basename(target), _("Extract to %s") % out)
+                        new_files = self.startExtracting(klass, fid, p.password.strip().splitlines(), thread)
+                        self.logDebug("Extracted: %s" % new_files)
+                        self.setPermissions(new_files)
+
+                        for file in new_files:
+                            if not exists(file):
+                                self.logDebug("new file %s does not exists" % file)
+                                continue
+                            if isfile(file):
+                                new_files_ids.append((file, fid)) #append as new target
+
+                files_ids = new_files_ids # also check extracted files
+
+
+    def startExtracting(self, plugin, fid, passwords, thread):
+        pyfile = self.core.files.getFile(fid)
+        if not pyfile: return []
+
+        pyfile.setCustomStatus(_("extracting"))
+        thread.addActive(pyfile) #keep this file until everything is done
+
+        try:
+            progress = lambda x: pyfile.setProgress(x)
+            success = False
+
+            if not plugin.checkArchive():
+                plugin.extract(progress)
+                success = True
+            else:
+                self.logInfo(basename(plugin.file), _("Password protected"))
+                self.logDebug("Passwords: %s" % str(passwords))
+                for pw in passwords + self.getPasswords():
+                    try:
+                        if plugin.checkPassword(pw):
+                            plugin.extract(progress, pw)
+                            self.addPassword(pw)
+                            success = True
+                            break
+                    except WrongPassword:
+                        self.logDebug("Tried wrong password %s" % pw)
+
+            if not success:
+                self.logError(basename(plugin.file), _("Wrong password"))
+                return []
+
+            if self.core.debug:
+                self.logDebug("Would delete: %s" % ",".join(plugin.getDeleteFiles()))
+
+            if self.getConfig("deletearchive"):
+                files = plugin.getDeleteFiles()
+                self.logInfo(_("Deleting %s files") % len(files))
+                for f in files:
+                    if exists(f): remove(f)
+                    else: self.logDebug("%s does not exists" % f)
+
+            self.logInfo(basename(plugin.file), _("Extracting finished"))
+            self.core.hookManager.unrarFinished(plugin.out, plugin.file)
+
+            return plugin.getExtractedFiles()
+
+
+        except ArchiveError, e:
+            self.logError(basename(plugin.file), _("Archive Error"), str(e))
+        except CRCError:
+            self.logError(basename(plugin.file), _("CRC Mismatch"))
+        except Exception, e:
+            if self.core.debug:
+                print_exc()
+            self.logError(basename(plugin.file), _("Unkown Error"), str(e))
+
+        return []
+
+    @Expose
+    def getPasswords(self):
+        """ List of saved passwords """
+        pwfile = self.getConfig("passwordfile")
+        if not exists(pwfile):
+            open(pwfile, "wb").close()
+
+        passwords = []
+        f = open(pwfile, "rb")
+        for pw in f.readline():
+            passwords.append(pw)
+        f.close()
+
+        return passwords
+
+    @Expose
+    def addPassword(self, pw):
+        """  Adds a password to saved list"""
+        pwfile = self.getConfig("passwordfile")
+
+        passwords = []
+        f = open(pwfile, "rb")
+        for pw in f.readline():
+            passwords.append(pw)
+        f.close()
+
+        if pw in passwords: passwords.remove(pw)
+        passwords.insert(0, pw)
+
+        f = open(pwfile, "wb")
+        f.writelines(passwords)
+        f.close()
+
+    def setPermissions(self, files):
+        for f in files:
+            if not exists(f): continue
+            try:
+                if self.core.config["permission"]["change_file"]:
+                    if isfile(f):
+                        chmod(f, int(self.core.config["permission"]["file"], 8))
+                    elif isdir(f):
+                        chmod(f, int(self.core.config["permission"]["folder"], 8))
+
+                if self.core.config["permission"]["change_dl"] and os.name != "nt":
+                    uid = getpwnam(self.config["permission"]["user"])[2]
+                    gid = getgrnam(self.config["permission"]["group"])[2]
+                    chown(f, uid, gid)
+            except Exception, e:
+                self.log.warning(_("Setting User and Group failed"), e)
+
+    def archiveError(self, msg):
+        raise ArchiveError(msg)
+
+    def wrongPassword(self):
+        raise WrongPassword()
+
+    def crcError(self):
+        raise CRCError()
+
+
+class AbtractExtractor:
+    @staticmethod
+    def checkDeps():
+        """ Check if system statisfy dependencies
+        :return: boolean
+        """
+        return True
+
+    @staticmethod
+    def getTargets(files_ids):
+        """ Filter suited targets from list of filename id tuple list
+        :param files_ids: List of filepathes
+        :return: List of targets, id tuple list
+        """
+        raise NotImplementedError
+
+
+    def __init__(self, m, file, out, fullpath, overwrite, renice):
+        """Initialize extractor for specifiy file
+
+        :param m: ExtractArchive Hook plugin
+        :param file: Absolute filepath
+        :param out: Absolute path to destination directory
+        :param fullpath: extract to fullpath
+        :param overwrite: Overwrite existing archives
+        :param renice: Renice value
+        """
+        self.m = m
+        self.file = file
+        self.out = out
+        self.fullpath = fullpath
+        self.overwrite = overwrite
+        self.renice = renice
+        self.files = [] # Store extracted files here
+
+
+    def init(self):
+        """ Initialize additional data structures """
+        pass
+
+
+    def checkArchive(self):
+        """Check if password if needed. Raise ArchiveError if integrity is
+        questionable.
+
+        :return: boolean
+        :raises ArchiveError
+        """
+        return False
+
+    def checkPassword(self, password):
+        """ Check if the given password is/might be correct.
+        If it can not be decided at this point return true.
+
+        :param password:
+        :return: boolean
+        """
+        return True
+
+    def extract(self, progress, password=None):
+        """Extract the archive. Raise specific errors in case of failure.
+
+        :param progress: Progress function, call this to update status
+        :param password password to use
+        :raises WrongPassword
+        :raises CRCError
+        :raises ArchiveError
+        :return:
+        """
+        raise NotImplementedError
+
+    def getDeleteFiles(self):
+        """Return list of files to delete, do *not* delete them here.
+
+        :return: List with paths of files to delete
+        """
+        raise NotImplementedError
+
+    def getExtractedFiles(self):
+        """Populate self.files at some point while extracting"""
+        return self.files
\ No newline at end of file
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
index d22e54b61..67f95b018 100644
--- a/module/plugins/internal/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -14,145 +14,170 @@
     You should have received a copy of the GNU General Public License
     along with this program; if not, see <http://www.gnu.org/licenses/>.
     
-    @author: mkaay
+    @author: RaNaN
 """
-from __future__ import with_statement
 
-import sys
 import os
-from os.path import exists, join, isabs, isdir
-from os import remove, makedirs, rmdir, listdir, chmod
-from traceback import print_exc
+from os.path import join
 
-from module.plugins.hooks.ExtractArchive import AbtractExtractor
-from module.lib.pyunrar import Unrar, WrongPasswordError, CommandError, UnknownError, LowRamError
+from subprocess import Popen, PIPE
 
+from module.plugins.hooks.ExtractArchive import AbtractExtractor
 from module.utils import save_join
 
-
+try:
+    import pexpect #used for progress
+    PEXPECT = True
+except ImportError:
+    PEXPECT = False
 
 import re
 
 class UnRar(AbtractExtractor):
+    __name__ = "UnRar"
+    __version__ = "0.1"
 
-    def removeFiles(self, pack, fname):
-        if not self.getConfig("deletearchive"):
-            return
-        m = self.re_splitfile.search(fname)
+    re_splitfile = re.compile("(.*)\.part(\d+)\.rar$")
 
-        download_folder = self.core.config['general']['download_folder']
-        if self.core.config['general']['folder_per_package']:
-            folder = join(download_folder, pack.folder.decode(sys.getfilesystemencoding()))
-        else:
-            folder = download_folder
-        if m:
-            nre = re.compile("%s\.part\d+\.rar" % m.group(1))
-            for fid, data in pack.getChildren().iteritems():
-                if nre.match(data["name"]):
-                    remove(join(folder, data["name"]))
-        elif not m and fname.endswith(".rar"):
-            nre = re.compile("^%s\.r..$" % fname.replace(".rar", ""))
-            for fid, data in pack.getChildren().iteritems():
-                if nre.match(data["name"]):
-                    remove(join(folder, data["name"]))
-
-    def packageFinished(self, pack):
-        if pack.password and pack.password.strip() and pack.password.strip() != "None":
-            self.addPassword(pack.password.splitlines())
-        files = []
-
-        for fid, data in pack.getChildren().iteritems():
-            m = self.re_splitfile.search(data["name"])
-            if m and int(m.group(2)) == 1:
-                files.append((fid, m.group(0)))
-            elif not m and data["name"].endswith(".rar"):
-                files.append((fid, data["name"]))
-
-        for fid, fname in files:
-            self.core.log.info(_("starting Unrar of %s") % fname)
-            pyfile = self.core.files.getFile(fid)
-            pyfile.setStatus("processing")
-
-            def s(p):
-                pyfile.setProgress(p)
-
-            download_folder = self.core.config['general']['download_folder']
-            self.core.log.debug(_("download folder %s") % download_folder)
-
-            folder = save_join(download_folder, pack.folder, "")
-
-
-            destination = folder
-            if self.getConfig("unrar_destination") and not self.getConfig("unrar_destination").lower() == "none":
-                destination = self.getConfig("unrar_destination")
-                sub = "."
-                if self.core.config['general']['folder_per_package']:
-                    sub = pack.folder.decode(sys.getfilesystemencoding())
-                if isabs(destination):
-                    destination = join(destination, sub, "")
-                else:
-                    destination = join(folder, destination, sub, "")
-
-            self.core.log.debug(_("Destination folder %s") % destination)
-            if not exists(destination):
-                self.core.log.info(_("Creating destination folder %s") % destination)
-                makedirs(destination)
-
-            u = Unrar(join(folder, fname), tmpdir=join(folder, "tmp"),
-                      ramSize=(self.ram if self.getConfig("ramwarning") else 0), cpu=self.getConfig("renice"))
-            try:
-                success = u.crackPassword(passwords=self.passwords, statusFunction=s, overwrite=True,
-                                          destination=destination, fullPath=self.getConfig("fullpath"))
-            except WrongPasswordError:
-                self.core.log.info(_("Unrar of %s failed (wrong password)") % fname)
-                continue
-            except CommandError, e:
-                if self.core.debug:
-                    print_exc()
-                if re.search("Cannot find volume", e.stderr):
-                    self.core.log.info(_("Unrar of %s failed (missing volume)") % fname)
-                    continue
-                try:
-                    if e.getExitCode() == 1 and len(u.listContent(u.getPassword())) == 1:
-                        self.core.log.info(_("Unrar of %s ok") % fname)
-                        self.removeFiles(pack, fname)
-                except:
-                    if self.core.debug:
-                        print_exc()
-                    self.core.log.info(_("Unrar of %s failed") % fname)
-                    continue
-            except LowRamError:
-                self.log.warning(_(
-                    "Your ram amount of %s MB seems not sufficient to unrar this file. You can deactivate this warning and risk instability") % self.ram)
-                continue
-            except UnknownError:
-                if self.core.debug:
-                    print_exc()
-                self.core.log.info(_("Unrar of %s failed") % fname)
-                continue
+    @staticmethod
+    def checkDeps():
+        return True #TODO
+
+    @staticmethod
+    def getTargets(files_ids):
+        result = []
+
+        for file, id in files_ids:
+            if not file.endswith(".rar"): continue
+
+            match = UnRar.re_splitfile.findall(file)
+            if match:
+                #only add first parts
+                if int(match[0][1]) == 1:
+                    result.append((file, id))
             else:
-                if success:
-                    self.core.log.info(_("Unrar of %s ok") % fname)
-                    self.removeFiles(pack, fname)
-
-                    if os.name != "nt" and self.core.config['permission']['change_dl'] and\
-                       self.core.config['permission']['change_file']:
-                        ownerUser = self.core.config['permission']['user']
-                        fileMode = int(self.core.config['permission']['file'], 8)
-
-                        self.core.log.debug("Setting destination file/directory owner / mode to %s / %s"
-                        % (ownerUser, fileMode))
-
-                        uinfo = getpwnam(ownerUser)
-                        self.core.log.debug("Uid/Gid is %s/%s." % (uinfo.pw_uid, uinfo.pw_gid))
-                        self.setOwner(destination, uinfo.pw_uid, uinfo.pw_gid, fileMode)
-                        self.core.log.debug("The owner/rights have been successfully changed.")
-
-                    self.core.hookManager.unrarFinished(folder, fname)
-                else:
-                    self.core.log.info(_("Unrar of %s failed (wrong password or bad parts)") % fname)
-            finally:
-                pyfile.setProgress(100)
-                pyfile.setStatus("finished")
-                pyfile.release()
+                result.append((file, id))
+
+        return result
+
+
+    def init(self):
+        self.passwordProtected = False
+        self.headerProtected = False  #list files will not work without password
+        self.smallestFile = None  #small file to test passwords
+        self.password = ""  #save the correct password
+
+    def checkArchive(self):
+        p = self.call_unrar("l", "-v", self.file)
+        out, err = p.communicate()
+        if "Corrupt file or wrong password." in err:
+            self.passwordProtected = True
+            self.headerProtected = True
+            return True
+
+        self.listContent()
+        if not self.files:
+            self.m.archiveError("Empty Archive")
+
+        return False
+
+    def checkPassword(self, password):
+        if not self.passwordProtected: return True
+
+        if self.headerProtected:
+            p = self.call_unrar("l", "-v", self.file, password=password)
+            out, err = p.communicate()
+            if "Corrupt file or wrong password." in err:
+                return False
+
+        return True
+
+
+    def extract(self, progress, password=None):
+        command = "x" if self.fullpath else "e"
+
+        if PEXPECT:
+            p = self.call_unrar(command, self.file, self.out, password=password, pexpect=True)
+            #renice(p.pid, self.renice)
+
+            cpl = p.compile_pattern_list([pexpect.EOF, "(\d+)\s?%"])
+            while True:
+                i = p.expect_list(cpl, timeout=None)
+                if i == 0: # EOF
+                    break #exited
+                elif i == 1:
+                    perc =  p.match.group(1)
+                    progress(int(perc))
+            # pexpect thinks process is still alive (just like popen) - very strange behavior
+            p.terminated = True
+
+        else:
+            #subprocess - no progress
+            p = self.call_unrar(command, self.file, self.out, password=password)
+            renice(p.pid, self.renice)
+
+            progress(0)
+            out, err = p.communicate() #wait for process
+            progress(100)
+
+        #TODO, check for errors
+
+
+    def getDeleteFiles(self):
+        #TODO
+        return []
+
+    def listContent(self):
+        command = "vb" if self.fullpath else "lb"
+        p = self.call_unrar(command, "-v", self.file, password=self.password)
+        out, err = p.communicate()
+
+        result = set()
+
+        for f in out.splitlines():
+            f = f.strip()
+            result.add(save_join(self.out, f))
+
+        self.files = result
+
+
+    def call_unrar(self, command, *xargs, **kwargs):
+        if os.name == "nt":
+            cmd = join(pypath, "UnRAR.exe")
+        else:
+            cmd = "unrar"
+
+        args = []
+
+        #overwrite flag
+        args.append("-o+") if self.overwrite else args.append("-o-")
+
+        # assume yes on all queries
+        args.append("-y")
+
+        #set a password
+        if "password" in kwargs and kwargs["password"]:
+            args.append("-p%s" % kwargs["password"])
+        else:
+            args.append("-p-")
+
+
+        #NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue
+        call = [cmd, command] + args + list(xargs)
+        self.m.logDebug(" ".join(call))
+
+        if PEXPECT and "pexpect" in kwargs:
+            #use pexpect if available
+            p = pexpect.spawn(" ".join(call))
+        else:
+            p = Popen(call, stdout=PIPE, stderr=PIPE)
+
+        return p
+
 
+def renice(pid, value):
+    if os.name != "nt" and value:
+        try:
+            Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1)
+        except:
+            print "Renice failed"
\ No newline at end of file