unrar hook :D, account fix, other fixes

6 files changed, 509 insertions, 9 deletions

diff --git a/module/FileDatabase.py b/module/FileDatabase.py
index d45763c05..4757831e7 100644
--- a/module/FileDatabase.py
+++ b/module/FileDatabase.py
@@ -435,7 +435,7 @@ class FileHandler:
         ids = self.db.getUnfinished(pyfile.packageid)
         if not ids or (pyfile.id in ids and len(ids) == 1):
             if not pyfile.package().setFinished:
-                self.core.log.info("Package finished: %s" % pyfile.package().name)
+                self.core.log.info(_("Package finished: %s") % pyfile.package().name)
                 self.core.hookManager.packageFinished(pyfile.package())
                 pyfile.package().setFinished = True
                 
@@ -827,6 +827,9 @@ class PyFile():
         self.active = False #obsolete?
         self.abort = False
         self.reconnected = False
+        
+        #hook progress
+        self.alternativePercent = None
 
         self.m.cache[int(id)] = self
         
@@ -961,6 +964,7 @@ class PyFile():
     
     def getPercent(self):
         """ get % of download """
+        if self.alternativePercent: return self.alternativePercent
         try:
             return int((float(self.plugin.req.dl_arrived)  / self.plugin.req.dl_size) * 100)
         except:
@@ -1021,7 +1025,7 @@ class PyPackage():
 
     def getChildren(self):
         """get information about contained links"""
-        raise NotImplementedError
+        return self.m.getPackageData(self.id)["links"]
     
     def setPriority(self, priority):
         self.priority = priority
diff --git a/module/HookManager.py b/module/HookManager.py
index 41e4c5ef0..3df8a1fc7 100644
--- a/module/HookManager.py
+++ b/module/HookManager.py
@@ -106,7 +106,7 @@ class HookManager():
         for plugin in self.plugins:
             if plugin.isActivated():
                 if "packageFinished" in plugin.__threaded__:
-                    self.startThread(plugin.packageFinished, pyfile)
+                    self.startThread(plugin.packageFinished, package)
                 else:
                     plugin.packageFinished(package)
     
diff --git a/module/PluginThread.py b/module/PluginThread.py
index f6707a908..c666e7875 100644
--- a/module/PluginThread.py
+++ b/module/PluginThread.py
@@ -28,6 +28,7 @@ from module.plugins.Plugin import Fail
 from module.plugins.Plugin import Reconnect
 from module.plugins.Plugin import Retry
 from pycurl import error
+from module.FileDatabase import PyFile
 
 ########################################################################
 class PluginThread(Thread):
@@ -255,8 +256,9 @@ class HookThread(PluginThread):
         self.active = pyfile
 
         m.localThreads.append(self)
-
-        pyfile.setStatus("processing")
+        
+        if isinstance(pyfile, PyFile):
+            pyfile.setStatus("processing")
 
         self.start()
 
@@ -265,8 +267,8 @@ class HookThread(PluginThread):
 
 
         self.m.localThreads.remove(self)
-        self.active.finishIfDone()
-
+        if isinstance(self.active, PyFile):
+            self.active.finishIfDone()
 
 ########################################################################
 class InfoThread(PluginThread):
@@ -303,4 +305,4 @@ class InfoThread(PluginThread):
 
                 self.m.core.log.debug("Finished Info Fetching for %s" % pluginname)
 
-                self.m.core.files.save()
\ No newline at end of file
+                self.m.core.files.save()
diff --git a/module/plugins/Account.py b/module/plugins/Account.py
index bdbbd4c1c..af8b6ebe8 100644
--- a/module/plugins/Account.py
+++ b/module/plugins/Account.py
@@ -40,7 +40,7 @@ class Account():
     
     def setAccounts(self, accounts):
         self.accounts = accounts
-        for user, data in self.accounts:
+        for user, data in self.accounts.iteritems():
             self.login(user, data)
     
     def updateAccounts(self, user, password, options):
diff --git a/module/plugins/hooks/UnRar.py b/module/plugins/hooks/UnRar.py
new file mode 100644
index 000000000..82c99a575
--- /dev/null
+++ b/module/plugins/hooks/UnRar.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+    
+    @author: mkaay
+"""
+
+from __future__ import with_statement
+
+from module.plugins.Hook import Hook
+from module.pyunrar import Unrar, WrongPasswordError, CommandError
+
+from os.path import exists, join
+from os import remove
+import re
+
+class UnRar(Hook):
+    __name__ = "UnRar"
+    __version__ = "0.1"
+    __description__ = """unrar"""
+    __config__ = [ ("activated", "bool", "Activated", True),
+                   ("fullpath", "bool", "extract full path", True),
+                   ("overwrite", "bool", "overwrite files", True),
+                   ("passwordfile", "str", "unrar passoword file", "unrar_passwords.txt"),
+                   ("deletearchive", "bool", "delete archives when done", False) ]
+    __threaded__ = ["packageFinished"]
+    __author_name__ = ("mkaay")
+    __author_mail__ = ("mkaay@mkaay.de")
+    
+    def setup(self):
+        self.comments = ["# one password each line"]
+        self.passwords = []
+        if exists(self.getConfig("passwordfile")):
+            with open(self.getConfig("passwordfile"), "r") as f:
+                for l in f.readlines():
+                    l = l.strip("\n\r")
+                    if l and not l.startswith("#"):
+                        self.passwords.append(l)
+        else:
+            with open(self.getConfig("passwordfile"), "w") as f:
+                f.writelines(self.comments)
+        self.re_splitfile = re.compile("(.*)\.part(\d+)\.rar$")
+    
+    def addPassword(self, pw):
+        if not pw in self.passwords:
+            self.passwords.insert(0, pw)
+            with open(self.getConfig("passwordfile"), "w") as f:
+                f.writelines(self.comments)
+                f.writelines(self.passwords)
+        
+    def removeFiles(self, pack, fname):
+        if not self.getConfig("deletearchive"):
+            return
+        m = self.re_splitfile.search(fname)
+                
+        download_folder = self.core.config['general']['download_folder']
+        if self.core.config['general']['folder_per_package']:
+            folder = join(download_folder, pack.folder.decode(sys.getfilesystemencoding()))
+        else:
+            folder = download_folder
+        if m:
+            nre = re.compile("%s\.part\d+\.rar" % m.group(1))
+            for fid, data in pack.getChildren().iteritems():
+                if nre.match(data["name"]):
+                    remove(join(folder, data["name"]))
+        elif not m and fname.endswith(".rar"):
+            nre = re.compile("^%s\.r..$" % fname.replace(".rar",""))
+            for fid, data in pack.getChildren().iteritems():
+                if nre.match(data["name"]):
+                    remove(join(folder, data["name"]))
+    
+    def packageFinished(self, pack):
+        if pack.password:
+            self.addPassword(pack.password)
+        files = []
+        for fid, data in pack.getChildren().iteritems():
+            m = self.re_splitfile.search(data["name"])
+            if m and int(m.group(2)) == 1:
+                files.append((fid,m.group(0)))
+            elif not m and data["name"].endswith(".rar"):
+                files.append((fid,data["name"]))
+        
+        for fid, fname in files:
+            pyfile = self.core.files.getFile(fid)
+            pyfile.setStatus("custom")
+            def s(p):
+                pyfile.alternativePercent = p
+                
+            download_folder = self.core.config['general']['download_folder']
+            if self.core.config['general']['folder_per_package']:
+                folder = join(download_folder, pack.folder.decode(sys.getfilesystemencoding()))
+            else:
+                folder = download_folder
+            
+            u = Unrar(join(folder, fname))
+            try:
+                u.crackPassword(passwords=self.passwords, statusFunction=s, overwrite=True, destination=folder)
+            except WrongPasswordError:
+                continue
+            except CommandError as e:
+                if re.search("Cannot find volume", e.stderr):
+                    continue
+                try:
+                    if e.getExitCode() == 1 and len(u.listContent(u.getPassword())) == 1:
+                        self.removeFiles(pack, fname)
+                except:
+                    continue
+            else:
+                self.removeFiles(pack, fname)
+            finally:
+                pyfile.alternativePercent = None
+
diff --git a/module/pyunrar.py b/module/pyunrar.py
new file mode 100644
index 000000000..4db3c3a19
--- /dev/null
+++ b/module/pyunrar.py
@@ -0,0 +1,370 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+    
+    @author: mkaay <mkaay@mkaay.de>
+"""
+
+from subprocess import Popen, PIPE
+import re
+from time import sleep
+from tempfile import mkdtemp
+from shutil import rmtree, move
+from shutil import Error as FileError
+from os.path import join, abspath, basename, dirname
+from os import remove, makedirs
+
+EXITMAP = {
+    255: ("USER BREAK 	User stopped the process"),
+    9: ("CREATE ERROR", "Create file error"),
+    8: ("MEMORY ERROR", "Not enough memory for operation"),
+    7: ("USER ERROR", "Command line option error"),
+    6: ("OPEN ERROR", "Open file error"),
+    5: ("WRITE ERROR", "Write to disk error"),
+    4: ("LOCKED ARCHIVE", "Attempt to modify an archive previously locked by the 'k' command"),
+    3: ("CRC ERROR", "A CRC error occurred when unpacking"),
+    2: ("FATAL ERROR", "A fatal error occurred"),
+    1: ("WARNING", "Non fatal error(s) occurred"),
+    0: ("SUCCESS", "Successful operation (User exit)"),
+}
+
+class UnknownError(Exception):
+    pass
+
+class NoFilesError(Exception):
+    pass
+
+class WrongPasswordError(Exception):
+    pass
+
+class CommandError(Exception):
+    def __init__(self, ret=None, stdout=None, stderr=None):
+        self.ret = ret
+        self.stdout = stdout
+        self.stderr = stderr
+    
+    def __str__(self):
+        return "%s %s %s" % (EXITMAP[self.ret], self.stdout, self.stderr)
+    
+    def __repr__(self):
+        try:
+            return "<CommandError %s (%s)>" % (EXITMAP[self.ret][0], EXITMAP[self.ret][1])
+        except:
+            return "<CommandError>"
+    
+    def getExitCode(self):
+        return self.ret
+    
+    def getMappedExitCode(self):
+        return EXITMAP[self.ret]
+
+class Unrar():
+    def __init__(self, archive):
+        """
+            archive should be be first or only part
+        """
+        self.archive = archive
+        self.pattern = None
+        m = re.match("^(.*).part(\d+).rar$", archive)
+        if m:
+            self.pattern = "%s.part*.rar" % m.group(1)
+        else: #old style
+            self.pattern = "%s.r*" % archive.replace(".rar", "")
+        self.cmd = "unrar"
+        self.encrypted = None
+        self.headerEncrypted = None
+        self.smallestFiles = None
+        self.password = None
+    
+    def listContent(self, password=None):
+        """
+            returns a list with all infos to the files in the archive
+            dict keys: name, version, method, crc, attributes, time, date, ratio, size_packed, size
+            @return list(dict, dict, ...)
+        """
+        f = self.archive
+        if self.pattern:
+            f = self.pattern
+        args = [self.cmd, "v"]
+        if password:
+            args.append("-p%s" % password)
+        else:
+            args.append("-p-")
+        args.append(f)
+        p = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        ret = p.wait()
+        if ret == 3:
+            self.headerEncrypted = True
+            raise WrongPasswordError()
+        elif ret == 0 and password:
+            self.headerEncrypted = False
+        o = p.stdout.read()
+        inList = False
+        infos = {}
+        nameLine = False
+        name = ""
+        for line in o.split("\n"):
+            if line == "-"*79:
+                inList = not inList
+                continue
+            if inList:
+                nameLine = not nameLine
+                if nameLine:
+                    name = line
+                    if name[0] == "*": #check for pw indicator
+                        name = name[1:]
+                        self.encrypted = True
+                    name = name.strip()
+                    continue
+                s = line.split(" ")
+                s = [e for e in s if e]
+                s.reverse()
+                d = {}
+                for k, v in zip(["version", "method", "crc", "attributes", "time", "date", "ratio", "size_packed", "size"], s[0:9]):
+                    d[k] = v
+                #if d["crc"] == "00000000" and len(d["method"]) == 2:
+                if re.search("d", d["attributes"].lower()): #directory
+                    continue
+                d["name"] = name
+                d["size_packed"] = int(d["size_packed"])
+                d["size"] = int(d["size"])
+                if infos.has_key(name):
+                    infos[name]["size_packed"] = infos[name]["size_packed"] + d["size_packed"]
+                    infos[name]["crc"].append(d["crc"])
+                else:
+                    infos[name] = d
+                    infos[name]["crc"] = [d["crc"]]
+        infos = infos.values()
+        return infos
+    
+    def listSimple(self, password=None):
+        """
+            return a list with full path to all files
+            @return list
+        """
+        l = self.listContent(password=password)
+        return [e["name"] for e in l]
+    
+    def getSmallestFile(self, password=None):
+        """
+            return the file info for the smallest file
+            @return dict
+        """
+        files = self.listContent(password=password)
+        smallest = (-1, -1)
+        for i, f in enumerate(files):
+            if f["size"] < smallest[1] or smallest[1] == -1:
+                smallest = (i, f["size"])
+        if smallest[0] == -1:
+            raise UnknownError()
+        self.smallestFiles = files[smallest[0]]
+        return files[smallest[0]]
+    
+    def needPassword(self):
+        """
+            do we need a password?
+            @return bool
+        """
+        if not self.smallestFiles:
+            try:
+                self.getSmallestFile()
+            except WrongPasswordError:
+                return True
+        return self.headerEncrypted or self.encrypted
+    
+    def checkPassword(self, password, statusFunction=None):
+        """
+            check if password is okay
+            @return bool
+        """
+        if not self.needPassword():
+            return True
+        f = self.archive
+        if self.pattern:
+            f = self.pattern
+        args = [self.cmd, "t", "-p%s" % password, f]
+        try:
+            args.append(self.getSmallestFile(password)["name"])
+        except WrongPasswordError:
+            return False
+        p = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        (ret, out) = self.processOutput(p, statusFunction)
+        if ret == 3:
+            raise False
+        elif ret == 0:
+            return True
+        else:
+            raise UnknownError()
+    
+    def extract(self, password=None, fullPath=True, files=[], exclude=[], destination=None, overwrite=False, statusFunction=None):
+        """
+            extract the archive
+            @return bool: extract okay?
+            raises WrongPasswordError or CommandError
+        """
+        f = self.archive
+        if self.pattern:
+            f = self.pattern
+        args = [self.cmd]
+        if fullPath:
+            args.append("x")
+        else:
+            args.append("e")
+        if not password:
+            password = "-"
+        if overwrite:
+            args.append("-o+")
+        else:
+            args.append("-o-")
+        args.append("-p%s" % password)
+        args.append(f)
+        if files:
+            args.extend([e for e in files])
+        if exclude:
+            args.extend(["-x%s" % e for e in exclude])
+        if destination:
+            args.append(destination)
+        p = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        (ret, out) = self.processOutput(p, statusFunction)
+        if ret == 3:
+            raise WrongPasswordError()
+        elif ret == 0:
+            return True
+        else:
+            raise CommandError(ret=ret, stdout=out, stderr=p.stderr.read())
+    
+    def crackPassword(self, passwords=[], fullPath=True, destination=None, overwrite=False, statusFunction=None, exclude=[]):
+        """
+            check password list until the right one is found and extract the archive
+            @return bool: password found?
+        """
+        correctPassword = None
+        if self.needPassword():
+            for password in passwords:
+                sf = []
+                try:
+                    sf.append(self.getSmallestFile(password)["name"])
+                except WrongPasswordError:
+                    continue
+                tdir = mkdtemp(prefix="rar")
+                try:
+                    self.extract(password=password, fullPath=fullPath, destination=tdir, overwrite=overwrite, statusFunction=statusFunction, files=sf)
+                except WrongPasswordError:
+                    continue
+                else:
+                    if not destination:
+                        destination = "."
+                    if overwrite:
+                        try:
+                            remove(abspath(join(destination, sf[0])))
+                        except OSError as e:
+                            if not e.errno == 2:
+                                raise e
+                    f = sf[0]
+                    d = destination
+                    if fullPath:
+                        try:
+                            makedirs(dirname(join(abspath(destination), sf[0])))
+                        except OSError as e:
+                            if not e.errno == 17:
+                                raise e
+                        d = join(destination, dirname(f))
+                    else:
+                        f = basename(f)
+                    try:
+                        move(join(tdir, f), abspath(d))
+                    except FileError:
+                        pass
+                    exclude.append(sf[0])
+                    correctPassword = password
+                    break
+                finally:
+                    rmtree(tdir)
+                    pass
+        try:
+            self.extract(password=correctPassword, fullPath=fullPath, destination=destination, overwrite=overwrite, statusFunction=statusFunction, exclude=exclude)
+            self.password = correctPassword
+            return True
+        except WrongPasswordError:
+            return False
+    
+    def processOutput(self, p, statusFunction=None):
+        """
+            internal method
+            parse the progress output of the rar/unrar command
+            @return int: exitcode
+                    string: command output
+        """
+        ret = None
+        out = ""
+        tmp = None
+        count = 0
+        perc = 0
+        tperc = "0"
+        last = None
+        digits = "1 2 3 4 5 6 7 8 9 0".split(" ")
+        if not statusFunction:
+            statusFunction = lambda p: None
+        statusFunction(0)
+        while ret == None or tmp:
+            tmp = p.stdout.read(1)
+            if tmp:
+                out += tmp
+                if tmp == chr(8):
+                    if last == tmp:
+                        count += 1
+                        tperc = "0"
+                    else:
+                        count = 0
+                        if perc < int(tperc):
+                            perc = int(tperc)
+                            statusFunction(perc)
+                elif count >= 3:
+                    if tmp == "\n":
+                        count = 0
+                    elif tmp in digits:
+                        tperc += tmp
+                last = tmp
+            else:
+                sleep(0.01)
+            ret = p.poll()
+        statusFunction(100)
+        return ret, out
+    
+    def getPassword(self):
+        """
+            return the correct password
+            works only in conjunction with 'crackPassword'
+            @return string: password
+        """
+        return self.password
+
+if __name__ == "__main__":
+    from pprint import pprint
+    u = Unrar("archive.part1.rar", multi=True)
+    u = Unrar("parchive.part1.rar", multi=True)
+    pprint(u.listContent())
+    u = Unrar("pharchive.part1.rar", multi=True)
+    pprint(u.listContent(password="test"))
+    u = Unrar("bigarchive.rar")
+    pprint(u.listContent())
+    print u.getSmallestFile()
+    try:
+        def s(p):
+            print p
+        print u.crackPassword(passwords=["test1", "ggfd", "423r", "test"], destination=".", statusFunction=s, overwrite=True)
+    except CommandError as e:
+        print e