fix in unrar plugin

6 files changed, 135 insertions, 68 deletions

diff --git a/module/Api.py b/module/Api.py
index be72ee359..221809dc2 100644
--- a/module/Api.py
+++ b/module/Api.py
@@ -240,7 +240,7 @@ class Api(Iface):
 
         return data
 
-    def addPackage(self, name, links, dest):
+    def addPackage(self, name, links, dest=Destination.Queue):
         """Adds a package, with links to desired destination.
 
         :param name: name of the new package
@@ -343,6 +343,19 @@ class Api(Iface):
         return [self.addPackage(name, urls, dest) for name, urls
                 in self.generatePackages(links).iteritems()]
 
+    def checkAndAddPackages(self, links, dest=Destination.Queue):
+        """Checks online status, retrieves names, and will add packages.\
+        Because of this packages are not added immediatly, only for internal use.
+
+        :param links: list of urls
+        :param dest: `Destination`
+        :return: None
+        """
+        data = self.core.pluginManager.parseUrls(urls)
+        self.core.threadManager.createResultThread(data, True)
+
+        
+
     def getPackageData(self, pid):
         """Returns complete information about package, and included files.
 
diff --git a/module/PluginThread.py b/module/PluginThread.py
index a036b4878..0a8151a25 100644
--- a/module/PluginThread.py
+++ b/module/PluginThread.py
@@ -407,7 +407,7 @@ class HookThread(PluginThread):
 
 
 class InfoThread(PluginThread):
-    def __init__(self, manager, data, pid=-1, rid=-1):
+    def __init__(self, manager, data, pid=-1, rid=-1, add=False):
         """Constructor"""
         PluginThread.__init__(self, manager)
 
@@ -416,6 +416,7 @@ class InfoThread(PluginThread):
         # [ .. (name, plugin) .. ]
 
         self.rid = rid #result id
+        self.add = add #add packages instead of return result
 
         self.cache = [] #accumulated data
 
@@ -440,6 +441,29 @@ class InfoThread(PluginThread):
                     self.fetchForPlugin(pluginname, plugin, urls, self.updateDB)
                     self.m.core.files.save()
 
+        elif self.add:
+            for pluginname, urls in plugins.iteritems():
+                plugin = self.m.core.pluginManager.getPlugin(pluginname, True)
+                if hasattr(plugin, "getInfo"):
+                    self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True)
+
+                else:
+                    #generate default result
+                    result = [(url, 0, 3, url) for url in urls]
+
+                    self.updateCache(pluginname, result)
+
+
+            packs = parseNames([(name, url) for name, x,y, url in self.cache])
+
+            self.m.core.log.debug("Fetched and generated %d packages" % len(packs))
+
+            for k, v in packs:
+                self.m.core.api.addPackage(k, v)
+
+            #empty cache
+            del self.cache[:]
+
         else: #post the results
 
             self.m.infoResults[self.rid] = {}
@@ -489,6 +513,9 @@ class InfoThread(PluginThread):
 
             self.cache = []
 
+    def updateCache(self, plugin, result):
+        self.cache.extend(result)
+
     def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None):
         try:
             result = [] #result loaded from cache
diff --git a/module/ThreadManager.py b/module/ThreadManager.py
index 07ab69548..ce5b7bd1d 100644
--- a/module/ThreadManager.py
+++ b/module/ThreadManager.py
@@ -88,14 +88,14 @@ class ThreadManager:
         PluginThread.InfoThread(self, data, pid)
 
     @lock
-    def createResultThread(self, data):
+    def createResultThread(self, data, add=False):
         """ creates a thread to fetch online status, returns result id """
         self.timestamp = time() + 5 * 60
 
         rid = self.resultIDs
         self.resultIDs += 1
 
-        PluginThread.InfoThread(self, data, rid=rid)
+        PluginThread.InfoThread(self, data, rid=rid, add=add)
 
         return rid
 
diff --git a/module/common/APIExerciser.py b/module/common/APIExerciser.py
index b9e67d824..d86e7266b 100644
--- a/module/common/APIExerciser.py
+++ b/module/common/APIExerciser.py
@@ -4,10 +4,12 @@ import string
 from threading import Thread
 from random import choice, random, sample, randint
 from time import time, sleep
+from math import floor
+import gc
 
 from traceback import print_exc, format_exc
 
-from module.remote.thriftbackend.ThriftClient import ThriftClient
+from module.remote.thriftbackend.ThriftClient import ThriftClient, Destination
 
 def createURLs():
     """ create some urls, some may fail """
@@ -24,6 +26,9 @@ def createURLs():
 AVOID = (0,3,8)
 
 class APIExerciser(Thread):
+
+    idPool = 0
+
     def __init__(self, core, thrift=False):
         Thread.__init__(self)
         self.setDaemon(True)
@@ -37,6 +42,12 @@ class APIExerciser(Thread):
         else:
             self.api = core.api
 
+
+        self.id = self.idPool
+
+        self.core.log.info("API Excerciser started %d" % self.id)
+        APIExerciser.idPool += 1
+
         self.start()
 
     def run(self):
@@ -49,15 +60,17 @@ class APIExerciser(Thread):
             try:
                 self.testAPI()
             except Exception:
+                self.core.log.error("Excerciser %d throw an execption" % self.id)
                 print_exc()
                 out.write(format_exc() + 2 * "\n")
                 out.flush()
 
             if not self.count % 100:
-                print "Tested %s api calls" % self.count
+                self.core.log.info("Exerciser %d tested %d api calls" % (self.id, self.count))
             if not self.count % 1000:
                 out.write("Tested %s api calls\n" % self.count)
                 out.flush()
+                self.core.log.info("Collected garbage: %d" % gc.collect())
 
 
                 #sleep(random() / 500)
@@ -82,7 +95,7 @@ class APIExerciser(Thread):
         name = "".join(sample(string.ascii_letters, 10))
         urls = createURLs()
 
-        self.api.addPackage(name, urls, 0)
+        self.api.addPackage(name, urls, choice([Destination.Queue, Destination.Collector]))
 
 
     def deleteFiles(self):
@@ -98,12 +111,12 @@ class APIExerciser(Thread):
 
 
     def deletePackages(self):
-        info = self.api.getQueue()
+        info = choice([self.api.getQueue(), self.api.getCollector()])
         if not info: return
 
         pids = [p.pid for p in info]
         if len(pids):
-            pids = sample(pids, randint(1, max(len(pids) / 2, 1)))
+            pids = sample(pids, randint(1,  max(floor(len(pids) / 2.5), 1)))
             self.api.deletePackages(pids)
 
     def getFileData(self):
diff --git a/module/plugins/hooks/UnRar.py b/module/plugins/hooks/UnRar.py
index 6dfe2c195..c636664ee 100644
--- a/module/plugins/hooks/UnRar.py
+++ b/module/plugins/hooks/UnRar.py
@@ -24,27 +24,31 @@ from module.plugins.Hook import Hook
 from module.lib.pyunrar import Unrar, WrongPasswordError, CommandError, UnknownError, LowRamError
 from traceback import print_exc
 
+import os
 from os.path import exists, join, isabs, isdir
 from os import remove, makedirs, rmdir, listdir, chown, chmod
-from pwd import getpwnam
+
+if os.name != "nt":
+    from pwd import getpwnam
+
 import re
 
 class UnRar(Hook):
     __name__ = "UnRar"
     __version__ = "0.11"
     __description__ = """unrar"""
-    __config__ = [ ("activated", "bool", "Activated", False),
-                   ("fullpath", "bool", "extract full path", True),
-                   ("overwrite", "bool", "overwrite files", True),
-                   ("passwordfile", "str", "unrar password file", "unrar_passwords.txt"),
-                   ("deletearchive", "bool", "delete archives when done", False),
-                   ("ramwarning", "bool", "warn about low ram", True),
-                   ("renice", "int", "Cpu Priority", 10),
-                   ("unrar_destination", "str", "Unpack files to", "")]
+    __config__ = [("activated", "bool", "Activated", False),
+        ("fullpath", "bool", "extract full path", True),
+        ("overwrite", "bool", "overwrite files", True),
+        ("passwordfile", "str", "unrar password file", "unrar_passwords.txt"),
+        ("deletearchive", "bool", "delete archives when done", False),
+        ("ramwarning", "bool", "warn about low ram", True),
+        ("renice", "int", "Cpu Priority", 10),
+        ("unrar_destination", "str", "Unpack files to", "")]
     __threaded__ = ["packageFinished"]
     __author_name__ = ("mkaay")
     __author_mail__ = ("mkaay@mkaay.de")
-    
+
     def setup(self):
         self.comments = ["# one password each line"]
         self.passwords = []
@@ -71,31 +75,31 @@ class UnRar(Hook):
             self.ram = 0
 
         self.ram /= 1024
-        
-    def setOwner(self,d,uid,gid,mode):        
+
+    def setOwner(self, d, uid, gid, mode):
         if not exists(d):
-           self.core.log.debug(_("Directory %s does not exist!") % d)
-           return
-        fileList=listdir(d)        
-        for fileEntry in fileList:
-           fullEntryName=join(d,fileEntry)
-           if isdir(fullEntryName):
-              self.setOwner(fullEntryName,uid,gid,mode)
-           try:
-              chown(fullEntryName,uid,gid)
-              chmod(fullEntryName,mode)
-           except:
-              self.core.log.debug(_("Chown/Chmod for %s failed") % fullEntryName)
-              self.core.log.debug(_("Exception: %s") % sys.exc_info()[0])
-              continue
+            self.core.log.debug(_("Directory %s does not exist!") % d)
+            return
+
+        for fileEntry in listdir(d):
+            fullEntryName = join(d, fileEntry)
+            if isdir(fullEntryName):
+                self.setOwner(fullEntryName, uid, gid, mode)
+            try:
+                chown(fullEntryName, uid, gid)
+                chmod(fullEntryName, mode)
+            except:
+                self.core.log.debug(_("Chown/Chmod for %s failed") % fullEntryName)
+                self.core.log.debug(_("Exception: %s") % sys.exc_info()[0])
+                continue
         try:
-           chown(d,uid,gid)
-           chmod(d,mode)
+            chown(d, uid, gid)
+            chmod(d, mode)
         except:
-           self.core.log.debug(_("Chown/Chmod for %s failed") % d)
-           self.core.log.debug(_("Exception: %s") % sys.exc_info()[0])
-           return
-         
+            self.core.log.debug(_("Chown/Chmod for %s failed") % d)
+            self.core.log.debug(_("Exception: %s") % sys.exc_info()[0])
+            return
+
     def addPassword(self, pws):
         if not type(pws) == list: pws = [pws]
         pws.reverse()
@@ -105,14 +109,14 @@ class UnRar(Hook):
             self.passwords.insert(0, pw)
 
         with open(self.getConfig("passwordfile"), "w") as f:
-            f.writelines([c+"\n" for c in self.comments])
-            f.writelines([p+"\n" for p in self.passwords])
-        
+            f.writelines([c + "\n" for c in self.comments])
+            f.writelines([p + "\n" for p in self.passwords])
+
     def removeFiles(self, pack, fname):
         if not self.getConfig("deletearchive"):
             return
         m = self.re_splitfile.search(fname)
-                
+
         download_folder = self.core.config['general']['download_folder']
         if self.core.config['general']['folder_per_package']:
             folder = join(download_folder, pack.folder.decode(sys.getfilesystemencoding()))
@@ -124,11 +128,11 @@ class UnRar(Hook):
                 if nre.match(data["name"]):
                     remove(join(folder, data["name"]))
         elif not m and fname.endswith(".rar"):
-            nre = re.compile("^%s\.r..$" % fname.replace(".rar",""))
+            nre = re.compile("^%s\.r..$" % fname.replace(".rar", ""))
             for fid, data in pack.getChildren().iteritems():
                 if nre.match(data["name"]):
                     remove(join(folder, data["name"]))
-    
+
     def packageFinished(self, pack):
         if pack.password and pack.password.strip() and pack.password.strip() != "None":
             self.addPassword(pack.password.splitlines())
@@ -137,15 +141,16 @@ class UnRar(Hook):
         for fid, data in pack.getChildren().iteritems():
             m = self.re_splitfile.search(data["name"])
             if m and int(m.group(2)) == 1:
-                files.append((fid,m.group(0)))
+                files.append((fid, m.group(0)))
             elif not m and data["name"].endswith(".rar"):
-                files.append((fid,data["name"]))
-        
+                files.append((fid, data["name"]))
+
         for fid, fname in files:
             self.core.log.info(_("starting Unrar of %s") % fname)
             pyfile = self.core.files.getFile(fid)
             pyfile.setStatus("processing")
             pyfile.progress.setRange(0, 100)
+
             def s(p):
                 pyfile.progress.setValue(p)
 
@@ -170,12 +175,14 @@ class UnRar(Hook):
 
             self.core.log.debug(_("Destination folder %s") % destination)
             if not exists(destination):
-               self.core.log.info(_("Creating destination folder %s") % destination)
-               makedirs(destination)
+                self.core.log.info(_("Creating destination folder %s") % destination)
+                makedirs(destination)
 
-            u = Unrar(join(folder, fname), tmpdir=join(folder, "tmp"), ramSize=(self.ram if self.getConfig("ramwarning") else 0), cpu=self.getConfig("renice"))
+            u = Unrar(join(folder, fname), tmpdir=join(folder, "tmp"),
+                      ramSize=(self.ram if self.getConfig("ramwarning") else 0), cpu=self.getConfig("renice"))
             try:
-                success = u.crackPassword(passwords=self.passwords, statusFunction=s, overwrite=True, destination=destination, fullPath=self.getConfig("fullpath"))
+                success = u.crackPassword(passwords=self.passwords, statusFunction=s, overwrite=True,
+                                          destination=destination, fullPath=self.getConfig("fullpath"))
             except WrongPasswordError:
                 self.core.log.info(_("Unrar of %s failed (wrong password)") % fname)
                 continue
@@ -195,7 +202,8 @@ class UnRar(Hook):
                     self.core.log.info(_("Unrar of %s failed") % fname)
                     continue
             except LowRamError:
-                self.log.warning(_("Your ram amount of %s MB seems not sufficient to unrar this file. You can deactivate this warning and risk instability") % self.ram)
+                self.log.warning(_(
+                    "Your ram amount of %s MB seems not sufficient to unrar this file. You can deactivate this warning and risk instability") % self.ram)
                 continue
             except UnknownError:
                 if self.core.debug:
@@ -207,19 +215,24 @@ class UnRar(Hook):
                     self.core.log.info(_("Unrar of %s ok") % fname)
                     self.removeFiles(pack, fname)
                     if self.core.config['general']['folder_per_package']:
-                       if self.getConfig("deletearchive"):
-                          self.core.log.debug(_("Deleting package directory %s...") % folder)
-                          rmdir(folder)
-                    self.core.log.debug(_("Package directory %s has been deleted.") % folder)
-                    ownerUser=self.core.config['permission']['user']
-                    uinfo=getpwnam(ownerUser)
-                    fileMode=int(self.core.config['permission']['file'],8)
-                    self.core.log.debug(_("Setting destination file/directory owner to %s.") % ownerUser)
-                    self.core.log.debug(_("Setting destination file/directory mode to %s.") % fileMode)
-                    self.core.log.debug(_("Uid is %s.") % uinfo.pw_uid)
-                    self.core.log.debug(_("Gid is %s.") % uinfo.pw_gid)
-                    self.setOwner(destination,uinfo.pw_uid,uinfo.pw_gid,fileMode)
-                    self.core.log.debug(_("The owner/rights have been successfully changed."))
+                        if self.getConfig("deletearchive"):
+                            self.core.log.debug(_("Deleting package directory %s...") % folder)
+                            rmdir(folder)
+                            self.core.log.debug(_("Package directory %s has been deleted.") % folder)
+
+                    if os.name != "nt" and self.core.config['permission']['change_dl'] and\
+                       self.core.config['permission']['change_file']:
+                        ownerUser = self.core.config['permission']['user']
+                        fileMode = int(self.core.config['permission']['file'], 8)
+
+                        self.core.log.debug("Setting destination file/directory owner / mode to %s / %s"
+                        % (ownerUser, fileMode))
+
+                        uinfo = getpwnam(ownerUser)
+                        self.core.log.debug("Uid/Gid is %s/%s." % (uinfo.pw_uid, uinfo.pw_gid))
+                        self.setOwner(destination, uinfo.pw_uid, uinfo.pw_gid, fileMode)
+                        self.core.log.debug("The owner/rights have been successfully changed.")
+
                     self.core.hookManager.unrarFinished(folder, fname)
                 else:
                     self.core.log.info(_("Unrar of %s failed (wrong password or bad parts)") % fname)
diff --git a/module/remote/thriftbackend/ThriftClient.py b/module/remote/thriftbackend/ThriftClient.py
index b3e6ac6ed..c8e8f2bd7 100644
--- a/module/remote/thriftbackend/ThriftClient.py
+++ b/module/remote/thriftbackend/ThriftClient.py
@@ -16,6 +16,7 @@ from Socket import Socket
 from Protocol import Protocol
 
 from thriftgen.pyload import Pyload
+from thriftgen.pyload.ttypes import *
 from thriftgen.pyload.Pyload import PackageDoesNotExists
 from thriftgen.pyload.Pyload import FileDoesNotExists