scheduler optimization, SJOrg fix, better crypter limit

8 files changed, 97 insertions, 42 deletions

diff --git a/module/AccountManager.py b/module/AccountManager.py
index 6f2d9c186..202032052 100644
--- a/module/AccountManager.py
+++ b/module/AccountManager.py
@@ -38,6 +38,9 @@ class AccountManager():
         self.plugins = {}
 
         self.initAccountPlugins()
+        
+        self.accountInfoCache = []
+        
         self.loadAccounts()
 
         self.saveAccounts() # save to add categories to conf
@@ -155,18 +158,27 @@ class AccountManager():
             self.saveAccounts()
             p.getAllAccounts(force=True)
     
-    def getAccountInfos(self, force=False):
+    def getAccountInfos(self, force=False, cache=False):
         data = {}
+        if not force:
+            return self.accountInfoCache
+        elif not cache:
+            self.core.scheduler.addJob(0, self.core.accountManager.cacheAccountInfos) #prevent gui from blocking
+            return self.accountInfoCache
+        
         for p in self.accounts.keys():
             if self.accounts[p]:
                 p = self.getAccountPlugin(p)
                 data[p.__name__] = p.getAllAccounts(force)
             else:
                 data[p] = []
+        self.accountInfoCache = data
+        e = AccountUpdateEvent()
+        self.core.pullManager.addEvent(e)
         return data
 
     def cacheAccountInfos(self):
-        self.getAccountInfos()
+        self.getAccountInfos(True, True)
     
     def sendChange(self):
         e = AccountUpdateEvent()
diff --git a/module/PluginThread.py b/module/PluginThread.py
index f12639719..9cce3a42e 100644
--- a/module/PluginThread.py
+++ b/module/PluginThread.py
@@ -309,11 +309,6 @@ class DecrypterThread(PluginThread):
         pyfile = self.active
         retry = False
         
-        if not self.active.plugin.multiDL:
-            while self.m.isOccupiedCrypter(self.active.pluginname):
-                sleep(0.5)
-            self.m.addOccupiedCrypter(self.active.pluginname)
-        
         try:
             self.m.log.info(_("Decrypting starts: %s") % self.active.name)
             self.active.plugin.preprocessing(self)
@@ -348,8 +343,6 @@ class DecrypterThread(PluginThread):
             
             self.m.log.info(_("Retrying %s") % self.active.name)
             retry = True
-            if not self.active.plugin.multiDL:
-                self.m.removeOccupiedCrypter(pyfile.pluginname)
             return self.run()
 
         except Exception, e:
@@ -367,8 +360,7 @@ class DecrypterThread(PluginThread):
 
         finally:
             if not retry:
-                if not self.active.plugin.multiDL:
-                    self.m.removeOccupiedCrypter(pyfile.pluginname)
+                self.m.removeOccupiedCrypter(pyfile.pluginname)
                 self.active.release()
                 self.active = False
                 self.m.core.files.save()
diff --git a/module/Scheduler.py b/module/Scheduler.py
index 018b94e10..2a76bb071 100644
--- a/module/Scheduler.py
+++ b/module/Scheduler.py
@@ -17,41 +17,81 @@
     @author: mkaay
 """
 
-from time import sleep
-from Queue import Queue
+from time import sleep, time
+from Queue import PriorityQueue, Empty
 from threading import Thread
 
+class AlreadyCalled(Exception):
+    pass
+
+def callInThread(f, *args, **kwargs):
+    class FThread(Thread):
+        def run(self):
+            f(*args, **kwargs)
+    t = FThread()
+    t.start()
+
+class Deferred():
+    def __init__(self):
+        self.call = []
+        self.result = ()
+    
+    def addCallback(self, f, *cargs, **ckwargs):
+        self.call.append((f, cargs, ckwargs))
+        if self.result:
+            args, kwargs = self.result
+            args.extend(cargs)
+            kwargs.update(ckwargs)
+            callInThread(f, *args, **kwargs)
+    
+    def callback(self, *args, **kwargs):
+        if self.result:
+            raise AlreadyCalled
+        self.result = (args, kwargs)
+        for f, cargs, ckwargs in self.call:
+            args.extend(cargs)
+            kwargs.update(ckwargs)
+            callInThread(f, *args, **kwargs)
+
 class Scheduler():
     def __init__(self, core):
         self.core = core
         
-        self.queue = Queue()
+        self.queue = PriorityQueue()
     
-    def addJob(self, time, call, args=[], kwargs={}, done=None):
-        j = Job(time, call, args, kwargs, done)
-        self.queue.put(j)
+    def addJob(self, t, call, args=[], kwargs={}):
+        d = Deferred()
+        t += time()
+        j = Job(t, call, args, kwargs, d)
+        self.queue.put((t, j))
+        return d
+    
+    def work(self):
+        while True:
+            try:
+                t, j = self.queue.get(False)
+            except Empty:
+                break
+            else:
+                if t <= time():
+                    j.start()
+                else:
+                    self.queue.put((t, j))
 
 class Job(Thread):
-    def __init__(self, time, call, args=[], kwargs={}, done=None):
+    def __init__(self, time, call, args=[], kwargs={}, deferred=None):
         Thread.__init__(self)
         self.time = float(time)
-        self.interval = 0.5
         self.call = call
-        self.done = done
+        self.deferred = deferred
         self.args = args
         self.kwargs = kwargs
     
     def run(self):
-        while self.time > 0:
-            sleep(self.interval)
-            self.time -= self.interval
-        self.work()
-    
-    def work(self):
         ret = self.call(*self.args, **self.kwargs)
-        if self.done is None:
+        if self.deferred is None:
             return
         if ret is None:
-            self.done()
+            self.deferred.callback()
         else:
-            self.done(ret)
+            self.deferred.callback(ret)
diff --git a/module/ThreadManager.py b/module/ThreadManager.py
index d17281e9d..a2a8b0411 100644
--- a/module/ThreadManager.py
+++ b/module/ThreadManager.py
@@ -63,21 +63,21 @@ class ThreadManager:
     #----------------------------------------------------------------------
     def addOccupiedCrypter(self, name):
         self.occupiedCrypterLock.acquire()
-        if not name in self.occupiedCrypter:
-            self.occupiedCrypter.append(name)
-            print True
+        self.occupiedCrypter.append(name)
         self.occupiedCrypterLock.release()
     
     def removeOccupiedCrypter(self, name):
         self.occupiedCrypterLock.acquire()
         if name in self.occupiedCrypter:
-            print True
             self.occupiedCrypter.remove(name)
         self.occupiedCrypterLock.release()
     
     def isOccupiedCrypter(self, name):
         self.occupiedCrypterLock.acquire()
-        ret = name in self.occupiedCrypter
+        ret = 0
+        for plugin in self.occupiedCrypter:
+            if name == plugin:
+                ret += 1
         self.occupiedCrypterLock.release()
         return ret
     
@@ -253,11 +253,19 @@ class ThreadManager:
                     job = self.core.files.getDecryptJob()
                     if job:
                         job.initPlugin()
-                        thread = PluginThread.DecrypterThread(self, job)
+                        if job.plugin.multiDL or self.isOccupiedCrypter(job.pluginname) < job.plugin.limitDL:
+                            thread = PluginThread.DecrypterThread(self, job)
+                            if not job.plugin.multiDL:
+                                print "add"
+                                self.addOccupiedCrypter(job.pluginname)
 
 
             else:
-                thread = PluginThread.DecrypterThread(self, job)
+                if job.plugin.multiDL or self.isOccupiedCrypter(job.pluginname) < job.plugin.limitDL:
+                    thread = PluginThread.DecrypterThread(self, job)
+                    if not job.plugin.multiDL:
+                        print "add"
+                        self.addOccupiedCrypter(job.pluginname)
 
     def cleanup(self):
         """do global cleanup"""
diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py
index 123c26eec..0abd11e82 100644
--- a/module/plugins/Crypter.py
+++ b/module/plugins/Crypter.py
@@ -38,6 +38,7 @@ class Crypter(Plugin):
         self.packages = []
         
         self.multiDL = True
+        self.limitDL = 0
         self.setup()
     
     #----------------------------------------------------------------------
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index b1cedb341..d642df728 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -85,6 +85,7 @@ class Plugin(object):
 
         self.wantReconnect = False
         self.multiDL = True
+        self.limitDL = 0
 
         self.waitUntil = 0 # time() + wait in seconds
         self.waiting = False
diff --git a/module/plugins/crypter/SerienjunkiesOrg.py b/module/plugins/crypter/SerienjunkiesOrg.py
index 709bc7630..07a73d0b0 100644
--- a/module/plugins/crypter/SerienjunkiesOrg.py
+++ b/module/plugins/crypter/SerienjunkiesOrg.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import re
+from time import sleep
 
 from module.plugins.Crypter import Crypter
 from module.BeautifulSoup import BeautifulSoup
@@ -34,10 +35,12 @@ class SerienjunkiesOrg(Crypter):
         self.hosterMapReverse = dict((v,k) for k, v in self.hosterMap.iteritems())
     
         self.multiDL = False
+        self.limitDL = 4
     
     def getSJSrc(self, url):
         src = self.req.load(str(url))
         if not src.find("Enter Serienjunkies") == -1:
+            sleep(1)
             src = self.req.load(str(url))
         return src
     
@@ -115,6 +118,7 @@ class SerienjunkiesOrg(Crypter):
             packageName = soup.find("h1", attrs={"class":"wrap"}).text
             captchaTag = soup.find(attrs={"src":re.compile("^/secure/")})
             if not captchaTag:
+                sleep(1)
                 self.retry()
             
             captchaUrl = "http://download.serienjunkies.org"+captchaTag["src"]
@@ -128,7 +132,9 @@ class SerienjunkiesOrg(Crypter):
             rawLinks = soup.findAll(attrs={"action": re.compile("^http://download.serienjunkies.org/")})
             
             if not len(rawLinks) > 0:
+                sleep(1)
                 self.retry()
+                return
             
             links = []
             for link in rawLinks:
diff --git a/pyLoadCore.py b/pyLoadCore.py
index b8d192e8b..04591d0b5 100755
--- a/pyLoadCore.py
+++ b/pyLoadCore.py
@@ -328,14 +328,9 @@ class Core(object):
                 exit()
 
             self.threadManager.work()
+            self.scheduler.work()
             self.hookManager.periodical()
 
-            try:
-                j = self.scheduler.queue.get(False)
-                j.start()
-            except:
-                pass
-
     def init_server(self):
         try:
             server_addr = (self.config['remote']['listenaddr'], int(self.config['remote']['port']))