diff options
| -rw-r--r-- | module/Api.py | 2 | ||||
| -rw-r--r-- | module/HookManager.py | 191 | ||||
| -rw-r--r-- | module/interaction/EventManager.py | 151 | ||||
| -rw-r--r-- | module/plugins/Crypter.py | 4 | ||||
| -rw-r--r-- | module/plugins/Hook.py | 32 | ||||
| -rw-r--r-- | module/plugins/internal/MultiHoster.py | 2 | ||||
| -rw-r--r-- | module/threads/DecrypterThread.py | 2 | ||||
| -rw-r--r-- | module/threads/HookThread.py | 9 | ||||
| -rw-r--r-- | module/threads/InfoThread.py | 48 | ||||
| -rw-r--r-- | module/utils/__init__.py | 19 | ||||
| -rw-r--r-- | module/web/pyload_app.py | 1 | ||||
| -rwxr-xr-x | pyLoadCore.py | 8 |
12 files changed, 216 insertions, 253 deletions
diff --git a/module/Api.py b/module/Api.py index bafb69408..769fcdf0d 100644 --- a/module/Api.py +++ b/module/Api.py @@ -810,7 +810,7 @@ class Api(Iface): def getEvents(self, uuid): """Lists occured events, may be affected to changes in future. - :param uuid: + :param uuid: self assigned string uuid which has to be unique :return: list of `Events` """ # TODO diff --git a/module/HookManager.py b/module/HookManager.py index 7057b6ee6..41908bc47 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -18,7 +18,7 @@ """ import __builtin__ -import traceback +from traceback import print_exc from thread import start_new_thread from threading import RLock @@ -38,11 +38,9 @@ class HookManager: __builtin__.hookManager = self #needed to let hooks register themself self.log = self.core.log - self.plugins = [] - self.pluginMap = {} - self.methods = {} #dict of names and list of methods usable by rpc - - self.events = {} # contains events + self.plugins = {} + self.methods = {} # dict of names and list of methods usable by rpc + self.events = {} # Contains event that will be registred self.lock = RLock() self.createIndex() @@ -53,17 +51,19 @@ class HookManager: # manage hooks an config change self.addEvent("configChanged", self.manageHooks) - def try_catch(func): - def new(*args): + @lock + def callInHooks(self, event, *args): + """ Calls a method in hook and catch / log errors""" + for plugin in self.plugins.itervalues(): try: + func = getattr(plugin, event) return func(*args) except Exception, e: - args[0].log.error(_("Error executing hooks: %s") % str(e)) - if args[0].core.debug: - traceback.print_exc() - - return new + plugin.logError(_("Error executing %s" % event), e) + if self.core.debug: + print_exc() + self.dispatchEvent(event, *args) def addRPC(self, plugin, func, doc): plugin = plugin.rpartition(".")[2] @@ -79,14 +79,12 @@ class HookManager: else: args = literal_eval(args) - plugin = self.pluginMap[plugin] + plugin = self.plugins[plugin] f = getattr(plugin, func) return f(*args) - + @lock def createIndex(self): - plugins = [] - active = [] deactive = [] @@ -97,10 +95,9 @@ class HookManager: if self.core.config.get(pluginname, "activated"): pluginClass = self.core.pluginManager.loadClass("hooks", pluginname) if not pluginClass: continue - + plugin = pluginClass(self.core, self) - plugins.append(plugin) - self.pluginMap[pluginClass.__name__] = plugin + self.plugins[pluginClass.__name__] = plugin if plugin.isActivated(): active.append(pluginClass.__name__) else: @@ -110,15 +107,12 @@ class HookManager: except: self.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) if self.core.debug: - traceback.print_exc() + print_exc() self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active))) self.log.info(_("Deactivate plugins: %s") % ", ".join(sorted(deactive))) - self.plugins = plugins - def manageHooks(self, plugin, name, value): - # check if section was a plugin if plugin not in self.core.pluginManager.getPlugins("hooks"): return @@ -128,12 +122,11 @@ class HookManager: elif name == "activated" and not value: self.deactivateHook(plugin) + @lock def activateHook(self, plugin): - #check if already loaded - for inst in self.plugins: - if inst.__name__ == plugin: - return + if plugin in self.plugins: + return pluginClass = self.core.pluginManager.loadClass("hooks", plugin) @@ -142,105 +135,63 @@ class HookManager: self.log.debug("Plugin loaded: %s" % plugin) plugin = pluginClass(self.core, self) - self.plugins.append(plugin) - self.pluginMap[pluginClass.__name__] = plugin + self.plugins[pluginClass.__name__] = plugin - # call core Ready - start_new_thread(plugin.coreReady, tuple()) + # active the hook in new thread + start_new_thread(plugin.activate, tuple()) + @lock def deactivateHook(self, plugin): + if plugin not in self.plugins: + return + else: + hook = self.plugins[plugin] - hook = None - for inst in self.plugins: - if inst.__name__ == plugin: - hook = inst - - if not hook: return - - hook.unload() - self.log.debug("Plugin unloaded: %s" % plugin) + hook.deactivate() + self.log.debug("Plugin deactivated: %s" % plugin) #remove periodic call self.log.debug("Removed callback %s" % self.core.scheduler.removeJob(hook.cb)) - self.plugins.remove(hook) - del self.pluginMap[hook.__name__] + del self.plugins[hook.__name__] + #remove event listener + for f in dir(hook): + if f.startswith("__") or type(getattr(hook, f)) != MethodType: + continue + self.core.eventManager.removeFromEvents(getattr(hook, f)) - @try_catch - def coreReady(self): - for plugin in self.plugins: + def activateHooks(self): + for plugin in self.plugins.itervalues(): if plugin.isActivated(): - plugin.coreReady() - - self.dispatchEvent("coreReady") + plugin.activate() - @try_catch - def coreExiting(self): - for plugin in self.plugins: + def deactivateHooks(self): + """ Called when core is shutting down """ + for plugin in self.plugins.itervalues(): if plugin.isActivated(): - plugin.coreExiting() - - self.dispatchEvent("coreExiting") + plugin.deactivate() - @lock def downloadPreparing(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.downloadPreparing(pyfile) + self.callInHooks("downloadPreparing", pyfile) - self.dispatchEvent("downloadPreparing", pyfile) - - @lock def downloadFinished(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - if "downloadFinished" in plugin.__threaded__: - self.startThread(plugin.downloadFinished, pyfile) - else: - plugin.downloadFinished(pyfile) - - self.dispatchEvent("downloadFinished", pyfile) + self.callInHooks("downloadFinished", pyfile) - @lock - @try_catch def downloadFailed(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - if "downloadFailed" in plugin.__threaded__: - self.startThread(plugin.downloadFinished, pyfile) - else: - plugin.downloadFailed(pyfile) - - self.dispatchEvent("downloadFailed", pyfile) + self.callInHooks("downloadFailed", pyfile) - @lock def packageFinished(self, package): - for plugin in self.plugins: - if plugin.isActivated(): - if "packageFinished" in plugin.__threaded__: - self.startThread(plugin.packageFinished, package) - else: - plugin.packageFinished(package) - - self.dispatchEvent("packageFinished", package) + self.callInHooks("packageFinished", package) - @lock def beforeReconnecting(self, ip): - for plugin in self.plugins: - plugin.beforeReconnecting(ip) + self.callInHooks("beforeReconnecting", ip) - self.dispatchEvent("beforeReconnecting", ip) - - @lock def afterReconnecting(self, ip): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.afterReconnecting(ip) - - self.dispatchEvent("afterReconnecting", ip) + self.callInHooks("afterReconnecting", ip) + @lock def startThread(self, function, *args, **kwargs): - t = HookThread(self.core.threadManager, function, args, kwargs) + HookThread(self.core.threadManager, function, args, kwargs) def activePlugins(self): """ returns all active plugins """ @@ -249,42 +200,24 @@ class HookManager: def getAllInfo(self): """returns info stored by hook plugins""" info = {} - for name, plugin in self.pluginMap.iteritems(): + for name, plugin in self.plugins.iteritems(): if plugin.info: #copy and convert so str - info[name] = dict([(x, str(y) if not isinstance(y, basestring) else y) for x, y in plugin.info.iteritems()]) + info[name] = dict( + [(x, str(y) if not isinstance(y, basestring) else y) for x, y in plugin.info.iteritems()]) return info - def getInfo(self, plugin): info = {} - if plugin in self.pluginMap and self.pluginMap[plugin].info: + if plugin in self.plugins and self.plugins[plugin].info: info = dict([(x, str(y) if not isinstance(y, basestring) else y) - for x, y in self.pluginMap[plugin].info.iteritems()]) + for x, y in self.plugins[plugin].info.iteritems()]) return info - def addEvent(self, event, func): - """Adds an event listener for event name""" - if event in self.events: - self.events[event].append(func) - else: - self.events[event] = [func] - - def removeEvent(self, event, func): - """removes previously added event listener""" - if event in self.events: - self.events[event].remove(func) - - def dispatchEvent(self, event, *args): - """dispatches event with args""" - if event in self.events: - for f in self.events[event]: - try: - f(*args) - except Exception, e: - self.log.warning("Error calling event handler %s: %s, %s, %s" - % (event, f, args, str(e))) - if self.core.debug: - traceback.print_exc() + def addEvent(self, *args): + self.core.eventManager.addEvent(*args) + + def dispatchEvent(self, *args): + self.core.eventManager.dispatchEvent(*args) diff --git a/module/interaction/EventManager.py b/module/interaction/EventManager.py index 931f50446..38faa3c46 100644 --- a/module/interaction/EventManager.py +++ b/module/interaction/EventManager.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from time import time -from module.utils import uniqify +from traceback import print_exc +from time import time class EventManager: """ @@ -10,70 +10,119 @@ class EventManager: **Known Events:** Most hook methods exists as events. These are some additional known events. - ===================== ============== ================================== - Name Arguments Description - ===================== ============== ================================== - downloadPreparing fid A download was just queued and will be prepared now. - downloadStarts fid A plugin will immediately starts the download afterwards. - linksAdded links, pid Someone just added links, you are able to modify the links. - allDownloadsProcessed Every link was handled, pyload would idle afterwards. - allDownloadsFinished Every download in queue is finished. - unrarFinished folder, fname An Unrar job finished - configChanged sec,opt,value The config was changed. - ===================== ============== ================================== + ===================== ================ =========================================================== + Name Arguments Description + ===================== ================ =========================================================== + metaEvent eventName, *args Called for every event, with eventName and orginal args + downloadPreparing fid A download was just queued and will be prepared now. + downloadStarts fid A plugin will immediately starts the download afterwards. + linksAdded links, pid Someone just added links, you are able to modify the links. + allDownloadsProcessed Every link was handled, pyload would idle afterwards. + allDownloadsFinished Every download in queue is finished. + unrarFinished folder, fname An Unrar job finished + configChanged sec, opt, value The config was changed. + ===================== ================ =========================================================== | Notes: | allDownloadsProcessed is *always* called before allDownloadsFinished. | configChanged is *always* called before pluginConfigChanged. """ + + CLIENT_EVENTS = ("packageUpdated", "packageInserted", "linkUpdated", "packageDeleted") + def __init__(self, core): self.core = core - self.clients = [] - - def newClient(self, uuid): - self.clients.append(Client(uuid)) + self.log = core.log - def clean(self): - for n, client in enumerate(self.clients): - if client.lastActive + 30 < time(): - del self.clients[n] + # uuid : list of events + self.clients = {} + self.events = {"metaEvent": []} def getEvents(self, uuid): - events = [] - validUuid = False - for client in self.clients: - if client.uuid == uuid: - client.lastActive = time() - validUuid = True - while client.newEvents(): - events.append(client.popEvent().toList()) - break - if not validUuid: - self.newClient(uuid) - events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] - return uniqify(events, repr) - - def addEvent(self, event): - for client in self.clients: - client.addEvent(event) - - def dispatchEvent(self, *args): - pass + """ Get accumulated events for uuid since last call, this also registeres new client """ + if uuid not in self.clients: + self.clients[uuid] = Client() + return self.clients[uuid].get() + + def addEvent(self, event, func): + """Adds an event listener for event name""" + if event in self.events: + if func in self.events[event]: + self.log.debug("Function already registered %s" % func) + else: + self.events[event].append(func) + else: + self.events[event] = [func] + + def removeEvent(self, event, func): + """removes previously added event listener""" + if event in self.events: + self.events[event].remove(func) + + def dispatchEvent(self, event, *args): + """dispatches event with args""" + for f in self.events["metaEvent"]: + try: + f(event, *args) + except Exception, e: + self.log.warning("Error calling event handler %s: %s, %s, %s" + % ("metaEvent", f, args, str(e))) + if self.core.debug: + print_exc() + + if event in self.events: + for f in self.events[event]: + try: + f(*args) + except Exception, e: + self.log.warning("Error calling event handler %s: %s, %s, %s" + % (event, f, args, str(e))) + if self.core.debug: + print_exc() + + # append to client event queue + if event in self.CLIENT_EVENTS: + for uuid, client in self.clients.items(): + if client.delete(): + del self.clients[uuid] + else: + client.append(event, args) + + + def removeFromEvents(self, func): + """ Removes func from all known events """ + for name, events in self.events.iteritems(): + if func in events: + events.remove(func) + class Client: - def __init__(self, uuid): - self.uuid = uuid + + # delete clients after this time + TIMEOUT = 60 * 60 + # max events, if this value is reached you should assume that older events were dropped + MAX = 30 + + def __init__(self): self.lastActive = time() self.events = [] - def newEvents(self): - return len(self.events) > 0 + def delete(self): + return self.lastActive + self.TIMEOUT < time() + + def append(self, event, args): + ev = (event, args) + if ev not in self.events: + self.events.insert(0, ev) - def popEvent(self): - if not len(self.events): - return None - return self.events.pop(0) + del self.events[self.MAX:] + + + def get(self): + self.lastActive = time() + + events = self.events + self.events = [] - def addEvent(self, event): - self.events.append(event) + return [(ev, [str(x) for x in args]) for ev, args in events]
\ No newline at end of file diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index fe7f0deb8..5d164da64 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -4,7 +4,7 @@ from traceback import print_exc from module.Api import Destination from module.common.packagetools import parseNames -from module.utils import to_list, has_method +from module.utils import to_list, has_method, uniqify from module.utils.fs import exists, remove, fs_encode from Base import Base, Retry @@ -83,7 +83,7 @@ class Crypter(Base): else: # single url ret.append(url_or_pack) # eliminate duplicates - return set(ret) + return uniqify(ret) def __init__(self, core, package=None, password=None): Base.__init__(self, core) diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index fe464bdaa..76bc19dbe 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -19,6 +19,9 @@ from traceback import print_exc +from functools import wraps +from module.utils import has_method + from Base import Base class Expose(object): @@ -35,6 +38,7 @@ def AddEventListener(event): return f return _klass + class ConfigHandler(object): """ register method as config handler """ def __new__(cls, f, *args, **kwargs): @@ -42,13 +46,14 @@ class ConfigHandler(object): return f def threaded(f): + @wraps(f) def run(*args,**kwargs): hookManager.startThread(f, *args, **kwargs) return run class Hook(Base): """ - Base class for hook plugins. + Base class for hook plugins. Please use @threaded decorator for all longer running task. """ #: automatically register event listeners for functions, attribute will be deleted dont use it yourself @@ -79,16 +84,16 @@ class Hook(Base): for event, funcs in self.event_map.iteritems(): if type(funcs) in (list, tuple): for f in funcs: - self.manager.addEvent(event, getattr(self,f)) + self.evm.addEvent(event, getattr(self,f)) else: - self.manager.addEvent(event, getattr(self,funcs)) + self.evm.addEvent(event, getattr(self,funcs)) #delete for various reasons self.event_map = None if self.event_list: for f in self.event_list: - self.manager.addEvent(f, getattr(self,f)) + self.evm.addEvent(f, getattr(self,f)) self.event_list = None @@ -121,23 +126,16 @@ class Hook(Base): """ more init stuff if needed """ pass - def unload(self): - """ called when hook was deactivated """ - pass - - def deactivate(self): - pass - def activate(self): - pass + """ Used to activate the hook """ + if has_method(self.__class__, "coreReady"): + self.logDebug("Deprecated method .coreReady() use activated() instead") + self.coreReady() - #event methods - overwrite these if needed - def coreReady(self): + def deactivate(self): + """ Used to deactivate the hook. """ pass - def coreExiting(self): - pass - def downloadPreparing(self, pyfile): pass diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 872e0b770..1629fdc5f 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -80,6 +80,6 @@ class MultiHoster(Hook): hoster[self.__name__] = new - def unload(self): + def deactivate(self): for hoster in self.supported: self.core.pluginManager.restoreState("hoster", hoster)
\ No newline at end of file diff --git a/module/threads/DecrypterThread.py b/module/threads/DecrypterThread.py index a1b7e4f38..8edb97c34 100644 --- a/module/threads/DecrypterThread.py +++ b/module/threads/DecrypterThread.py @@ -4,6 +4,7 @@ from time import sleep from traceback import print_exc +from module.utils import uniqify from module.plugins.Base import Retry from module.plugins.Crypter import Package @@ -54,6 +55,7 @@ class DecrypterThread(BaseThread): plugin.logDebug("Decrypted", plugin_result) result.extend(plugin_result) + result = uniqify(result) pack_names = {} urls = [] diff --git a/module/threads/HookThread.py b/module/threads/HookThread.py index fe4a2a651..bffa72ca0 100644 --- a/module/threads/HookThread.py +++ b/module/threads/HookThread.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from copy import copy +from traceback import print_exc from BaseThread import BaseThread @@ -48,6 +49,14 @@ class HookThread(BaseThread): del self.kwargs["thread"] self.f(*self.args, **self.kwargs) + except Exception, e: + if hasattr(self.f, "im_self"): + hook = self.f.im_self + hook.logError(_("An Error occured"), e) + if self.m.core.debug: + print_exc() + self.writeDebugReport(hook.__name__, plugin=hook) + finally: local = copy(self.active) for x in local: diff --git a/module/threads/InfoThread.py b/module/threads/InfoThread.py index 596153c4b..c1e4458ef 100644 --- a/module/threads/InfoThread.py +++ b/module/threads/InfoThread.py @@ -30,7 +30,7 @@ class InfoThread(BaseThread): """run method""" plugins = {} - container = [] + crypter = {} for url, plugin in self.data: if plugin in plugins: @@ -42,8 +42,7 @@ class InfoThread(BaseThread): # filter out crypter plugins for name in self.m.core.pluginManager.getPlugins("crypter"): if name in plugins: - container.extend([(name, url) for url in plugins[name]]) - + crypter[name] = plugins[name] del plugins[name] #directly write to database @@ -60,11 +59,10 @@ class InfoThread(BaseThread): self.m.core.files.save() else: #post the results - #TODO: finer crypter control - for name, url in container: + for name, urls in crypter: #attach container content try: - data = self.decryptContainer(name, url) + data = self.decrypt(name, urls) except: print_exc() self.m.log.error("Could not decrypt container.") @@ -169,34 +167,14 @@ class InfoThread(BaseThread): cb(pluginname, result) - def decryptContainer(self, plugin, url): - data = [] - # only works on container plugins - - self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) - - # dummy pyfile - pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) - - pyfile.initPlugin() - - # little plugin lifecycle - try: - pyfile.plugin.setup() - pyfile.plugin.loadToDisk() - pyfile.plugin.decrypt(pyfile) - pyfile.plugin.deleteTmp() - - for pack in pyfile.plugin.packages: - pyfile.plugin.urls.extend(pack[1]) - - data, crypter = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) + def decrypt(self, plugin, urls): + self.m.log.debug("Pre decrypting %s" % plugin) + klass = self.m.core.pluginManager.loadClass("crypter", plugin) - self.m.log.debug("Got %d links." % len(data)) - - except Exception, e: - self.m.log.debug("Pre decrypting error: %s" % str(e)) - finally: - pyfile.release() + # only decrypt files + if has_method(klass, "decryptFile"): + urls = p.decrypt(urls) + data, crypter = self.m.core.pluginManager.parseUrls(urls) + return data - return data + return [] diff --git a/module/utils/__init__.py b/module/utils/__init__.py index a237fde9b..46621c685 100644 --- a/module/utils/__init__.py +++ b/module/utils/__init__.py @@ -72,21 +72,10 @@ def freeSpace(folder): return free_space(folder) -def uniqify(seq, idfun=None): -# order preserving - if idfun is None: - def idfun(x): return x - seen = {} - result = [] - for item in seq: - marker = idfun(item) - # in old Python versions: - # if seen.has_key(marker) - # but in new ones: - if marker in seen: continue - seen[marker] = 1 - result.append(item) - return result +def uniqify(seq): #by Dave Kirby + """ removes duplicates from list, preserve order """ + seen = set() + return [x for x in seq if x not in seen and not seen.add(x)] def parseFileSize(string, unit=None): #returns bytes diff --git a/module/web/pyload_app.py b/module/web/pyload_app.py index a025f6bcb..f73defb45 100644 --- a/module/web/pyload_app.py +++ b/module/web/pyload_app.py @@ -506,6 +506,7 @@ def setup(): return render_to_response('setup.html', {"user": False, "perms": False}) +@login_required("STATUS") @route("/info") def info(): conf = PYLOAD.getConfigDict() diff --git a/pyLoadCore.py b/pyLoadCore.py index f8b1ad6e8..54dc9ca39 100755 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -439,9 +439,10 @@ class Core(object): self.running = True self.log.info(_("Activating Plugins...")) - self.hookManager.coreReady() + self.hookManager.activateHooks() self.log.info(_("pyLoad is up and running")) + self.eventManager.dispatchEvent("coreReady") #test api # from module.common.APIExerciser import startApiExerciser @@ -550,10 +551,13 @@ class Core(object): def shutdown(self): self.log.info(_("shutting down...")) + self.eventManager.dispatchEvent("coreShutdown") try: if self.config['webinterface']['activated'] and hasattr(self, "webserver"): self.webserver.quit() + + for thread in self.threadManager.threads: thread.put("quit") pyfiles = self.files.cache.values() @@ -561,7 +565,7 @@ class Core(object): for pyfile in pyfiles: pyfile.abortDownload() - self.hookManager.coreExiting() + self.hookManager.deactivateHooks() except: if self.debug: |