diff options
Diffstat (limited to 'pyload')
-rw-r--r-- | pyload/AddonManager.py | 5 | ||||
-rw-r--r-- | pyload/Api.py | 3 | ||||
-rw-r--r-- | pyload/FileManager.py | 2 | ||||
-rw-r--r-- | pyload/PluginManager.py | 390 | ||||
-rw-r--r-- | pyload/api/FileApi.py | 2 | ||||
-rw-r--r-- | pyload/datatypes/PyFile.py | 13 | ||||
-rw-r--r-- | pyload/plugins/MultiHoster.py | 1 | ||||
-rw-r--r-- | pyload/plugins/addons/MultiHoster.py | 23 | ||||
-rw-r--r-- | pyload/remote/apitypes.py | 9 | ||||
-rw-r--r-- | pyload/remote/pyload.thrift | 4 | ||||
-rw-r--r-- | pyload/threads/InfoThread.py | 2 | ||||
-rw-r--r-- | pyload/utils/PluginLoader.py | 296 | ||||
-rw-r--r-- | pyload/web/app/scripts/utils/apitypes.js | 2 | ||||
-rw-r--r-- | pyload/web/pyload_app.py | 1 |
14 files changed, 431 insertions, 322 deletions
diff --git a/pyload/AddonManager.py b/pyload/AddonManager.py index 75ff4ebc9..7935ff112 100644 --- a/pyload/AddonManager.py +++ b/pyload/AddonManager.py @@ -23,7 +23,6 @@ from threading import RLock from types import MethodType from pyload.threads.AddonThread import AddonThread -from pyload.PluginManager import literal_eval from utils import lock, to_string class AddonManager: @@ -150,7 +149,7 @@ class AddonManager: self.core.eventManager.removeFromEvents(getattr(addon, f)) def activateAddons(self): - self.log.info(_("Activating Plugins...")) + self.log.info(_("Activating addons...")) for plugin in self.plugins.itervalues(): if plugin.isActivated(): self.call(plugin, "activate") @@ -159,7 +158,7 @@ class AddonManager: def deactivateAddons(self): """ Called when core is shutting down """ - self.log.info(_("Deactivating Plugins...")) + self.log.info(_("Deactivating addons...")) for plugin in self.plugins.itervalues(): self.call(plugin, "deactivate") diff --git a/pyload/Api.py b/pyload/Api.py index 81e39d82d..afd2bb406 100644 --- a/pyload/Api.py +++ b/pyload/Api.py @@ -40,7 +40,8 @@ stateMap = { DownloadState.All: frozenset(getattr(DownloadStatus, x) for x in dir(DownloadStatus) if not x.startswith("_")), DownloadState.Finished: frozenset((DownloadStatus.Finished, DownloadStatus.Skipped)), DownloadState.Unfinished: None, # set below - DownloadState.Failed: frozenset((DownloadStatus.Failed, DownloadStatus.TempOffline, DownloadStatus.Aborted)), + DownloadState.Failed: frozenset((DownloadStatus.Failed, DownloadStatus.TempOffline, DownloadStatus.Aborted, + DownloadStatus.NotPossible)), DownloadState.Unmanaged: None, #TODO } diff --git a/pyload/FileManager.py b/pyload/FileManager.py index b1d3891e9..614418f99 100644 --- a/pyload/FileManager.py +++ b/pyload/FileManager.py @@ -52,7 +52,7 @@ class FileManager: # translations self.statusMsg = [_("none"), _("offline"), _("online"), _("queued"), _("paused"), _("finished"), _("skipped"), _("failed"), _("starting"), - _("waiting"), _("downloading"), _("temp. offline"), _("aborted"), + _("waiting"), _("downloading"), _("temp. offline"), _("aborted"), _("not possible"), _("decrypting"), _("processing"), _("custom"), _("unknown")] self.files = {} # holds instances for files diff --git a/pyload/PluginManager.py b/pyload/PluginManager.py index 6886903cc..2e3c66e03 100644 --- a/pyload/PluginManager.py +++ b/pyload/PluginManager.py @@ -15,222 +15,66 @@ # @author: RaNaN, mkaay ############################################################################### -import re import sys -from os import listdir, makedirs -from os.path import isfile, join, exists, abspath, basename -from sys import version_info -from time import time -from collections import defaultdict +from os.path import abspath, join +from pyload.utils.PluginLoader import LoaderFactory, PluginLoader -from pyload.lib.SafeEval import const_eval as literal_eval -from pyload.plugins.Base import Base -from new_collections import namedtuple +class PluginMatcher(object): + """ Abstract class that allows modify which plugins to match and to load """ -#TODO: ignores not updatable + def matchURL(self, url): + return None -# ignore these plugin configs, mainly because plugins were wiped out -IGNORE = ( - "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('addons', 'UnRar'), - 'EasyShareCom', 'FlyshareCz' -) - -PluginTuple = namedtuple("PluginTuple", "version re deps category user path") - -class BaseAttributes(defaultdict): - """ Dictionary that loads defaults values from Base object """ - def __missing__(self, key): - attr = "__%s__" % key - if not hasattr(Base, attr): - return defaultdict.__missing__(self, key) - - return getattr(Base, attr) + def getPlugin(self, plugin, name): + return False class PluginManager: - ROOT = "pyload.plugins." - LOCALROOT = "localplugins." - TYPES = ("crypter", "hoster", "accounts", "addons", "network", "internal") - - BUILTIN = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s?(True|False|None|[0-9x.]+)', re.I) - SINGLE = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?<!")"(?!")|\'|\().*(?:(?<!")"(?!")|\'|\)))', - re.I) - # note the nongreedy character: that means we can not embed list and dicts - MULTI = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I) + ROOT = "pyload.plugins" + LOCALROOT = "localplugins" - NO_MATCH = re.compile(r'^no_match$') + MATCH_HISTORY = 10 + DEFAULT_PLUGIN = "BasePlugin" def __init__(self, core): self.core = core - - #self.config = self.core.config self.log = core.log - self.plugins = {} - self.modules = {} # cached modules - self.history = [] # match history to speedup parsing (type, name) - self.user_context = {} # plugins working with user context - self.createIndex() + # cached modules (type, name) + self.modules = {} + # match history to speedup parsing (type, name) + self.history = [] #register for import addon sys.meta_path.append(self) - def logDebug(self, type, plugin, msg): - self.log.debug("Plugin %s | %s: %s" % (type, plugin, msg)) - - def createIndex(self): - """create information for all plugins available""" # add to path, so we can import from userplugins sys.path.append(abspath("")) + self.loader = LoaderFactory(PluginLoader(abspath(self.LOCALROOT), self.LOCALROOT, self.core.config), + PluginLoader(abspath(join(pypath, "pyload", "plugins")), self.ROOT, + self.core.config)) - if not exists("userplugins"): - makedirs("userplugins") - if not exists(join("userplugins", "__init__.py")): - f = open(join("userplugins", "__init__.py"), "wb") - f.close() - - a = time() - for type in self.TYPES: - self.plugins[type] = self.parse(type) - - self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000) - - def parse(self, folder, home=None): - """ Analyze and parses all plugins in folder """ - plugins = {} - if home: - pfolder = join("userplugins", folder) - if not exists(pfolder): - makedirs(pfolder) - if not exists(join(pfolder, "__init__.py")): - f = open(join(pfolder, "__init__.py"), "wb") - f.close() - - else: - pfolder = join(pypath, "pyload", "plugins", folder) - - for f in listdir(pfolder): - if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( - "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): - continue - elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): - continue - elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): - continue - - # replace suffix and version tag - name = f[:-3] - if name[-1] == ".": name = name[:-4] - - plugin = self.parsePlugin(join(pfolder, f), folder, name, home) - if plugin: - plugins[name] = plugin - - if not home: - temp = self.parse(folder, plugins) - plugins.update(temp) - - return plugins - - def parseAttributes(self, filename, name, folder=""): - """ Parse attribute dict from plugin""" - data = open(filename, "rb") - content = data.read() - data.close() - - attrs = BaseAttributes() - for m in self.BUILTIN.findall(content) + self.SINGLE.findall(content) + self.MULTI.findall(content): - #replace gettext function and eval result - try: - attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) - except: - self.logDebug(folder, name, "Error when parsing: %s" % m[-1]) - self.core.print_exc() - - if not hasattr(Base, "__%s__" % m[0]): - if m[0] != "type": #TODO remove type from all plugins, its not needed - self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) - - return attrs - - def parsePlugin(self, filename, folder, name, home=None): - """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. - - :arg home: dict with plugins, of which the found one will be matched against (according version) - :returns PluginTuple""" - - attrs = self.parseAttributes(filename, name, folder) - if not attrs: return - - version = 0 - - if "version" in attrs: - try: - version = float(attrs["version"]) - except ValueError: - self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) - version = 9 #TODO remove when plugins are fixed, causing update loops - else: - self.logDebug(folder, name, "No version attribute") - - # home contains plugins from pyload root - if home and name in home: - if home[name].version >= version: - return - - if name in IGNORE or (folder, name) in IGNORE: - return - - if "pattern" in attrs and attrs["pattern"]: - try: - plugin_re = re.compile(attrs["pattern"], re.I) - except: - self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) - plugin_re = self.NO_MATCH - else: - plugin_re = self.NO_MATCH - - deps = attrs["dependencies"] - category = attrs["category"] if folder == "addons" else "" + self.loader.checkVersions() - # create plugin tuple - plugin = PluginTuple(version, plugin_re, deps, category, bool(home), filename) + # plugin matcher to overwrite some behaviour + self.matcher = [] - # These have none or their own config - if folder in ("internal", "accounts", "network"): - return plugin + def addMatcher(self, matcher, index=0): + """ Inserts matcher at given index, first position by default """ + if not isinstance(matcher, PluginMatcher): + raise TypeError("Expected type of PluginMatcher, got %s instead" % type(matcher)) - if folder == "addons" and "config" not in attrs and not attrs["internal"]: - attrs["config"] = (["activated", "bool", "Activated", False],) + if matcher in self.matcher: + self.matcher.remove(matcher) - if "config" in attrs and attrs["config"] is not None: - config = attrs["config"] - desc = attrs["description"] - expl = attrs["explanation"] - - # Convert tuples to list - config = [list(x) for x in config] - - if folder == "addons" and not attrs["internal"]: - for item in config: - if item[0] == "activated": break - else: # activated flag missing - config.insert(0, ("activated", "bool", "Activated", False)) - - # Everything that is no addon and user_context=True, is added to dict - if folder != "addons" or attrs["user_context"]: - self.user_context[name] = True - - try: - self.core.config.addConfigSection(name, name, desc, expl, config) - except: - self.logDebug(folder, name, "Invalid config %s" % config) - - return plugin + self.matcher.insert(index, matcher) + def removeMatcher(self, matcher): + """ Removes a matcher if it exists """ + if matcher in self.matcher: + self.matcher.remove(matcher) def parseUrls(self, urls): """parse plugins for given list of urls, separate to crypter and hoster""" @@ -245,7 +89,7 @@ class PluginManager: found = False for ptype, name in self.history: - if self.plugins[ptype][name].re.match(url): + if self.loader.getPlugin(ptype, name).re.match(url): res[ptype].append((url, name)) found = (ptype, name) break # need to exit this loop first @@ -257,95 +101,97 @@ class PluginManager: continue for ptype in ("crypter", "hoster"): - for name, plugin in self.plugins[ptype].iteritems(): - if plugin.re.match(url): - res[ptype].append((url, name)) - self.history.insert(0, (ptype, name)) - del self.history[10:] # cut down to size of 10 - found = True - break + for loader in self.loader: + for name, plugin in loader.getPlugins(ptype).iteritems(): + if plugin.re.match(url): + res[ptype].append((url, name)) + self.history.insert(0, (ptype, name)) + del self.history[10:] # cut down to size of 10 + found = True + break if not found: - res["hoster"].append((url, "BasePlugin")) + res["hoster"].append((url, self.DEFAULT_PLUGIN)) return res["hoster"], res["crypter"] - def getPlugins(self, type): - return self.plugins.get(type, None) + def getPlugins(self, plugin): + """ Get all plugins of a certain type in a dict """ + plugins = {} + for loader in self.loader: + plugins.update(loader.getPlugins(plugin)) + return plugins def findPlugin(self, name, pluginlist=("hoster", "crypter")): - for ptype in pluginlist: - if name in self.plugins[ptype]: - return ptype, self.plugins[ptype][name] + # TODO: use matcher + for loader in self.loader: + for plugin in pluginlist: + if loader.hasPlugin(plugin, name): + return plugin, loader.getPlugin(plugin, name) + return None, None - def getPluginModule(self, name): - """ Decprecated: return plugin module from hoster|crypter""" - self.log.debug("Deprecated method: .getPluginModule()") - type, plugin = self.findPlugin(name) - return self.loadModule(type, name) + def getPluginClass(self, name, overwrite=True): + """Gives the plugin class of a hoster or crypter plugin - def getPluginClass(self, name): - """ return plugin class from hoster|crypter, always the not overwritten one """ + :param overwrite: allow the use of overwritten plugins + """ + # TODO: use matcher type, plugin = self.findPlugin(name) return self.loadClass(type, name) - # MultiHoster will overwrite this - getPlugin = getPluginClass + def loadAttributes(self, plugin, name): + for loader in self.loader: + if loader.hasPlugin(plugin, name): + return loader.loadAttributes(plugin, name) - def loadAttributes(self, type, name): - plugin = self.plugins[type][name] - return self.parseAttributes(plugin.path, name, type) + return {} - def loadModule(self, type, name): + def loadModule(self, plugin, name): """ Returns loaded module for plugin - :param type: plugin type, subfolder of module.plugins - :param name: + :param plugin: plugin type, subfolder of module.plugins """ - plugins = self.plugins[type] - if name in plugins: - if (type, name) in self.modules: return self.modules[(type, name)] - try: - # convert path to python recognizable import - path = basename(plugins[name].path).replace(".pyc", "").replace(".py", "") - module = __import__(self.ROOT + "%s.%s" % (type, path), globals(), locals(), path) - self.modules[(type, name)] = module # cache import, maybe unneeded - return module - except Exception, e: - self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) - self.core.print_exc() - - def loadClass(self, type, name): + if (plugin, name) in self.modules: return self.modules[(plugin, name)] + for loader in self.loader: + if loader.hasPlugin(plugin, name): + try: + module = loader.loadModule(plugin, name) + # cache import + self.modules[(plugin, name)] = module + return module + except Exception, e: + self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) + self.core.print_exc() + + def loadClass(self, plugin, name): """Returns the class of a plugin with the same name""" - module = self.loadModule(type, name) + module = self.loadModule(plugin, name) if module: return getattr(module, name) def find_module(self, fullname, path=None): - #redirecting imports if necesarry - if fullname.startswith(self.ROOT) or fullname.startswith(self.LOCALROOT): #separate pyload plugins - if fullname.startswith(self.LOCALROOT): - user = 1 - else: - user = 0 #used as bool and int + #redirecting imports if necessary + for loader in self.loader: + if not fullname.startswith(loader.package): + continue + + # TODO not well tested + offset = 1 - loader.package.count(".") split = fullname.split(".") - if len(split) != 4 - user: return - type, name = split[2 - user:4 - user] + if len(split) != 4 - offset: return + plugin, name = split[2 - offset:4 - offset] - if type in self.plugins and name in self.plugins[type]: - #userplugin is a newer version - if not user and self.plugins[type][name].user: - return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name].user: + # check if a different loader than the current one has the plugin + # in this case import needs redirect + for l2 in self.loader: + if l2 is not loader and l2.hasPlugin(plugin, name): return self # TODO: Remove when all plugin imports are adapted if "module" in fullname: return self - def load_module(self, name, replace=True): if name not in sys.modules: #could be already in modules @@ -361,6 +207,7 @@ class PluginManager: self.log.debug("Old import reference detected, use %s" % name) replace = False + # TODO: this still works but does not respect other loaders if replace: if self.ROOT in name: newname = name.replace(self.ROOT, self.LOCALROOT) @@ -382,48 +229,21 @@ class PluginManager: def reloadPlugins(self, type_plugins): """ reloads and reindexes plugins """ - if not type_plugins: return False - - self.log.debug("Request reload of plugins: %s" % type_plugins) - - as_dict = {} - for t, n in type_plugins: - if t in as_dict: - as_dict[t].append(n) - else: - as_dict[t] = [n] - - # we do not reload addons or internals, would cause to much side effects - if "addons" in as_dict or "internal" in as_dict: - return False - - for type in as_dict.iterkeys(): - for plugin in as_dict[type]: - if plugin in self.plugins[type]: - if (type, plugin) in self.modules: - self.log.debug("Reloading %s" % plugin) - reload(self.modules[(type, plugin)]) - - # index re-creation - for type in ("crypter", "container", "hoster", "captcha", "accounts"): - self.plugins[type] = self.parse(type) - - if "accounts" in as_dict: #accounts needs to be reloaded - self.core.accountManager.initPlugins() - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - - return True + # TODO + # check if reloadable + # reload + # save new plugins + # update index + # reload accounts def isUserPlugin(self, plugin): """ A plugin suitable for multiple user """ - return plugin in self.user_context - - def isPluginType(self, plugin, type): - return plugin in self.plugins[type] + return any(l.isUserPlugin(plugin) for l in self.loader) def getCategory(self, plugin): - if plugin in self.plugins["addons"]: - return self.plugins["addons"][plugin].category or "addon" + plugin = self.loader.getPlugin("addons", plugin) + if plugin: + return plugin.category or "addon" def loadIcon(self, name): """ load icon for single plugin, base64 encoded""" diff --git a/pyload/api/FileApi.py b/pyload/api/FileApi.py index 2ca409165..984729b8c 100644 --- a/pyload/api/FileApi.py +++ b/pyload/api/FileApi.py @@ -74,14 +74,12 @@ class FileApi(ApiComponent): raise FileDoesNotExists(fid) return info - @RequirePerm(Permission.Download) def getFilePath(self, fid): """ Internal method to get the filepath""" info = self.getFileInfo(fid) pack = self.core.files.getPackage(info.package) return pack.getPath(), info.name - @RequirePerm(Permission.All) def findFiles(self, pattern): return self.core.files.getTree(-1, True, DownloadState.All, pattern) diff --git a/pyload/datatypes/PyFile.py b/pyload/datatypes/PyFile.py index de7288d22..3ce114beb 100644 --- a/pyload/datatypes/PyFile.py +++ b/pyload/datatypes/PyFile.py @@ -37,10 +37,11 @@ statusMap = { "downloading": 10, "temp. offline": 11, "aborted": 12, - "decrypting": 13, - "processing": 14, - "custom": 15, - "unknown": 16, + "not possible": 13, + "decrypting": 14, + "processing": 15, + "custom": 16, + "unknown": 17, } @@ -138,7 +139,7 @@ class PyFile(object): def initPlugin(self): """ inits plugin instance """ if not self.plugin: - self.pluginclass = self.m.core.pluginManager.getPlugin(self.pluginname) + self.pluginclass = self.m.core.pluginManager.getPluginClass(self.pluginname) self.plugin = self.pluginclass(self) @read_lock @@ -160,7 +161,7 @@ class PyFile(object): self.setStatus(status) def getStatusName(self): - if self.status not in (13, 14) or not self.statusname: + if self.status not in (15, 16) or not self.statusname: return self.m.statusMsg[self.status] else: return self.statusname diff --git a/pyload/plugins/MultiHoster.py b/pyload/plugins/MultiHoster.py index bc7a0de75..6b48e99fb 100644 --- a/pyload/plugins/MultiHoster.py +++ b/pyload/plugins/MultiHoster.py @@ -11,6 +11,7 @@ def normalize(domain): """ Normalize domain/plugin name, so they are comparable """ return remove_chars(domain.strip().lower(), "-.") + #noinspection PyUnresolvedReferences class MultiHoster(Account): """ diff --git a/pyload/plugins/addons/MultiHoster.py b/pyload/plugins/addons/MultiHoster.py index 21529eb1a..2d4029dd6 100644 --- a/pyload/plugins/addons/MultiHoster.py +++ b/pyload/plugins/addons/MultiHoster.py @@ -6,18 +6,18 @@ from types import MethodType from pyload.plugins.MultiHoster import MultiHoster as MultiHosterAccount, normalize from pyload.plugins.Addon import Addon, AddEventListener -from pyload.PluginManager import PluginTuple +from pyload.PluginManager import PluginMatcher - -class MultiHoster(Addon): +class MultiHoster(Addon, PluginMatcher): __version__ = "0.1" __internal__ = True __description__ = "Gives ability to use MultiHoster services." __config__ = [] - __author_mail__ = ("pyLoad Team",) + __author__ = ("pyLoad Team",) __author_mail__ = ("support@pyload.org",) #TODO: multiple accounts - multihoster / config options + # TODO: rewrite for new plugin manager def init(self): @@ -90,17 +90,8 @@ class MultiHoster(Addon): def activate(self): self.refreshAccounts() - # new method for plugin manager - def getPlugin(self2, name): - if name in self.plugins: - return self.plugins[name] - return self2.getPluginClass(name) - - pm = self.core.pluginManager - pm.getPlugin = MethodType(getPlugin, pm, object) - + self.core.pluginManager.addMatcher(self) def deactivate(self): - #restore state - pm = self.core.pluginManager - pm.getPlugin = pm.getPluginClass + + self.core.pluginManager.removeMatcher(self) diff --git a/pyload/remote/apitypes.py b/pyload/remote/apitypes.py index 53d2de6d2..287a5f096 100644 --- a/pyload/remote/apitypes.py +++ b/pyload/remote/apitypes.py @@ -33,10 +33,11 @@ class DownloadStatus: Downloading = 10 TempOffline = 11 Aborted = 12 - Decrypting = 13 - Processing = 14 - Custom = 15 - Unknown = 16 + NotPossible = 13 + Decrypting = 14 + Processing = 15 + Custom = 16 + Unknown = 17 class FileStatus: Ok = 0 diff --git a/pyload/remote/pyload.thrift b/pyload/remote/pyload.thrift index 3d0f201e7..9bcc2ce89 100644 --- a/pyload/remote/pyload.thrift +++ b/pyload/remote/pyload.thrift @@ -11,9 +11,8 @@ typedef list<string> LinkList typedef string PluginName typedef string JSONString -// NA - Not Available enum DownloadStatus { - NA, + NA, // No downloads status set Offline, Online, Queued, @@ -26,6 +25,7 @@ enum DownloadStatus { Downloading, TempOffline, Aborted, + NotPossible, Decrypting, Processing, Custom, diff --git a/pyload/threads/InfoThread.py b/pyload/threads/InfoThread.py index f516d2cca..f39ac41f2 100644 --- a/pyload/threads/InfoThread.py +++ b/pyload/threads/InfoThread.py @@ -56,7 +56,7 @@ class InfoThread(DecrypterThread): cb = self.updateDB if self.pid > 1 else self.updateResult for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPluginModule(pluginname) + plugin = self.m.core.pluginManager.loadModule("hoster", pluginname) klass = self.m.core.pluginManager.getPluginClass(pluginname) if has_method(klass, "getInfo"): self.fetchForPlugin(klass, urls, cb) diff --git a/pyload/utils/PluginLoader.py b/pyload/utils/PluginLoader.py new file mode 100644 index 000000000..8caac9528 --- /dev/null +++ b/pyload/utils/PluginLoader.py @@ -0,0 +1,296 @@ +############################################################################### +# Copyright(c) 2008-2013 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + +import re + +from os import listdir, makedirs +from os.path import isfile, join, exists, basename +from sys import version_info +from time import time +from collections import defaultdict +from logging import getLogger + +from pyload.lib.SafeEval import const_eval as literal_eval +from pyload.plugins.Base import Base + +from new_collections import namedtuple + +PluginTuple = namedtuple("PluginTuple", "version re deps category user path") + + +class BaseAttributes(defaultdict): + """ Dictionary that loads defaults values from Base object """ + + def __missing__(self, key): + attr = "__%s__" % key + if not hasattr(Base, attr): + return defaultdict.__missing__(self, key) + + return getattr(Base, attr) + +class LoaderFactory: + """ Container for multiple plugin loaders """ + + def __init__(self, *loader): + self.loader = list(loader) + + def __iter__(self): + return self.loader.__iter__() + + + def checkVersions(self): + """ Reduces every plugin loader to the globally newest version. + Afterwards every plugin is unique across all available loader """ + for plugin_type in self.loader[0].iterTypes(): + for loader in self.loader: + # iterate all plugins + for plugin, info in loader.getPlugins(plugin_type).iteritems(): + # now iterate all other loaders + for l2 in self.loader: + if l2 is not loader: + l2.removePlugin(plugin_type, plugin, info.version) + + def getPlugin(self, plugin, name): + """ retrieve a plugin from an available loader """ + for loader in self.loader: + if loader.hasPlugin(plugin, name): + return loader.getPlugin(plugin, name) + + +class PluginLoader: + """ + Class to provide and load plugins from the file-system + """ + TYPES = ("crypter", "hoster", "accounts", "addons", "network", "internal") + + BUILTIN = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(True|False|None|[0-9x.]+)', re.I) + SINGLE = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?<!")"(?!")|\'|\().*(?:(?<!")"(?!")|\'|\)))', + re.I) + # note the nongreedy character: that means we can not embed list and dicts + MULTI = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I) + + NO_MATCH = re.compile(r'^no_match$') + + def __init__(self, path, package, config): + self.path = path + self.package = package + self.config = config + self.log = getLogger("log") + self.plugins = {} + + self.createIndex() + + def logDebug(self, plugin, name, msg): + self.log.debug("Plugin %s | %s: %s" % (plugin, name, msg)) + + def createIndex(self): + """create information for all plugins available""" + + if not exists(self.path): + makedirs(self.path) + if not exists(join(self.path, "__init__.py")): + f = open(join(self.path, "__init__.py"), "wb") + f.close() + + a = time() + for plugin in self.TYPES: + self.plugins[plugin] = self.parse(plugin) + + self.log.debug("Created index of plugins for %s in %.2f ms", self.path, (time() - a) * 1000) + + def parse(self, folder): + """ Analyze and parses all plugins in folder """ + plugins = {} + pfolder = join(self.path, folder) + if not exists(pfolder): + makedirs(pfolder) + if not exists(join(pfolder, "__init__.py")): + f = open(join(pfolder, "__init__.py"), "wb") + f.close() + + for f in listdir(pfolder): + if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( + "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): + if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): + continue + elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): + continue + elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): + continue + + # replace suffix and version tag + name = f[:-3] + if name[-1] == ".": name = name[:-4] + + plugin = self.parsePlugin(join(pfolder, f), folder, name) + if plugin: + plugins[name] = plugin + + return plugins + + def parseAttributes(self, filename, name, folder=""): + """ Parse attribute dict from plugin""" + data = open(filename, "rb") + content = data.read() + data.close() + + attrs = BaseAttributes() + for m in self.BUILTIN.findall(content) + self.SINGLE.findall(content) + self.MULTI.findall(content): + #replace gettext function and eval result + try: + attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) + except Exception, e: + self.logDebug(folder, name, "Error when parsing: %s" % m[-1]) + self.log.debug(str(e)) + + if not hasattr(Base, "__%s__" % m[0]): + if m[0] != "type": #TODO remove type from all plugins, its not needed + self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) + + return attrs + + def parsePlugin(self, filename, folder, name): + """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. + + :arg home: dict with plugins, of which the found one will be matched against (according version) + :returns PluginTuple""" + + attrs = self.parseAttributes(filename, name, folder) + if not attrs: return + + version = 0 + if "version" in attrs: + try: + version = float(attrs["version"]) + except ValueError: + self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) + version = 9 #TODO remove when plugins are fixed, causing update loops + else: + self.logDebug(folder, name, "No version attribute") + + if "pattern" in attrs and attrs["pattern"]: + try: + plugin_re = re.compile(attrs["pattern"], re.I) + except: + self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) + plugin_re = self.NO_MATCH + else: + plugin_re = self.NO_MATCH + + deps = attrs["dependencies"] + category = attrs["category"] if folder == "addons" else "" + + # create plugin tuple + # user_context=True is the default for non addons plugins + plugin = PluginTuple(version, plugin_re, deps, category, + bool(folder != "addons" or attrs["user_context"]), filename) + + # These have none or their own config + if folder in ("internal", "accounts", "network"): + return plugin + + if folder == "addons" and "config" not in attrs and not attrs["internal"]: + attrs["config"] = (["activated", "bool", "Activated", False],) + + if "config" in attrs and attrs["config"] is not None: + config = attrs["config"] + desc = attrs["description"] + expl = attrs["explanation"] + + # Convert tuples to list + config = [list(x) for x in config] + + if folder == "addons" and not attrs["internal"]: + for item in config: + if item[0] == "activated": break + else: # activated flag missing + config.insert(0, ("activated", "bool", "Activated", False)) + + try: + self.config.addConfigSection(name, name, desc, expl, config) + except: + self.logDebug(folder, name, "Invalid config %s" % config) + + return plugin + + def iterPlugins(self): + """ Iterates over all plugins returning (type, name, info) with info as PluginTuple """ + + for plugin, data in self.plugins.iteritems(): + for name, info in data.iteritems(): + yield plugin, name, info + + def iterTypes(self): + """ Iterate over the available plugin types """ + + for plugin in self.plugins.iterkeys(): + yield plugin + + def hasPlugin(self, plugin, name): + """ Check if certain plugin is available """ + return plugin in self.plugins and name in self.plugins[plugin] + + def getPlugin(self, plugin, name): + """ Return plugin info for a single entity """ + try: + return self.plugins[plugin][name] + except KeyError: + return None + + def getPlugins(self, plugin): + """ Return all plugins of given plugin type """ + return self.plugins[plugin] + + def removePlugin(self, plugin, name, available_version=None): + """ Removes a plugin from the index. + Optionally only when its version is below or equal the available one + """ + try: + if available_version is not None: + if self.plugins[plugin][name] <= available_version: + del self.plugins[plugin][name] + else: + del self.plugins[plugin][name] + + # no errors are thrown if the plugin didn't existed + except KeyError: + return + + def isUserPlugin(self, name): + """ Determine if given plugin name is enable for user_context in any plugin type """ + for plugins in self.plugins: + if name in plugins and name[plugins].user: + return True + + return False + + def savePlugin(self, content): + """ Saves a plugin to disk """ + + def loadModule(self, plugin, name): + """ Returns loaded module for plugin + + :param plugin: plugin type, subfolder of module.plugins + :raises Exception: Everything could go wrong, failures needs to be catched + """ + plugins = self.plugins[plugin] + # convert path to python recognizable import + path = basename(plugins[name].path).replace(".pyc", "").replace(".py", "") + module = __import__(self.package + ".%s.%s" % (plugin, path), globals(), locals(), path) + return module + + def loadAttributes(self, plugin, name): + """ Same as `parseAttributes` for already indexed plugins """ + return self.parseAttributes(self.plugins[plugin][name].path, name, plugin)
\ No newline at end of file diff --git a/pyload/web/app/scripts/utils/apitypes.js b/pyload/web/app/scripts/utils/apitypes.js index 23d87def0..cb094a05b 100644 --- a/pyload/web/app/scripts/utils/apitypes.js +++ b/pyload/web/app/scripts/utils/apitypes.js @@ -4,7 +4,7 @@ define([], function() { 'use strict'; return { DownloadState: {'Failed': 3, 'All': 0, 'Unmanaged': 4, 'Finished': 1, 'Unfinished': 2}, - DownloadStatus: {'Downloading': 10, 'NA': 0, 'Processing': 14, 'Waiting': 9, 'Decrypting': 13, 'Paused': 4, 'Failed': 7, 'Finished': 5, 'Skipped': 6, 'Unknown': 16, 'Aborted': 12, 'Online': 2, 'TempOffline': 11, 'Offline': 1, 'Custom': 15, 'Starting': 8, 'Queued': 3}, + DownloadStatus: {'NotPossible': 13, 'Downloading': 10, 'NA': 0, 'Processing': 15, 'Waiting': 9, 'Decrypting': 14, 'Paused': 4, 'Failed': 7, 'Finished': 5, 'Skipped': 6, 'Unknown': 17, 'Aborted': 12, 'Online': 2, 'TempOffline': 11, 'Offline': 1, 'Custom': 16, 'Starting': 8, 'Queued': 3}, FileStatus: {'Remote': 2, 'Ok': 0, 'Missing': 1}, InputType: {'PluginList': 13, 'Multiple': 11, 'Int': 2, 'NA': 0, 'Time': 7, 'List': 12, 'Bool': 8, 'File': 3, 'Text': 1, 'Table': 14, 'Folder': 4, 'Password': 6, 'Click': 9, 'Select': 10, 'Textbox': 5}, Interaction: {'Captcha': 2, 'All': 0, 'Query': 4, 'Notification': 1}, diff --git a/pyload/web/pyload_app.py b/pyload/web/pyload_app.py index 1c89e2ada..50d9b9731 100644 --- a/pyload/web/pyload_app.py +++ b/pyload/web/pyload_app.py @@ -43,6 +43,7 @@ def serve_icon(path): @route("/download/:fid") @login_required('Download') def download(fid, api): + # TODO: check owner ship path, name = api.getFilePath(fid) return static_file(name, path, download=True) |