From caeacf013f06784ce8dbf0fa9d05c51606aba36a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 10 Oct 2015 12:20:38 +0200 Subject: [UpdateManager] Make really faster! --- module/plugins/hooks/UpdateManager.py | 251 ++++++++++++++++++++-------------- 1 file changed, 145 insertions(+), 106 deletions(-) diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py index fdaa928eb..34b992d5b 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/hooks/UpdateManager.py @@ -16,7 +16,7 @@ from module.utils import fs_encode, save_join as fs_join class UpdateManager(Addon): __name__ = "UpdateManager" __type__ = "hook" - __version__ = "0.58" + __version__ = "1.00" __status__ = "testing" __config__ = [("activated" , "bool", "Activated" , True ), @@ -32,7 +32,10 @@ class UpdateManager(Addon): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - SERVER_URL = "http://updatemanager.pyload.org" + _VERSION = re.compile(r'__version__.*=.*("|\')([\d.]+)') + + SERVER_URL = "http://updatemanager.pyload.org" + PERIODICAL_INTERVAL = 3 * 60 * 60 #: 3 hours @@ -40,6 +43,7 @@ class UpdateManager(Addon): if self.checkonstart: self.pyload.api.pauseServer() self.update() + if self.do_restart is False: self.pyload.api.unpauseServer() @@ -47,7 +51,7 @@ class UpdateManager(Addon): def init(self): - self.info = {'pyload': False, 'version': None, 'plugins': False, 'last_check': time.time()} + self.info = {'pyload': False, 'plugins': False, 'last_check': time.time()} self.mtimes = {} #: Store modification time for each plugin self.event_map = {'allDownloadsProcessed': "all_downloads_processed"} @@ -93,14 +97,12 @@ class UpdateManager(Addon): """ Reload and reindex all modified plugins """ + reloads = [] modules = filter( lambda m: m and (m.__name__.startswith("module.plugins.") or m.__name__.startswith("userplugins.")) and m.__name__.count(".") >= 2, sys.modules.values() ) - - reloads = [] - for m in modules: root, type, name = m.__name__.rsplit(".", 2) id = (type, name) @@ -113,6 +115,7 @@ class UpdateManager(Addon): if id not in self.mtimes: self.mtimes[id] = mtime + elif self.mtimes[id] < mtime: reloads.append(id) self.mtimes[id] = mtime @@ -120,14 +123,25 @@ class UpdateManager(Addon): return True if self.pyload.pluginManager.reloadPlugins(reloads) else False - def server_response(self): + def server_response(self, line=None): try: - return self.load(self.SERVER_URL, - get={'v': self.pyload.api.getServerVersion()}).splitlines() + html = self.load(self.SERVER_URL, + get={'v': self.pyload.api.getServerVersion()}) except Exception: self.log_warning(_("Unable to retrieve server to get updates")) + else: + res = html.splitlines() + + if line is not None: + try: + res = res[line] + except IndexError: + res = None + + return res + @Expose @threaded @@ -135,35 +149,34 @@ class UpdateManager(Addon): """ Check for updates """ - if self._update() == 2 and self.get_config('autorestart'): - if not self.pyload.api.statusDownloads(): - self.pyload.api.restart() - else: - self.do_restart = True - self.log_warning(_("Downloads are active, will restart once the download is done")) - self.pyload.api.pauseServer() + if self._update() is not 2 or not self.get_config('autorestart'): + return + + if not self.pyload.api.statusDownloads(): + self.pyload.api.restart() + else: + self.do_restart = True + self.log_warning(_("Downloads are active, will restart once the download is done")) + self.pyload.api.pauseServer() def _update(self): - data = self.server_response() + newversion = self.server_response(0) + self.info['pyload'] = False self.info['last_check'] = time.time() - if not data: + if not newversion: exitcode = 0 - elif data[0] == "None": + elif newversion == "None": self.log_info(_("No new pyLoad version available")) - exitcode = self._update_plugins(data[1:]) - - elif onlyplugin: - exitcode = 0 + exitcode = self.update_plugins() else: - self.log_info(_("*** New pyLoad Version %s available ***") % data[0]) + self.log_info(_("*** New pyLoad Version %s available ***") % newversion) self.log_info(_("*** Get it here: https://github.com/pyload/pyload/releases ***")) - self.info['pyload'] = True - self.info['version'] = data[0] + self.info['pyload'] = True exitcode = 3 #: Exit codes: @@ -174,64 +187,101 @@ class UpdateManager(Addon): return exitcode - def _update_plugins(self, data): - """ - Check for plugin updates - """ - exitcode = 0 - updated = [] + @Expose + def update_plugins(self): + updates = self.server_response() + + if not updates or updates[0] != "None": + return 0 - url = data[0] - schema = data[1].split('|') + updated = self._update_plugins(updates) + + if updated: + self.log_info(_("*** Plugins updated ***")) - VERSION = re.compile(r'__version__.*=.*("|\')([\d.]+)') + if self.pyload.pluginManager.reloadPlugins(updated): + exitcode = 1 + else: + self.log_warning(_("You have to restart pyLoad to reload the updated plugins")) + self.info['plugins'] = True + exitcode = 2 - if "BLACKLIST" in data: - blacklist = data[data.index('BLACKLIST') + 1:] - updatelist = data[2:data.index('BLACKLIST')] + self.manager.dispatchEvent("plugin_updated", updated) + else: + self.log_info(_("*** No plugin updates available ***")) + exitcode = 0 + + #: Exit codes: + #: 0 = No plugin updated + #: 1 = Plugins updated + #: 2 = Plugins updated, but restart required + return exitcode + + + def parse_list(self, list): + schema = list[2].split('|') + + if "BLACKLIST" in list: + blacklist = list[list.index('BLACKLIST') + 1:] + updatelist = list[3:list.index('BLACKLIST')] else: blacklist = [] - updatelist = data[2:] + updatelist = list[3:] - updatelist = [dict(zip(schema, x.split('|'))) for x in updatelist] - blacklist = [dict(zip(schema, x.split('|'))) for x in blacklist] + for l in updatelist, blacklist: + nl = [] + for line in l: + d = dict(zip(schema, line.split('|'))) + d['name'] = d['name'].rsplit('.py', 1)[0] + d['type'] = d['type'].rstrip('s') + nl.append(d) + l[:] = nl - if blacklist: - type_plugins = [(plugin['type'], plugin['name'].rsplit('.', 1)[0]) for plugin in blacklist] + updatelist = sorted(updatelist, key=operator.itemgetter("type", "name")) + blacklist = sorted(blacklist, key=operator.itemgetter("type", "name")) - #: Protect UpdateManager from self-removing - try: - type_plugins.remove(("hook", "UpdateManager")) - except ValueError: - pass + return updatelist, blacklist + + + def _update_plugins(self, updates): + """ + Check for plugin updates + """ + updated = [] + + updatelist, blacklist = self.parse_list(updates) + + url = updates[1] - for t, n in type_plugins: - for idx, plugin in enumerate(updatelist): - if n is plugin['name'] and t is plugin['type']: - updatelist.pop(idx) - break + if blacklist: + #@NOTE: Protect UpdateManager from self-removing + type_plugins = [(plugin['type'], plugin['name']) for plugin in blacklist \ + if plugin['name'] is not self.__name__ and plugin['type'] is not self.__type__] + + c = 1 + l = len(type_plugins) + for idx, plugin in enumerate(updatelist): + if c > l: + break + name = plugin['name'] + type = plugin['type'] + for t, n in type_plugins: + if n != name or t != type: + continue + updatelist.pop(idx) + c += 1 + break - for t, n in self.remove_plugins(sorted(type_plugins)): + for t, n in self.remove_plugins(type_plugins): self.log_info(_("Removed blacklisted plugin: %(type)s %(name)s") % { 'type': t.upper(), 'name': n, }) - for plugin in sorted(updatelist, key=operator.itemgetter("type", "name")): - filename = plugin['name'] - prefix = plugin['type'] - version = plugin['version'] - - if filename.endswith(".pyc"): - name = filename[:filename.find("_")] - else: - name = filename.replace(".py", "") - - #@TODO: Remove in 0.4.10 - if prefix.endswith("s"): - type = prefix[:-1] - else: - type = prefix + for plugin in updatelist: + name = plugin['name'] + type = plugin['type'] + version = plugin['version'] plugins = getattr(self.pyload.pluginManager, "%sPlugins" % type) @@ -239,50 +289,38 @@ class UpdateManager(Addon): newver = float(version) if not oldver: - msg = "New plugin: [%(type)s] %(name)s (v%(newver).2f)" + msg = "New plugin: %(type)s %(name)s (v%(newver).2f)" elif newver > oldver: - msg = "New version of plugin: [%(type)s] %(name)s (v%(oldver).2f -> v%(newver).2f)" + msg = "New version of plugin: %(type)s %(name)s (v%(oldver).2f -> v%(newver).2f)" else: continue - self.log_info(_(msg) % {'type' : type, - 'name' : name, - 'oldver': oldver, - 'newver': newver}) + self.log_info(_(msg) % {'type' : type.upper(), + 'name' : name, + 'oldver': oldver, + 'newver': newver}) try: - content = self.load(url % plugin, decode=False) - m = VERSION.search(content) + content = self.load(url % plugin + ".py", decode=False) + m = self._VERSION.search(content) if m and m.group(2) == version: - with open(fs_join("userplugins", prefix, filename), "wb") as f: + #@TODO: Remove in 0.4.10 + if type in ("account", "hook"): + folder = type + "s" + else: + folder = type + + with open(fs_join("userplugins", folder, name + ".py"), "wb") as f: f.write(fs_encode(content)) - updated.append((prefix, name)) + updated.append((type, name)) else: raise Exception(_("Version mismatch")) except Exception, e: - self.log_error(_("Error updating plugin: [%s] %s") % (type, name), e) + self.log_error(_("Error updating plugin: %s %s") % (type.upper(), name), e) - if updated: - self.log_info(_("*** Plugins updated ***")) - - if self.pyload.pluginManager.reloadPlugins(updated): - exitcode = 1 - else: - self.log_warning(_("You have to restart pyLoad to reload the updated plugins")) - self.info['plugins'] = True - exitcode = 2 - - self.manager.dispatchEvent("plugin_updated", updated) - else: - self.log_info(_("No plugin updates available")) - - #: Exit codes: - #: 0 = No plugin updated - #: 1 = Plugins updated - #: 2 = Plugins updated, but restart required - return exitcode + return updated #: Deprecated method, use `remove_plugins` instead @@ -309,16 +347,17 @@ class UpdateManager(Addon): for type, name in type_plugins: rootplugins = os.path.join(pypath, "module", "plugins") - for dir in ("userplugins", rootplugins): - py_filename = fs_join(dir, type, name + ".py") - - #@TODO: Remove in 0.4.10 - if type == "hook": - py_filename = fs_join(dir, "hooks" , name + ".py") + #@TODO: Remove in 0.4.10 + if type in ("account", "hook"): + folder = type + "s" + else: + folder = type + for dir in ("userplugins", rootplugins): + py_filename = fs_join(dir, folder, name + ".py") pyc_filename = py_filename + "c" - if type == "hook": + if type is "hook": try: self.manager.deactivateHook(name) -- cgit v1.2.3