diff options
Diffstat (limited to 'pyload/manager')
| -rw-r--r-- | pyload/manager/AccountManager.py | 100 | ||||
| -rw-r--r-- | pyload/manager/AddonManager.py | 4 | ||||
| -rw-r--r-- | pyload/manager/CaptchaManager.py | 16 | ||||
| -rw-r--r-- | pyload/manager/PluginManager.py | 222 | ||||
| -rw-r--r-- | pyload/manager/thread/PluginThread.py | 52 |
5 files changed, 219 insertions, 175 deletions
diff --git a/pyload/manager/AccountManager.py b/pyload/manager/AccountManager.py index ec092d740..3de656376 100644 --- a/pyload/manager/AccountManager.py +++ b/pyload/manager/AccountManager.py @@ -14,7 +14,7 @@ ACC_VERSION = 1 class AccountManager: """manages all accounts""" - #-------------------------------------------------------------------------- + #---------------------------------------------------------------------- def __init__(self, core): """Constructor""" @@ -24,6 +24,7 @@ class AccountManager: self.initPlugins() self.saveAccounts() # save to add categories to conf + def initPlugins(self): self.accounts = {} # key = ( plugin ) self.plugins = {} @@ -31,19 +32,25 @@ class AccountManager: self.initAccountPlugins() self.loadAccounts() + def getAccountPlugin(self, plugin): """get account instance for plugin or None if anonymous""" - if plugin in self.accounts: - if plugin not in self.plugins: - try: - self.plugins[plugin] = self.core.pluginManager.loadClass("account", plugin)(self, self.accounts[plugin]) - except TypeError: # The account class no longer exists (blacklisted plugin). Skipping the account to avoid crash - return None - - return self.plugins[plugin] - else: + try: + if plugin in self.accounts: + if plugin not in self.plugins: + klass = self.core.pluginManager.loadClass("accounts", plugin) + if klass: + self.plugins[plugin] = klass(self, self.accounts[plugin]) + else: #@NOTE: The account class no longer exists (blacklisted plugin). Skipping the account to avoid crash + raise + + return self.plugins[plugin] + else: + raise + except: return None + def getAccountPlugins(self): """ get all account instances""" @@ -53,26 +60,26 @@ class AccountManager: return plugins - #-------------------------------------------------------------------------- + + #---------------------------------------------------------------------- def loadAccounts(self): """loads all accounts available""" - if not exists("accounts.conf"): - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - - f = open("accounts.conf", "rb") - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - f.close() - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - self.core.log.warning(_("Account settings deleted, due to new config format.")) + try: + with open("accounts.conf", "a+") as f: + content = f.readlines() + version = content[0].split(":")[1].strip() if content else "" + + if not version or int(version) < ACC_VERSION: + copy("accounts.conf", "accounts.backup") + f.seek(0) + f.write("version: " + str(ACC_VERSION)) + + self.core.log.warning(_("Account settings deleted, due to new config format")) + return + + except IOError, e: + self.logError(e) return plugin = "" @@ -100,32 +107,38 @@ class AccountManager: name, sep, pw = line.partition(":") self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - #-------------------------------------------------------------------------- + + #---------------------------------------------------------------------- def saveAccounts(self): """save all account information""" - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION) + "\n") + try: + with open("accounts.conf", "wb") as f: + f.write("version: " + str(ACC_VERSION) + "\n") + + for plugin, accounts in self.accounts.iteritems(): + f.write("\n") + f.write(plugin + ":\n") + + for name,data in accounts.iteritems(): + f.write("\n\t%s:%s\n" % (name,data['password']) ) + if data['options']: + for option, values in data['options'].iteritems(): + f.write("\t@%s %s\n" % (option, " ".join(values))) - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin+":\n") + chmod(f.name, 0600) - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data['password']) ) - if data['options']: - for option, values in data['options'].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) + except Exception, e: + self.logError(e) - f.close() - chmod(f.name, 0600) - #-------------------------------------------------------------------------- + #---------------------------------------------------------------------- def initAccountPlugins(self): """init names""" for name in self.core.pluginManager.getAccountPlugins(): self.accounts[name] = {} + @lock def updateAccount(self, plugin , user, password=None, options={}): """add or update account""" @@ -137,6 +150,7 @@ class AccountManager: self.saveAccounts() if updated: p.scheduleRefresh(user, force=False) + @lock def removeAccount(self, plugin, user): """remove account""" @@ -147,6 +161,7 @@ class AccountManager: self.saveAccounts() + @lock def getAccountInfos(self, force=True, refresh=False): data = {} @@ -160,7 +175,7 @@ class AccountManager: p = self.getAccountPlugin(p) if p: data[p.__name__] = p.getAllAccounts(force) - else: # When an account has been skipped, p is None + else: #@NOTE: When an account has been skipped, p is None data[p] = [] else: data[p] = [] @@ -168,6 +183,7 @@ class AccountManager: self.core.pullManager.addEvent(e) return data + def sendChange(self): e = AccountUpdateEvent() self.core.pullManager.addEvent(e) diff --git a/pyload/manager/AddonManager.py b/pyload/manager/AddonManager.py index a81e0a74f..f8a16d807 100644 --- a/pyload/manager/AddonManager.py +++ b/pyload/manager/AddonManager.py @@ -141,8 +141,8 @@ class AddonManager: if self.core.debug: traceback.print_exc() - self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active))) - self.log.info(_("Deactivate plugins: %s") % ", ".join(sorted(deactive))) + self.log.info(_("Activated addons: %s") % ", ".join(sorted(active))) + self.log.info(_("Deactivated addons: %s") % ", ".join(sorted(deactive))) self.plugins = plugins diff --git a/pyload/manager/CaptchaManager.py b/pyload/manager/CaptchaManager.py index 06f1347fc..833e01a56 100644 --- a/pyload/manager/CaptchaManager.py +++ b/pyload/manager/CaptchaManager.py @@ -58,11 +58,11 @@ class CaptchaManager: self.lock.release() return None - def handleCaptcha(self, task): + def handleCaptcha(self, task, timeout=50): cli = self.core.isClientConnected() if cli: #client connected -> should solve the captcha - task.setWaiting(50) #wait 50 sec for response + task.setWaiting(timeout) #wait 50 sec for response for plugin in self.core.addonManager.activePlugins(): try: @@ -125,10 +125,10 @@ class CaptchaTask: self.status = "waiting" def isWaiting(self): - if self.result or self.error or time() > self.waitUntil: + if self.result or self.error or self.timedOut(): return False - - return True + else: + return True def isTextual(self): """ returns if text is written on the captcha """ @@ -149,10 +149,12 @@ class CaptchaTask: def invalid(self): """ indicates the captcha was not correct """ - [x.captchaInvalid(self) for x in self.handler] + for x in self.handler: + x.captchaInvalid(self) def correct(self): - [x.captchaCorrect(self) for x in self.handler] + for x in self.handler: + x.captchaCorrect(self) def __str__(self): return "<CaptchaTask '%s'>" % self.id diff --git a/pyload/manager/PluginManager.py b/pyload/manager/PluginManager.py index 38edfee7e..cbba0b67f 100644 --- a/pyload/manager/PluginManager.py +++ b/pyload/manager/PluginManager.py @@ -11,18 +11,16 @@ from traceback import print_exc from SafeEval import const_eval as literal_eval -from pyload.config.Parser import IGNORE - class PluginManager: ROOT = "pyload.plugins." USERROOT = "userplugins." TYPES = ("account", "addon", "base", "container", "crypter", "hook", "hoster", "internal", "ocr") - PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') - VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)') - CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) - DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') + PATTERN = re.compile(r'__pattern__\s*=\s*u?r("|\')([^"\']+)') + VERSION = re.compile(r'__version__\s*=\s*("|\')([\d.]+)') + CONFIG = re.compile(r'__config__\s*=\s*\[([^\]]+)', re.M) + DESC = re.compile(r'__description__\s*=\s*("|"""|\')([^"\']+)') def __init__(self, core): @@ -43,42 +41,40 @@ class PluginManager: sys.path.append(abspath("")) - if not exists("userplugins"): - makedirs("userplugins") - if not exists(join("userplugins", "__init__.py")): - f = open(join("userplugins", "__init__.py"), "wb") - f.close() - - self.plugins['account'] = self.accountPlugins = self.parse("account") - self.plugins['addon'] = self.addonPlugins = self.parse("addon") - self.plugins['base'] = self.basePlugins = self.parse("base") - self.plugins['container'] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins['crypter'] = self.crypterPlugins = self.parse("crypter", pattern=True) - # self.plugins['hook'] = self.hookPlugins = self.parse("hook") - self.plugins['hoster'] = self.hosterPlugins = self.parse("hoster", pattern=True) - self.plugins['internal'] = self.internalPlugins = self.parse("internal") - self.plugins['ocr'] = self.ocrPlugins = self.parse("ocr") - - self.log.debug("created index of plugins") - - def parse(self, folder, pattern=False, home={}): + #@NOTE: In 0.4.10 directory "accounts" changes to "account" and "hooks" changes to "addon" + self.plugins['accounts'] = self.accountPlugins = self.parse("accounts") + self.plugins['hooks'] = self.hookPlugins = self.parse("hooks") + + for type in set(self.TYPES) - set(('accounts', 'hooks')): + self.plugins[type] = self.parse(type) + setattr(self, "%sPlugins" % type, self.plugins[type]) + + self.log.debug("Created index of plugins") + + + def parse(self, folder, rootplugins={}): """ returns dict with information home contains parsed plugins from pyload. - - { - name : {path, version, config, (pattern, re), (plugin, class)} - } - """ + plugins = {} - if home: - pfolder = join("userplugins", folder) - if not exists(pfolder): - makedirs(pfolder) - if not exists(join(pfolder, "__init__.py")): - f = open(join(pfolder, "__init__.py"), "wb") - f.close() + + if rootplugins: + try: + pfolder = join("userplugins", folder) + if not exists(pfolder): + makedirs(pfolder) + + for ifile in (join("userplugins", "__init__.py"), + join(pfolder, "__init__.py")): + if not exists(ifile): + f = open(ifile, "wb") + f.close() + + except IOError, e: + self.logCritical(e) + return rootplugins else: pfolder = join(pypath, "pyload", "plugins", folder) @@ -86,19 +82,27 @@ class PluginManager: for f in listdir(pfolder): if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - data = open(join(pfolder, f)) - content = data.read() - data.close() - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): + try: + with open(join(pfolder, f)) as data: + content = data.read() + + except IOError, e: + self.logError(e) continue - elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): + + if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): #@TODO: Remove in 0.4.10 + continue + + elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): #@TODO: Remove in 0.4.10 continue - elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): + + elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): #@TODO: Remove in 0.4.10 continue name = f[:-3] - if name[-1] == ".": name = name[:-4] + if name[-1] == ".": + name = name[:-4] version = self.VERSION.findall(content) if version: @@ -106,64 +110,54 @@ class PluginManager: else: version = 0 - # home contains plugins from pyload root - if home and name in home: - if home[name]['v'] >= version: + if rootplugins and name in rootplugins: + if rootplugins[name]['version'] >= version: continue - if name in IGNORE or (folder, name) in IGNORE: - continue - plugins[name] = {} - plugins[name]['v'] = version + plugins[name]['version'] = version module = f.replace(".pyc", "").replace(".py", "") # the plugin is loaded from user directory - plugins[name]['user'] = True if home else False + plugins[name]['user'] = True if rootplugins else False plugins[name]['name'] = module - if pattern: - pattern = self.PATTERN.findall(content) - - if pattern: - pattern = pattern[0][1] - else: - pattern = "^unmachtable$" + pattern = self.PATTERN.findall(content) - plugins[name]['pattern'] = pattern + if pattern: + pattern = pattern[0][1] try: - plugins[name]['re'] = re.compile(pattern) + regexp = re.compile(pattern) except: - self.log.error(_("%s has a invalid pattern.") % name) + self.log.error(_("%s has a invalid pattern") % name) + pattern = r'^unmatchable$' + regexp = re.compile(pattern) + plugins[name]['pattern'] = pattern + plugins[name]['re'] = regexp # internals have no config if folder == "internal": - self.config.deleteConfig(name) + self.core.config.deleteConfig(name) continue config = self.CONFIG.findall(content) if config: - config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] + try: + config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) + desc = self.DESC.findall(content) + desc = desc[0][1] if desc else "" - if folder == "addon": - append = True - for item in config: - if item[0] == "activated": append = False + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] - # activated flag missing - if append: config.append(["activated", "bool", "Activated", False]) + if folder not in ("accounts", "internal") and not [True for item in config if item[0] == "activated"]: + config.insert(0, ["activated", "bool", "Activated", False if folder == "addon" else True]) - try: self.config.addPluginConfig(name, config, desc) except: self.log.error("Invalid config in %s: %s" % (name, config)) @@ -178,9 +172,8 @@ class PluginManager: except: self.log.error("Invalid config in %s: %s" % (name, config)) - if not home: - temp = self.parse(folder, pattern, plugins) - plugins.update(temp) + if not rootplugins and plugins: #: Double check + plugins.update(self.parse(folder, plugins)) return plugins @@ -218,18 +211,20 @@ class PluginManager: return res + def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): for ptype in pluginlist: if name in self.plugins[ptype]: return self.plugins[ptype][name], ptype return None, None + def getPlugin(self, name, original=False): """return plugin module from hoster|decrypter|container""" plugin, type = self.findPlugin(name) if not plugin: - self.log.warning("Plugin %s not found." % name) + self.log.warning("Plugin %s not found" % name) plugin = self.hosterPlugins['BasePlugin'] if "new_module" in plugin and not original: @@ -237,6 +232,7 @@ class PluginManager: return self.loadModule(type, name) + def getPluginName(self, name): """ used to obtain new name if other plugin was injected""" plugin, type = self.findPlugin(name) @@ -246,6 +242,7 @@ class PluginManager: return name + def loadModule(self, type, name): """ Returns loaded module for plugin @@ -254,26 +251,41 @@ class PluginManager: """ plugins = self.plugins[type] if name in plugins: - if "module" in plugins[name]: return plugins[name]['module'] + if "module" in plugins[name]: + return plugins[name]['module'] + try: module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]['name']), globals(), locals(), - plugins[name]['name']) - plugins[name]['module'] = module #cache import, maybe unneeded - return module + plugins[name]['name']) + except Exception, e: - self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) + self.log.error(_("Error importing plugin: [%(type)s] %(name)s (v%(version).2f) | %(errmsg)s") + % {'name': name, 'type': type, 'version': plugins[name]['version'], "errmsg": str(e)}) if self.core.debug: print_exc() + else: + plugins[name]['module'] = module #: cache import, maybe unneeded + + self.log.debug(_("Loaded plugin: [%(type)s] %(name)s (v%(version).2f)") + % {'name': name, 'type': type, 'version': plugins[name]['version']}) + return module + + def loadClass(self, type, name): """Returns the class of a plugin with the same name""" module = self.loadModule(type, name) - if module: return getattr(module, name) + if module: + return getattr(module, name) + else: + return None + def getAccountPlugins(self): """return list of account plugin names""" return self.accountPlugins.keys() + def find_module(self, fullname, path=None): #redirecting imports if necesarry if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins @@ -300,7 +312,8 @@ class PluginManager: newname = name.replace(self.ROOT, self.USERROOT) else: newname = name.replace(self.USERROOT, self.ROOT) - else: newname = name + else: + newname = name base, plugin = newname.rsplit(".", 1) @@ -333,25 +346,32 @@ class PluginManager: as_dict[t] = [n] for type in as_dict.iterkeys(): + # we do not reload hooks or internals, would cause to much side effects + if type in ("addon", "internal"): + flag = False + continue + for plugin in as_dict[type]: if plugin in self.plugins[type] and "module" in self.plugins[type][plugin]: - self.log.debug("Reloading %s" % plugin) - id = (type, plugin) + self.log.debug("Reloading plugin: [%(type)s] %(name)s" % {'name': plugin, 'type': type}) + try: reload(self.plugins[type][plugin]['module']) + except Exception, e: - self.log.error("Error when reloading %s" % id, str(e)) + self.log.error("Error when reloading plugin: [%(type)s] %(name)s" % {'name': plugin, 'type': type}, e) continue + else: - reloaded.append(id) - - #index creation - self.plugins['account'] = self.accountPlugins = self.parse("account") - self.plugins['container'] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins['crypter'] = self.crypterPlugins = self.parse("crypter", pattern=True) - # self.plugins['hook'] = self.hookPlugins = self.parse("hook") - self.plugins['hoster'] = self.hosterPlugins = self.parse("hoster", pattern=True) - self.plugins['ocr'] = self.ocrPlugins = self.parse("ocr") + reloaded.append((type, plugin)) + + #index creation + self.plugins[type] = self.parse(type) + + if type is "accounts": #@TODO: Remove this check in 0.4.10 + self.accountPlugins = self.plugins[type] + else: + setattr(self, "%sPlugins" % type, self.plugins[type]) if "accounts" in as_dict: #: accounts needs to be reloaded diff --git a/pyload/manager/thread/PluginThread.py b/pyload/manager/thread/PluginThread.py index c5092a207..eba14b2f1 100644 --- a/pyload/manager/thread/PluginThread.py +++ b/pyload/manager/thread/PluginThread.py @@ -154,7 +154,7 @@ class DownloadThread(PluginThread): """Constructor""" PluginThread.__init__(self, manager) - self.queue = Queue() # job queue + self.queue = Queue() #: job queue self.active = False self.start() @@ -175,7 +175,8 @@ class DownloadThread(PluginThread): return True try: - if not pyfile.hasPlugin(): continue + if not pyfile.hasPlugin(): + continue #this pyfile was deleted while queueing pyfile.plugin.checkForSameFiles(starting=True) @@ -183,6 +184,7 @@ class DownloadThread(PluginThread): # start download self.m.core.addonManager.downloadPreparing(pyfile) + pyfile.error = "" pyfile.plugin.preprocessing(self) self.m.log.info(_("Download finished: %s") % pyfile.name) @@ -204,6 +206,9 @@ class DownloadThread(PluginThread): pyfile.setStatus("aborted") + if self.m.core.debug: + print_exc() + self.clean(pyfile) continue @@ -236,6 +241,9 @@ class DownloadThread(PluginThread): self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) pyfile.error = msg + if self.m.core.debug: + print_exc() + self.m.core.addonManager.downloadFailed(pyfile) self.clean(pyfile) continue @@ -357,41 +365,46 @@ class DecrypterThread(PluginThread): retry = False try: - self.m.log.info(_("Decrypting starts: %s") % self.active.name) - self.active.plugin.preprocessing(self) + self.m.log.info(_("Decrypting starts: %s") % pyfile.name) + pyfile.error = "" + pyfile.plugin.preprocessing(self) except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) + self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) return except Fail, e: msg = e.args[0] if msg == "offline": - self.active.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % self.active.name) + pyfile.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % pyfile.name) else: - self.active.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": msg}) - self.active.error = msg + pyfile.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + if self.m.core.debug: + print_exc() return except Abort: self.m.log.info(_("Download aborted: %s") % pyfile.name) pyfile.setStatus("aborted") - + + if self.m.core.debug: + print_exc() return except Retry: - self.m.log.info(_("Retrying %s") % self.active.name) + self.m.log.info(_("Retrying %s") % pyfile.name) retry = True return self.run() except Exception, e: - self.active.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": str(e)}) - self.active.error = str(e) + pyfile.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) if self.m.core.debug: print_exc() @@ -399,21 +412,14 @@ class DecrypterThread(PluginThread): return - finally: if not retry: - self.active.release() + pyfile.release() self.active = False self.m.core.files.save() self.m.localThreads.remove(self) exc_clear() - - #self.m.core.addonManager.downloadFinished(pyfile) - - - #self.m.localThreads.remove(self) - #self.active.finishIfDone() if not retry: pyfile.delete() |