diff options
Diffstat (limited to 'module/plugins/internal')
| -rw-r--r-- | module/plugins/internal/BasePlugin.py | 95 | ||||
| -rw-r--r-- | module/plugins/internal/DeadCrypter.py | 6 | ||||
| -rw-r--r-- | module/plugins/internal/DeadHoster.py | 6 | ||||
| -rw-r--r-- | module/plugins/internal/SimpleCrypter.py | 19 | ||||
| -rw-r--r-- | module/plugins/internal/UpdateManager.py | 300 | ||||
| -rw-r--r-- | module/plugins/internal/XFSAccount.py | 9 | ||||
| -rw-r--r-- | module/plugins/internal/XFSHoster.py | 12 | ||||
| -rw-r--r-- | module/plugins/internal/__init__.py | 1 |
8 files changed, 410 insertions, 38 deletions
diff --git a/module/plugins/internal/BasePlugin.py b/module/plugins/internal/BasePlugin.py new file mode 100644 index 000000000..792497449 --- /dev/null +++ b/module/plugins/internal/BasePlugin.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- + +import re + +from urllib import unquote +from urlparse import urljoin, urlparse + +from module.network.HTTPRequest import BadHeader +from module.plugins.internal.SimpleHoster import create_getInfo, fileUrl +from module.plugins.Hoster import Hoster + + +class BasePlugin(Hoster): + __name__ = "BasePlugin" + __type__ = "hoster" + __version__ = "0.34" + + __pattern__ = r'^unmatchable$' + + __description__ = """Base plugin when any other didnt fit""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + @classmethod + def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10 + url = unquote(url) + url_p = urlparse(url) + return {'name' : (url_p.path.split('/')[-1] + or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] + or url_p.netloc.split('.', 1)[0]), + 'size' : 0, + 'status': 3 if url else 8, + 'url' : url} + + + def setup(self): + self.chunkLimit = -1 + self.multiDL = True + self.resumeDownload = True + + + def process(self, pyfile): + """main function""" + + pyfile.name = self.getInfo(pyfile.url)['name'] + + if not pyfile.url.startswith("http"): + self.fail(_("No plugin matched")) + + for _i in xrange(5): + try: + link = fileUrl(self, unquote(pyfile.url)) + + if link: + self.download(link, ref=False, disposition=True) + else: + self.fail(_("File not found")) + + except BadHeader, e: + if e.code is 404: + self.offline() + + elif e.code in (401, 403): + self.logDebug("Auth required", "Received HTTP status code: %d" % e.code) + + account = self.core.accountManager.getAccountPlugin('Http') + servers = [x['login'] for x in account.getAllAccounts()] + server = urlparse(pyfile.url).netloc + + if server in servers: + self.logDebug("Logging on to %s" % server) + self.req.addAuth(account.getAccountData(server)['password']) + else: + pwd = self.getPassword() + if ':' in pwd: + self.req.addAuth(pwd) + else: + self.fail(_("Authorization required")) + else: + self.fail(e) + else: + break + else: + self.fail(_("No file downloaded")) #@TODO: Move to hoster class in 0.4.10 + + check = self.checkDownload({'empty file': re.compile(r'\A\Z'), + 'html file' : re.compile(r'\A\s*<!DOCTYPE html'), + 'html error': re.compile(r'\A\s*(<.+>)?\d{3}(\Z|\s+)')}) + if check: + self.fail(check.capitalize()) + + +getInfo = create_getInfo(BasePlugin) diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 866d177cf..ce56947fc 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.SimpleCrypter import create_getInfo -from module.plugins.Crypter import Crypter as _Crypter +from pyload.plugin.Crypter import Crypter as _Crypter +from pyload.plugin.internal.SimpleCrypter import create_getInfo class DeadCrypter(_Crypter): @@ -11,7 +11,7 @@ class DeadCrypter(_Crypter): __pattern__ = r'^unmatchable$' - __description__ = """ Crypter is no longer available """ + __description__ = """Crypter is no longer available""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index a6ad92607..132e4741a 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.SimpleHoster import create_getInfo -from module.plugins.Hoster import Hoster as _Hoster +from pyload.plugin.Hoster import Hoster as _Hoster +from pyload.plugin.internal.SimpleHoster import create_getInfo class DeadHoster(_Hoster): @@ -11,7 +11,7 @@ class DeadHoster(_Hoster): __pattern__ = r'^unmatchable$' - __description__ = """ Hoster is no longer available """ + __description__ = """Hoster is no longer available""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 856b548f0..dc34f864f 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -4,9 +4,9 @@ import re from urlparse import urljoin, urlparse -from module.plugins.Crypter import Crypter -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies -from module.utils import fixup +from pyload.plugin.Crypter import Crypter +from pyload.plugin.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies +from pyload.utils import fixup class SimpleCrypter(Crypter, SimpleHoster): @@ -68,19 +68,6 @@ class SimpleCrypter(Crypter, SimpleHoster): LOGIN_PREMIUM = False - #@TODO: Remove in 0.4.10 - def init(self): - account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") - account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name) - - if account and account.canUse(): - self.user, data = account.selectAccount() - self.req = account.getAccountRequest(self.user) - self.premium = account.isPremium(self.user) - - self.account = account - - def prepare(self): self.pyfile.error = "" #@TODO: Remove in 0.4.10 diff --git a/module/plugins/internal/UpdateManager.py b/module/plugins/internal/UpdateManager.py new file mode 100644 index 000000000..7bbfc0c79 --- /dev/null +++ b/module/plugins/internal/UpdateManager.py @@ -0,0 +1,300 @@ +# -*- coding: utf-8 -*- + +import re +import sys + +from operator import itemgetter +from os import path, remove, stat + +from pyload.network.RequestFactory import getURL +from pyload.plugin.Addon import Expose, Addon, threaded +from pyload.utils import safe_join + + +class UpdateManager(Addon): + __name__ = "UpdateManager" + __type__ = "addon" + __version__ = "0.40" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("mode" , "pyLoad + plugins;plugins only", "Check updates for" , "pyLoad + plugins"), + ("interval" , "int" , "Check interval in hours" , 8 ), + ("reloadplugins", "bool" , "Monitor plugins for code changes (debug mode only)", True ), + ("nodebugupdate", "bool" , "Don't check for updates in debug mode" , True )] + + __description__ = """Check for updates""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + # event_list = ["pluginConfigChanged"] + + SERVER_URL = "http://updatemanager.pyload.org" + MIN_INTERVAL = 6 * 60 * 60 #: 6h minimum check interval (value is in seconds) + + + def pluginConfigChanged(self, plugin, name, value): + if name == "interval": + interval = value * 60 * 60 + if self.MIN_INTERVAL <= interval != self.interval: + self.core.scheduler.removeJob(self.cb) + self.interval = interval + self.initPeriodical() + else: + self.logDebug("Invalid interval value, kept current") + + elif name == "reloadplugins": + if self.cb2: + self.core.scheduler.removeJob(self.cb2) + if value is True and self.core.debug: + self.periodical2() + + + def activate(self): + self.pluginConfigChanged(self.__name__, "interval", self.getConfig("interval")) + x = lambda: self.pluginConfigChanged(self.__name__, "reloadplugins", self.getConfig("reloadplugins")) + self.core.scheduler.addJob(10, x, threaded=False) + + + def deactivate(self): + self.pluginConfigChanged(self.__name__, "reloadplugins", False) + + + def setup(self): + self.cb2 = None + self.interval = self.MIN_INTERVAL + self.updating = False + self.info = {'pyload': False, 'version': None, 'plugins': False} + self.mtimes = {} #: store modification time for each plugin + + + def periodical2(self): + if not self.updating: + self.autoreloadPlugins() + + self.cb2 = self.core.scheduler.addJob(4, self.periodical2, threaded=False) + + + @Expose + def autoreloadPlugins(self): + """ reload and reindex all modified plugins """ + modules = filter( + lambda m: m and (m.__name__.startswith("pyload.plugin.") or + m.__name__.startswith("userplugins.")) and + m.__name__.count(".") >= 2, sys.modules.itervalues() + ) + + reloads = [] + + for m in modules: + root, type, name = m.__name__.rsplit(".", 2) + id = (type, name) + if type in self.core.pluginManager.plugins: + f = m.__file__.replace(".pyc", ".py") + if not path.isfile(f): + continue + + mtime = stat(f).st_mtime + + if id not in self.mtimes: + self.mtimes[id] = mtime + elif self.mtimes[id] < mtime: + reloads.append(id) + self.mtimes[id] = mtime + + return True if self.core.pluginManager.reloadPlugins(reloads) else False + + + def periodical(self): + if self.info['pyload'] or self.getConfig("nodebugupdate") and self.core.debug: + return + + self.updateThread() + + + def server_request(self): + try: + return getURL(self.SERVER_URL, get={'v': self.core.api.getServerVersion()}).splitlines() + except Exception: + self.logWarning(_("Unable to contact server to get updates")) + + + @threaded + def updateThread(self): + self.updating = True + + status = self.update(onlyplugin=self.getConfig("mode") == "plugins only") + + if status == 2: + self.core.api.restart() + else: + self.updating = False + + + @Expose + def updatePlugins(self): + """ simple wrapper for calling plugin update quickly """ + return self.update(onlyplugin=True) + + + @Expose + def update(self, onlyplugin=False): + """ check for updates """ + data = self.server_request() + + if not data: + exitcode = 0 + + elif data[0] == "None": + self.logInfo(_("No new pyLoad version available")) + updates = data[1:] + exitcode = self._updatePlugins(updates) + + elif onlyplugin: + exitcode = 0 + + else: + newversion = data[0] + self.logInfo(_("*** New pyLoad Version %s available ***") % newversion) + self.logInfo(_("*** Get it here: https://github.com/pyload/pyload/releases ***")) + exitcode = 3 + self.info['pyload'] = True + self.info['version'] = newversion + + return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required; 3 = No plugins updated, new pyLoad version available + + + def _updatePlugins(self, updates): + """ check for plugin updates """ + + if self.info['plugins']: + return False #: plugins were already updated + + exitcode = 0 + updated = [] + + vre = re.compile(r'__version__.*=.*("|\')([\d.]+)') + url = updates[0] + schema = updates[1].split('|') + + if "BLACKLIST" in updates: + blacklist = updates[updates.index('BLACKLIST') + 1:] + updates = updates[2:updates.index('BLACKLIST')] + else: + blacklist = None + updates = updates[2:] + + upgradable = sorted(map(lambda x: dict(zip(schema, x.split('|'))), updates), + key=itemgetter("type", "name")) + + for plugin in upgradable: + filename = plugin['name'] + type = plugin['type'] + version = plugin['version'] + + if filename.endswith(".pyc"): + name = filename[:filename.find("_")] + else: + name = filename.replace(".py", "") + + plugins = getattr(self.core.pluginManager, "%sPlugins" % type) + + oldver = float(plugins[name]['version']) if name in plugins else None + newver = float(version) + + if not oldver: + msg = "New plugin: [%(type)s] %(name)s (v%(newver).2f)" + elif newver > oldver: + msg = "New version of plugin: [%(type)s] %(name)s (v%(oldver).2f -> v%(newver).2f)" + else: + continue + + self.logInfo(_(msg) % {'type' : type, + 'name' : name, + 'oldver': oldver, + 'newver': newver}) + try: + content = getURL(url % plugin) + m = vre.search(content) + + if m and m.group(2) == version: + f = open(safe_join("userplugins", prefix, filename), "wb") + f.write(content) + f.close() + updated.append((prefix, name)) + else: + raise Exception, _("Version mismatch") + + except Exception, e: + self.logError(_("Error updating plugin %s") % filename, e) + + if blacklist: + blacklisted = map(lambda x: (x.split('|')[0], x.split('|')[1].rsplit('.', 1)[0]), blacklist) + + # Always protect internal plugins from removing + for i, n, t in blacklisted.enumerate(): + if t == "internal": + del blacklisted[i] + + blacklisted = sorted(blacklisted) + removed = self.removePlugins(blacklisted) + for t, n in removed: + self.logInfo(_("Removed blacklisted plugin [%(type)s] %(name)s") % { + 'type': t, + 'name': n, + }) + + if updated: + reloaded = self.core.pluginManager.reloadPlugins(updated) + if reloaded: + self.logInfo(_("Plugins updated and reloaded")) + exitcode = 1 + else: + self.logInfo(_("*** Plugins have been updated, but need a pyLoad restart to be reloaded ***")) + self.info['plugins'] = True + exitcode = 2 + else: + self.logInfo(_("No plugin updates available")) + + return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required + + + @Expose + def removePlugins(self, type_plugins): + """ delete plugins from disk """ + + if not type_plugins: + return + + self.logDebug("Requested deletion of plugins: %s" % type_plugins) + + removed = [] + + for type, name in type_plugins: + err = False + file = name + ".py" + + for root in ("userplugins", path.join(pypath, "pyload", "plugins")): + + filename = safe_join(root, type, file) + try: + remove(filename) + except Exception, e: + self.logDebug("Error deleting: %s" % path.basename(filename), e) + err = True + + filename += "c" + if path.isfile(filename): + try: + if type == "addon": + self.manager.deactivateAddon(name) + remove(filename) + except Exception, e: + self.logDebug("Error deleting: %s" % path.basename(filename), e) + err = True + + if not err: + id = (type, name) + removed.append(id) + + return removed #: return a list of the plugins successfully removed diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 845ea3230..2e6b7dc50 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -5,8 +5,8 @@ import re from time import gmtime, mktime, strptime from urlparse import urljoin -from module.plugins.Account import Account -from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies +from pyload.plugin.Account import Account +from pyload.plugin.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): @@ -39,11 +39,6 @@ class XFSAccount(Account): LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<' - def __init__(self, manager, accounts): #@TODO: remove in 0.4.10 - self.init() - return super(XFSAccount, self).__init__(manager, accounts) - - def init(self): if not self.HOSTER_DOMAIN: self.logError(_("Missing HOSTER_DOMAIN")) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index c4fc969ba..ecdd2f125 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -6,11 +6,9 @@ from random import random from time import sleep from urlparse import urljoin, urlparse -from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME - -from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia +from pyload.plugin.internal.captcha import ReCaptcha, SolveMedia from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, secondsToMidnight -from module.utils import html_unescape +from pyload.utils import html_unescape class XFSHoster(SimpleHoster): @@ -101,11 +99,7 @@ class XFSHoster(SimpleHoster): data = self.getPostParameters() - self.req.http.c.setopt(FOLLOWLOCATION, 0) - - self.html = self.load(pyfile.url, post=data, ref=True, decode=True) - - self.req.http.c.setopt(FOLLOWLOCATION, 1) + self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True, follow_location=False) m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) if m and not "op=" in m.group(1): diff --git a/module/plugins/internal/__init__.py b/module/plugins/internal/__init__.py index e69de29bb..40a96afc6 100644 --- a/module/plugins/internal/__init__.py +++ b/module/plugins/internal/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- |