diff options
Diffstat (limited to 'module/plugins/Plugin.py')
| -rw-r--r-- | module/plugins/Plugin.py | 422 |
1 files changed, 279 insertions, 143 deletions
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 15bf3971f..0a9c647fb 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -1,21 +1,6 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN, spoob, mkaay -""" +from __future__ import with_statement from time import time, sleep from random import randint @@ -30,8 +15,11 @@ if os.name != "nt": from grp import getgrnam from itertools import islice +from traceback import print_exc +from urlparse import urlparse + +from pyload.utils import fs_decode, fs_encode, safe_filename, safe_join -from module.utils import save_join, save_path, fs_encode, fs_decode def chunks(iterable, size): it = iter(iterable) @@ -69,28 +57,40 @@ class Base(object): def __init__(self, core): #: Core instance self.core = core - #: logging instance - self.log = core.log - #: core config - self.config = core.config - #log functions + + def _log(self, type, args): + msg = " | ".join([encode(a).strip() for a in args if a]) + logger = getattr(self.core.log, type) + logger("%s: %s" % (self.__name__, msg or _("%s MARK" % type.upper()))) + + + def logDebug(self, *args): + if self.core.debug: + return self._log("debug", args) + + def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + return self._log("info", args) + def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + return self._log("warning", args) + def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + return self._log("error", args) - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logCritical(self, *args): + return self._log("critical", args) + #: Deprecated method def setConf(self, option, value): """ see `setConfig` """ - self.core.config.setPlugin(self.__name__, option, value) + self.setConfig(option, value) + def setConfig(self, option, value): """ Set config value for current plugin @@ -99,11 +99,14 @@ class Base(object): :param value: :return: """ - self.setConf(option, value) + self.core.config.setPlugin(self.__name__, option, value) + + #: Deprecated method def getConf(self, option): """ see `getConfig` """ - return self.core.config.getPlugin(self.__name__, option) + return self.getConfig(option) + def getConfig(self, option): """ Returns config value for current plugin @@ -111,26 +114,31 @@ class Base(object): :param option: :return: """ - return self.getConf(option) + return self.core.config.getPlugin(self.__name__, option) + def setStorage(self, key, value): """ Saves a value persistently to the database """ self.core.db.setStorage(self.__name__, key, value) + def store(self, key, value): """ same as `setStorage` """ self.core.db.setStorage(self.__name__, key, value) + def getStorage(self, key=None, default=None): """ Retrieves saved value or dict of all saved entries if key is None """ - if key is not None: + if key: return self.core.db.getStorage(self.__name__, key) or default return self.core.db.getStorage(self.__name__, key) + def retrieve(self, *args, **kwargs): """ same as `getStorage` """ return self.getStorage(*args, **kwargs) + def delStorage(self, key): """ Delete entry in db """ self.core.db.delStorage(self.__name__, key) @@ -141,22 +149,33 @@ class Plugin(Base): Base plugin for hoster/crypter. Overwrite `process` / `decrypt` in your subclassed plugin. """ - __name__ = "Plugin" - __version__ = "0.4" - __pattern__ = None - __type__ = "hoster" - __config__ = [("name", "type", "desc", "default")] - __description__ = """Base Plugin""" - __author_name__ = ("RaNaN", "spoob", "mkaay") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") + __name__ = "Plugin" + __type__ = "hoster" + __version__ = "0.07" + + __pattern__ = r'^unmatchable$' + __config__ = [] #: [("name", "type", "desc", "default")] + + __description__ = """Base plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("spoob", "spoob@pyload.org"), + ("mkaay", "mkaay@mkaay.de")] + + + info = {} #: file info dict + def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) + #: engage wan reconnection self.wantReconnect = False - #: enables simultaneous processing of multiple downloads + + #: enable simultaneous processing of multiple downloads self.multiDL = True self.limitDL = 0 + #: chunk limit self.chunkLimit = 1 self.resumeDownload = False @@ -165,7 +184,9 @@ class Plugin(Base): self.waitUntil = 0 self.waiting = False - self.ocr = None #captcha reader instance + #: captcha reader instance + self.ocr = None + #: account handler instance, see :py:class:`Account` self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) @@ -174,7 +195,9 @@ class Plugin(Base): #: username/login self.user = None - if self.account and not self.account.canUse(): self.account = None + if self.account and not self.account.canUse(): + self.account = None + if self.account: self.user, data = self.account.selectAccount() #: Browser instance, see `network.Browser` @@ -190,37 +213,46 @@ class Plugin(Base): #: associated pyfile instance, see `PyFile` self.pyfile = pyfile + self.thread = None # holds thread in future #: location where the last call to download was saved self.lastDownload = "" #: re match of the last call to `checkDownload` self.lastCheck = None + #: js engine, see `JsEngine` self.js = self.core.js - self.cTask = None #captcha task - self.retries = 0 # amount of retries already made - self.html = None # some plugins store html code here + #: captcha task + self.cTask = None + + self.html = None #@TODO: Move to hoster class in 0.4.10 + self.retries = 0 self.init() + def getChunkCount(self): if self.chunkLimit <= 0: - return self.config["download"]["chunks"] - return min(self.config["download"]["chunks"], self.chunkLimit) + return self.core.config['download']['chunks'] + return min(self.core.config['download']['chunks'], self.chunkLimit) + def __call__(self): return self.__name__ + def init(self): """initialize the plugin (in addition to `__init__`)""" pass + def setup(self): """ setup for enviroment and other things, called before downloading (possibly more than one time)""" pass + def preprocessing(self, thread): """ handles important things to do before starting """ self.thread = thread @@ -241,12 +273,14 @@ class Plugin(Base): """the 'main' method of every plugin, you **have to** overwrite it""" raise NotImplementedError + def resetAccount(self): """ dont use account and retry download """ self.account = None self.req = self.core.requestFactory.getRequest(self.__name__) self.retry() + def checksum(self, local_file=None): """ return codes: @@ -256,51 +290,119 @@ class Plugin(Base): 10 - not implemented 20 - unknown error """ - #@TODO checksum check hook + #@TODO checksum check addon return True, 10 - def setWait(self, seconds, reconnect=False): + def setReconnect(self, reconnect): + reconnect = bool(reconnect) + self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + self.wantReconnect = reconnect + + + def setWait(self, seconds, reconnect=None): """Set a specific wait time later used with `wait` - + :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ - if reconnect: - self.wantReconnect = True - self.pyfile.waitUntil = time() + int(seconds) + wait_time = int(seconds) + 1 + wait_until = time() + wait_time + + self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), + "Wait: %d seconds" % wait_time) - def wait(self): + self.pyfile.waitUntil = wait_until + + if reconnect is not None: + self.setReconnect(reconnect) + + + def wait(self, seconds=None, reconnect=None): """ waits the time previously set """ + + pyfile = self.pyfile + + if seconds is not None: + self.setWait(seconds) + + if reconnect is not None: + self.setReconnect(reconnect) + self.waiting = True - self.pyfile.setStatus("waiting") - while self.pyfile.waitUntil > time(): - self.thread.m.reconnecting.wait(2) + status = pyfile.status + pyfile.setStatus("waiting") - if self.pyfile.abort: raise Abort - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect + self.logInfo(_("Wait: %d seconds") % (pyfile.waitUntil - time()), + _("Reconnect: %s") % self.wantReconnect) + + if self.account: + self.logDebug("Ignore reconnection due account logged") + + while pyfile.waitUntil > time(): + if pyfile.abort: + self.abort() + + sleep(1) + else: + while pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) + + if pyfile.abort: + self.abort() + + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + sleep(1) self.waiting = False - self.pyfile.setStatus("starting") + + pyfile.status = status + def fail(self, reason): """ fail and give reason """ raise Fail(reason) - def offline(self): + + def abort(self, reason=""): + """ abort and give reason """ + if reason: + self.pyfile.error = str(reason) + raise Abort + + + def error(self, reason="", type=""): + if not reason and not type: + type = "unknown" + + msg = _("%s error") % _(type.strip().capitalize()) if type else _("Error") + msg += ": " + reason.strip() if reason else "" + msg += _(" | Plugin may be out of date") + + raise Fail(msg) + + + def offline(self, reason=""): """ fail and indicate file is offline """ + if reason: + self.pyfile.error = str(reason) raise Fail("offline") - def tempOffline(self): + + def tempOffline(self, reason=""): """ fail and indicates file ist temporary offline, the core may take consequences """ + if reason: + self.pyfile.error = str(reason) raise Fail("temp. offline") - def retry(self, max_tries=3, wait_time=1, reason=""): + + def retry(self, max_tries=5, wait_time=1, reason=""): """Retries and begin again from the beginning :param max_tries: number of maximum retries @@ -308,26 +410,28 @@ class Plugin(Base): :param reason: reason for retrying, will be passed to fail if max_tries reached """ if 0 < max_tries <= self.retries: - if not reason: reason = "Max retries reached" - raise Fail(reason) + self.error(reason or _("Max retries reached"), "retry") - self.wantReconnect = False - self.setWait(wait_time) - self.wait() + self.wait(wait_time, False) self.retries += 1 raise Retry(reason) + def invalidCaptcha(self): + self.logError(_("Invalid captcha")) if self.cTask: self.cTask.invalid() + def correctCaptcha(self): + self.logInfo(_("Correct captcha")) if self.cTask: self.cTask.correct() + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual'): + result_type='textual', timeout=290): """ Loads a captcha and decrypts it with ocr, plugin, user input :param url: url of captcha image @@ -339,40 +443,41 @@ class Plugin(Base): :param result_type: 'textual' if text is written on the captcha\ or 'positional' for captcha where the user have to click\ on a specific region on the captcha - + :return: result of decrypting """ img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time())[-6:].replace(".", "") - temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - temp_file.write(img) - temp_file.close() - has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins + with open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: + tmpCaptcha.write(img) + + has_plugin = self.__name__ in self.core.pluginManager.ocrPlugins if self.core.captcha: - Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) + Ocr = self.core.pluginManager.loadClass("ocr", self.__name__) else: Ocr = None if Ocr and not forceUser: sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: raise Abort + if self.pyfile.abort: + self.abort() ocr = Ocr() - result = ocr.get_captcha(temp_file.name) + result = ocr.get_captcha(tmpCaptcha.name) else: captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) + task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) self.cTask = task - captchaManager.handleCaptcha(task) + captchaManager.handleCaptcha(task, timeout) while task.isWaiting(): if self.pyfile.abort: captchaManager.removeTask(task) - raise Abort + self.abort() sleep(1) captchaManager.removeTask(task) @@ -382,21 +487,21 @@ class Plugin(Base): elif task.error: self.fail(task.error) elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) + self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) result = task.result - self.log.debug("Received captcha result: %s" % str(result)) + self.logDebug("Received captcha result: %s" % result) if not self.core.debug: try: - remove(temp_file.name) - except: + remove(tmpCaptcha.name) + except Exception: pass return result - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True): """Load content at url and returns it :param url: @@ -404,35 +509,42 @@ class Plugin(Base): :param post: :param ref: :param cookies: - :param just_header: if True only the header will be retrieved and returned as dict + :param just_header: If True only the header will be retrieved and returned as dict :param decode: Wether to decode the output according to http header, should be True in most cases + :param follow_location: If True follow location else not + :param save_cookies: If True saves received cookies else discard them :return: Loaded content """ - if self.pyfile.abort: raise Abort - #utf8 vs decode -> please use decode attribute in all future plugins - if type(url) == unicode: url = str(url) + if self.pyfile.abort: + self.abort() + + if not url: + self.fail(_("No url given")) - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) + url = encode(url).strip() #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins if self.core.debug: - from inspect import currentframe + self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) - frame = currentframe() - if not exists(join("tmp", self.__name__)): - makedirs(join("tmp", self.__name__)) + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies) - f = open( - join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) - , "wb") - del frame # delete the frame or it wont be cleaned + if decode: + res = encode(res) + if self.core.debug: + from inspect import currentframe + + frame = currentframe() + framefile = safe_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) try: - tmp = res.encode("utf8") - except: - tmp = res + if not exists(join("tmp", self.__name__)): + makedirs(join("tmp", self.__name__)) - f.write(tmp) - f.close() + with open(framefile, "wb") as f: + del frame #: delete the frame or it wont be cleaned + f.write(res) + except IOError, e: + self.logError(e) if just_header: #parse header @@ -442,7 +554,7 @@ class Plugin(Base): if not line or ":" not in line: continue key, none, value = line.partition(":") - key = key.lower().strip() + key = key.strip().lower() value = value.strip() if key in header: @@ -456,6 +568,7 @@ class Plugin(Base): return res + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder @@ -468,34 +581,44 @@ class Plugin(Base): the filename will be changed if needed :return: The location where the file was saved """ + if self.pyfile.abort: + self.abort() + + if not url: + self.fail(_("No url given")) + + url = encode(url).strip() + + if self.core.debug: + self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) self.checkForSameFiles() self.pyfile.setStatus("downloading") - download_folder = self.config['general']['download_folder'] + download_folder = self.core.config['general']['download_folder'] - location = save_join(download_folder, self.pyfile.package().folder) + location = safe_join(download_folder, self.pyfile.package().folder) if not exists(location): - makedirs(location, int(self.core.config["permission"]["folder"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] + try: + makedirs(location, int(self.core.config['permission']['folder'], 8)) + if self.core.config['permission']['change_dl'] and os.name != "nt": + uid = getpwnam(self.core.config['permission']['user'])[2] + gid = getgrnam(self.core.config['permission']['group'])[2] chown(location, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + except Exception, e: + self.fail(e) # convert back to unicode location = fs_decode(location) - name = save_path(self.pyfile.name) + name = safe_filename(self.pyfile.name) filename = join(location, name) - self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) + self.core.addonManager.dispatchEvent("download-start", self.pyfile, url, filename) try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, @@ -504,31 +627,38 @@ class Plugin(Base): finally: self.pyfile.size = self.req.size - if disposition and newname and newname != name: #triple check, just to be sure - self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) - self.pyfile.name = newname - filename = join(location, newname) + if newname: + newname = urlparse(newname).path.split("/")[-1] - fs_filename = fs_encode(filename) + if disposition and newname != name: + self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) - if self.core.config["permission"]["change_file"]: - chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) + fs_filename = fs_encode(filename) - if self.core.config["permission"]["change_dl"] and os.name != "nt": + if self.core.config['permission']['change_file']: try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] + chmod(fs_filename, int(self.core.config['permission']['file'], 8)) + except Exception, e: + self.logWarning(_("Setting file mode failed"), e) + if self.core.config['permission']['change_dl'] and os.name != "nt": + try: + uid = getpwnam(self.core.config['permission']['user'])[2] + gid = getgrnam(self.core.config['permission']['group'])[2] chown(fs_filename, uid, gid) + except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) + self.logWarning(_("Setting User and Group failed"), e) self.lastDownload = filename return self.lastDownload + def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): """ checks the content of the last downloaded file, re match is saved to `lastCheck` - + :param rules: dict with names and rules to match (compiled regexp or strings) :param api_size: expected file size :param max_size: if the file is larger then it wont be checked @@ -537,21 +667,23 @@ class Plugin(Base): :return: dictionary key of the first rule that matched """ lastDownload = fs_encode(self.lastDownload) - if not exists(lastDownload): return None + if not exists(lastDownload): + return None size = stat(lastDownload) size = size.st_size if api_size and api_size <= size: return None elif size > max_size and not read_size: return None - self.log.debug("Download Check triggered") - f = open(lastDownload, "rb") - content = f.read(read_size if read_size else -1) - f.close() + self.logDebug("Download Check triggered") + + with open(lastDownload, "rb") as f: + content = f.read(read_size if read_size else -1) + #produces encoding errors, better log to other file in the future? - #self.log.debug("Content: %s" % content) + #self.logDebug("Content: %s" % content) for name, rule in rules.iteritems(): - if type(rule) in (str, unicode): + if isinstance(rule, basestring): if rule in content: if delete: remove(lastDownload) @@ -589,29 +721,33 @@ class Plugin(Base): 5, 7) and starting: #a download is waiting/starting and was appenrently started before raise SkipDownload(pyfile.pluginname) - download_folder = self.config['general']['download_folder'] - location = save_join(download_folder, pack.folder, self.pyfile.name) + download_folder = self.core.config['general']['download_folder'] + location = safe_join(download_folder, pack.folder, self.pyfile.name) if starting and self.core.config['download']['skip_existing'] and exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: - raise SkipDownload("File exists.") + raise SkipDownload("File exists") pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: if exists(location): raise SkipDownload(pyfile[0]) - self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) + self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name) + def clean(self): """ clean everything and remove references """ if hasattr(self, "pyfile"): del self.pyfile + if hasattr(self, "req"): self.req.close() del self.req + if hasattr(self, "thread"): del self.thread + if hasattr(self, "html"): del self.html |