# -*- coding: utf-8 -*- from __future__ import with_statement import os import re import urllib from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 from module.utils import fs_encode, fs_decode, html_unescape, save_join as fs_join def replace_patterns(string, ruleslist): for r in ruleslist: rf, rt = r string = re.sub(rf, rt, string) return string def set_cookies(cj, cookies): for cookie in cookies: if isinstance(cookie, tuple) and len(cookie) == 3: domain, name, value = cookie cj.setCookie(domain, name, value) def parseHtmlTagAttrValue(attr_name, tag): m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^>\s\"'][^>\s]*)\1" % attr_name, tag, re.I) return m.group(2) if m else None def parseHtmlForm(attr_str, html, input_names={}): for form in re.finditer(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, html, re.S | re.I): inputs = {} action = parseHtmlTagAttrValue("action", form.group('TAG')) for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?= `encode` in 0.4.10 log("%(plugin)s%(id)s: %(msg)s" % {'plugin': self.__name__, 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", 'msg' : msg or _(level.upper() + " MARK")}) def logDebug(self, *args): if self.core.debug: return self._log("debug", args) def logInfo(self, *args): return self._log("info", args) def logWarning(self, *args): return self._log("warning", args) def logError(self, *args): return self._log("error", args) def logCritical(self, *args): return self._log("critical", args) def setConfig(self, option, value): """ Set config value for current plugin :param option: :param value: :return: """ self.core.config.setPlugin(self.__name__, option, value) #: Deprecated method def setConf(self, *args, **kwargs): """ See `setConfig` """ return self.setConfig(*args, **kwargs) def getConfig(self, option, default="", plugin=None): """ Returns config value for current plugin :param option: :return: """ try: return self.core.config.getPlugin(plugin or self.__name__, option) except KeyError: self.logWarning(_("Config option or plugin not found")) return default #: Deprecated method def getConf(self, *args, **kwargs): """ See `getConfig` """ return self.getConfig(*args, **kwargs) def store(self, key, value): """ Saves a value persistently to the database """ self.core.db.setStorage(self.__name__, key, value) #: Deprecated method def setStorage(self, *args, **kwargs): """ Same as `setStorage` """ return self.store(*args, **kwargs) def retrieve(self, key, default=None): """ Retrieves saved value or dict of all saved entries if key is None """ return self.core.db.getStorage(self.__name__, key) or default #: Deprecated method def getStorage(self, *args, **kwargs): """ Same as `getStorage` """ return self.retrieve(*args, **kwargs) def delStorage(self, key): """ Delete entry in db """ self.core.db.delStorage(self.__name__, key) def fail(self, reason): """ Fail and give reason """ raise Fail(fs_encode(reason)) def error(self, reason="", type=_("Parse")): if not reason: type = _("Unknown") msg = _("%s error") % type.strip().capitalize() if type else _("Error") msg += (": %s" % reason.strip()) if reason else "" msg += _(" | Plugin may be out of date") raise Fail(msg) def fixurl(self, url): return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip() def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=True, req=None): """ Load content at url and returns it :param url: :param get: :param post: :param ref: :param cookies: :param just_header: If True only the header will be retrieved and returned as dict :param decode: Wether to decode the output according to http header, should be True in most cases :return: Loaded content """ if hasattr(self, 'pyfile') and self.pyfile.abort: self.abort() url = self.fixurl(url) if not url or not isinstance(url, basestring): self.fail(_("No url given")) if self.core.debug: self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) if req is None: if hasattr(self, "req"): req = self.req else: req = self.core.requestFactory.getRequest(self.__name__) res = req.load(url, get, post, ref, cookies, just_header, True, bool(decode)) if decode: res = html_unescape(res) if isinstance(decode, basestring): res = res.decode(decode) if self.core.debug: import inspect frame = inspect.currentframe() framefile = fs_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) try: if not os.path.exists(os.path.join("tmp", self.__name__)): os.makedirs(os.path.join("tmp", self.__name__)) with open(framefile, "wb") as f: del frame #: delete the frame or it wont be cleaned f.write(res.encode('utf8')) except IOError, e: self.logError(e) if just_header: #: parse header header = {"code": req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: continue key, none, value = line.partition(":") key = key.strip().lower() value = value.strip() if key in header: if type(header[key]) == list: header[key].append(value) else: header[key] = [header[key], value] else: header[key] = value res = header return res