diff options
Diffstat (limited to 'module/plugins/internal/Plugin.py')
| -rw-r--r-- | module/plugins/internal/Plugin.py | 237 |
1 files changed, 157 insertions, 80 deletions
diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 7b45c40a8..6d94ca1e4 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -6,14 +6,17 @@ import datetime import inspect import os import re +import time +import traceback import urllib +import urlparse -if os.name != "nt": +if os.name is not "nt": import grp import pwd from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 -from module.utils import fs_encode, fs_decode, html_unescape, save_join as fs_join +from module.utils import fs_encode, fs_decode, html_unescape, parseFileSize as parse_size, save_join as fs_join #@TODO: Move to utils in 0.4.10 @@ -22,7 +25,12 @@ def decode(string, encoding='utf8'): if type(string) is str: return string.decode(encoding, "replace") else: - return string + return unicode(string) + + +#@TODO: Remove in 0.4.10 +def _decode(*args, **kwargs): + return decode(*args, **kwargs) #@TODO: Move to utils in 0.4.10 @@ -31,14 +39,14 @@ def encode(string, encoding='utf8'): if type(string) is unicode: return string.encode(encoding, "replace") else: - return string + return str(string) #@TODO: Move to utils in 0.4.10 def exists(path): if os.path.exists(path): - if os.name == "nt": - dir, name = os.path.split(path) + if os.name is "nt": + dir, name = os.path.split(path.rstrip(os.sep)) return name in os.listdir(dir) else: return True @@ -46,9 +54,69 @@ def exists(path): return False +def fixurl(url, unquote=None): + newurl = urllib.unquote(url) + + if unquote is None: + unquote = newurl == url + + newurl = html_unescape(newurl.decode('unicode-escape')) + newurl = re.sub(r'[^:]/{2,}', '/', newurl).strip().lstrip('.') + + if not unquote: + newurl = urllib.quote(newurl) + + return newurl + + +def parse_name(string): + path = fixurl(decode(string), unquote=False) + url_p = urlparse.urlparse(path.rstrip('/')) + name = (url_p.path.split('/')[-1] or + url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or + url_p.netloc.split('.', 1)[0]) + + return urllib.unquote(name) + + #@TODO: Move to utils in 0.4.10 -def fixurl(url): - return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip().rstrip('/') +def str2int(string): + try: + return int(string) + except: + pass + + ones = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", + "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", + "sixteen", "seventeen", "eighteen", "nineteen"] + tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", + "eighty", "ninety"] + + o_tuple = [(w, i) for i, w in enumerate(ones)] + t_tuple = [(w, i * 10) for i, w in enumerate(tens)] + + numwords = dict(o_tuple + t_tuple) + tokens = re.split(r"[\s-]+", string.lower()) + + try: + return sum(numwords[word] for word in tokens) + except: + return 0 + + +def parse_time(string): + if re.search("da(il)?y|today", string): + time = seconds_to_midnight() + + else: + this = re.compile("this", re.I) + regex = re.compile(r'(\d+|\w+)\s*(hr|hour|min|sec|)', re.I) + + time = sum(1 if this.match(v) else str2int(v) * + {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, '': 1}[u.lower()] + for v, u in regex.findall(string)) + + return time #@TODO: Move to utils in 0.4.10 @@ -56,22 +124,35 @@ def timestamp(): return int(time.time() * 1000) -def seconds_to_midnight(gmt=0): - now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) - - if now.hour == 0 and now.minute < 10: - midnight = now +#@TODO: Move to utils in 0.4.10 +def which(program): + """ + Works exactly like the unix command which + Courtesy of http://stackoverflow.com/a/377028/675646 + """ + isExe = lambda x: os.path.isfile(x) and os.access(x, os.X_OK) + + fpath, fname = os.path.split(program) + + if fpath: + if isExe(program): + return program else: - midnight = now + datetime.timedelta(days=1) + for path in os.environ['PATH'].split(os.pathsep): + exe_file = os.path.join(path.strip('"'), program) + if isExe(exe_file): + return exe_file + - td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now +def seconds_to_midnight(utc=None): + if utc is None: + now = datetime.datetime.today() + else: + now = datetime.datetime.utcnow() + datetime.timedelta(hours=utc) - if hasattr(td, 'total_seconds'): - res = td.total_seconds() - else: #@NOTE: work-around for python 2.5 and 2.6 missing datetime.timedelta.total_seconds - res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 + midnight = now.replace(hour=0, minute=1, second=0, microsecond=0) + datetime.timedelta(days=1) - return int(res) + return (midnight - now).seconds def replace_patterns(string, ruleslist): @@ -112,7 +193,10 @@ def parse_html_form(attr_str, html, input_names={}): else: inputs[name] = value - if input_names: + if not input_names: + #: No attribute check + return action, inputs + else: #: Check input attributes for key, val in input_names.items(): if key in inputs: @@ -122,14 +206,12 @@ def parse_html_form(attr_str, html, input_names={}): continue elif hasattr(val, "search") and re.match(val, inputs[key]): continue - break #: Attibute value does not match + else: + break #: Attibute value does not match else: break #: Attibute name does not match else: return action, inputs #: Passed attribute check - else: - #: No attribute check - return action, inputs return {}, None #: No matching form found @@ -145,8 +227,8 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" - __type__ = "hoster" - __version__ = "0.30" + __type__ = "plugin" + __version__ = "0.44" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -154,10 +236,7 @@ class Plugin(object): __description__ = """Base plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), - ("spoob" , "spoob@pyload.org" ), - ("mkaay" , "mkaay@mkaay.de" ), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] def __init__(self, core): @@ -165,10 +244,16 @@ class Plugin(object): self.init() + def __repr__(self): + return "<%(type)s %(name)s>" % {'type': self.__type__.capitalize(), + 'name': self.__name__} + + def _init(self, core): - self.pyload = core - self.info = {} #: Provide information in dict here - self.req = None + self.pyload = core + self.info = {} #: Provide information in dict here + self.req = None #: Browser instance, see `network.Browser` + self.last_html = None def init(self): @@ -180,33 +265,37 @@ class Plugin(object): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) - msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in messages if a)) - log("%(plugintype)s %(pluginname)s%(id)s: %(msg)s" + msg = u" | ".join(decode(a).strip() for a in messages if a) + log("%(plugintype)s %(pluginname)s: %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, - 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", 'msg' : msg}) def log_debug(self, *args): - if self.pyload.debug: - return self._log("debug", self.__type__, self.__name__, args) + if not self.pyload.debug: + return + self._log("debug", self.__type__, self.__name__, args) def log_info(self, *args): - return self._log("info", self.__type__, self.__name__, args) + self._log("info", self.__type__, self.__name__, args) def log_warning(self, *args): - return self._log("warning", self.__type__, self.__name__, args) + self._log("warning", self.__type__, self.__name__, args) def log_error(self, *args): - return self._log("error", self.__type__, self.__name__, args) + self._log("error", self.__type__, self.__name__, args) + if self.pyload.debug: + traceback.print_exc() def log_critical(self, *args): return self._log("critical", self.__type__, self.__name__, args) + if self.pyload.debug: + traceback.print_exc() def set_permissions(self, path): @@ -225,7 +314,7 @@ class Plugin(object): self.log_warning(_("Setting path mode failed"), e) try: - if os.name != "nt" and self.pyload.config.get("permission", "change_dl"): + if os.name is not "nt" and self.pyload.config.get("permission", "change_dl"): uid = pwd.getpwnam(self.pyload.config.get("permission", "user"))[2] gid = grp.getgrnam(self.pyload.config.get("permission", "group"))[2] os.chown(path, uid, gid) @@ -240,7 +329,7 @@ class Plugin(object): return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) - def set_config(self, option, value): + def set_config(self, option, value, plugin=None): """ Set config value for current plugin @@ -248,7 +337,7 @@ class Plugin(object): :param value: :return: """ - self.pyload.config.setPlugin(self.__name__, option, value) + self.pyload.config.setPlugin(plugin or self.__name__, option, value) def get_config(self, option, default="", plugin=None): @@ -287,21 +376,10 @@ class Plugin(object): self.pyload.db.delStorage(self.__name__, key) - def fail(self, reason): + def fail(self, msg): """ - Fail and give reason + Fail and give msg """ - raise Fail(encode(reason)) #@TODO: Remove `encode` in 0.4.10 - - - def error(self, reason="", type=_("Parse")): - if not reason: - type = _("Unknown") - - msg = _("%s error") % type.strip().capitalize() if type else _("Error") - msg += (": %s" % reason.strip()) if reason else "" - msg += _(" | Plugin may be out of date") - raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 @@ -318,17 +396,11 @@ class Plugin(object): :param decode: Wether to decode the output according to http header, should be True in most cases :return: Loaded content """ - if hasattr(self, 'pyfile') and self.pyfile.abort: - self.abort() - - url = fixurl(url) - - if not url or not isinstance(url, basestring): - self.fail(_("No url given")) - if self.pyload.debug: self.log_debug("LOAD URL " + url, - *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) + + url = fixurl(url) #: Recheck in 0.4.10 if req is None: req = self.req or self.pyload.requestFactory.getRequest(self.__name__) @@ -337,15 +409,17 @@ class Plugin(object): if isinstance(cookies, list): set_cookies(req.cj, cookies) - res = req.load(url, get, post, ref, bool(cookies), just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 + html = req.load(url, get, post, ref, bool(cookies), just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 #@TODO: Move to network in 0.4.10 if decode: - res = html_unescape(res) + html = html_unescape(html) #@TODO: Move to network in 0.4.10 if isinstance(decode, basestring): - res = decode(res, decode) + html = _decode(html, decode) #@NOTE: Use `utils.decode()` in 0.4.10 + + self.last_html = html if self.pyload.debug: frame = inspect.currentframe() @@ -356,15 +430,18 @@ class Plugin(object): with open(framefile, "wb") as f: del frame #: Delete the frame or it wont be cleaned - f.write(encode(res)) + f.write(encode(html)) except IOError, e: self.log_error(e) - if just_header: - #: Parse header + if not just_header: + return html + + else: + #@TODO: Move to network in 0.4.10 header = {'code': req.code} - for line in res.splitlines(): + for line in html.splitlines(): line = line.strip() if not line or ":" not in line: continue @@ -380,20 +457,20 @@ class Plugin(object): header[key] = [header[key], value] else: header[key] = value - res = header - return res + return header def clean(self): """ - Clean everything and remove references + Remove references """ try: + self.req.clearCookies() self.req.close() + except Exception: pass - for a in ("pyfile", "thread", "html", "req"): - if hasattr(self, a): - setattr(self, a, None) + else: + self.req = None |