diff options
author | Walter Purcaro <vuolter@users.noreply.github.com> | 2015-10-18 19:08:02 +0200 |
---|---|---|
committer | Walter Purcaro <vuolter@users.noreply.github.com> | 2015-10-18 19:08:02 +0200 |
commit | b8e23365241337ebb8a42a0d93cbc67afbbee480 (patch) | |
tree | 5293dadec7c4aecf0fe32d9f6986c9987c1fbed9 /module | |
parent | [Hoster] Fix download routine + new method is_download (diff) | |
download | pyload-b8e23365241337ebb8a42a0d93cbc67afbbee480.tar.xz |
[Plugin] Minimize code + spare fixes
Diffstat (limited to 'module')
-rw-r--r-- | module/plugins/internal/MultiHoster.py | 56 | ||||
-rw-r--r-- | module/plugins/internal/Plugin.py | 312 | ||||
-rw-r--r-- | module/plugins/internal/SimpleHoster.py | 117 |
3 files changed, 66 insertions, 419 deletions
diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 5655571b8..a31a6843b 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -13,11 +13,11 @@ class MultiHoster(SimpleHoster): __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("activated" , "bool", "Activated" , True), - ("use_premium" , "bool", "Use premium account if available" , True), - ("fallback_premium", "bool", "Fallback to free download if premium fails", True), - ("chk_filesize" , "bool", "Check file size" , True), - ("revertfailed" , "bool", "Revert to standard download if fails" , True)] + __config__ = [("activated" , "bool", "Activated" , True), + ("use_premium" , "bool", "Use premium account if available" , True), + ("fallback" , "bool", "Fallback to free download if premium fails", True), + ("chk_filesize", "bool", "Check file size" , True), + ("revertfailed", "bool", "Revert to standard download if fails" , True)] __description__ = """Multi hoster plugin""" __license__ = "GPLv3" @@ -66,47 +66,13 @@ class MultiHoster(SimpleHoster): self.direct_dl = direct_dl - def process(self, pyfile): + def _process(self, thread): try: - self.prepare() - self.check_info() #@TODO: Remove in 0.4.10 + super(MultiHoster, self)._process(thread) - if self.direct_dl: - self.log_info(_("Looking for direct download link...")) - self.handle_direct(pyfile) - - if self.link or was_downloaded(): - self.log_info(_("Direct download link detected")) - else: - self.log_info(_("Direct download link not found")) - - if not self.link and not self.last_download: - self.preload() - - self.check_errors() - self.check_status(getinfo=False) - - if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): - self.log_info(_("Processing as premium download...")) - self.handle_premium(pyfile) - - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()): - self.log_info(_("Processing as free download...")) - self.handle_free(pyfile) - - if not self.last_download: - self.log_info(_("Downloading file...")) - self.download(self.link, disposition=self.DISPOSITION) - - self.check_download() - - except Fail, e: #@TODO: Move to PluginThread in 0.4.10 - if self.premium: - self.log_warning(_("Premium download failed")) - self.restart(premium=False) - - elif self.get_config("revertfailed", True) and \ - self.pyload.pluginManager.hosterPlugins[self.classname].get('new_module'): + except Fail, e: + if self.get_config("revertfailed", True) and \ + self.pyload.pluginManager.hosterPlugins[self.classname].get('new_module'): hdict = self.pyload.pluginManager.hosterPlugins[self.classname] tmp_module = hdict['new_module'] @@ -122,7 +88,7 @@ class MultiHoster(SimpleHoster): self.restart(_("Revert to original hoster plugin")) else: - raise Fail(encode(e)) #@TODO: Remove `encode` in 0.4.10 + raise Fail(e) def handle_premium(self, pyfile): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index f5db49d8b..0b5561df8 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -2,284 +2,19 @@ from __future__ import with_statement -import datetime import inspect import os -import re -import sys -import time -import traceback -import urllib -import urlparse - -import pycurl if os.name is not "nt": import grp import pwd -from module.common.json_layer import json_dumps, json_loads -from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 -from module.utils import (fs_encode, fs_decode, get_console_encoding, html_unescape, - parseFileSize as parse_size, save_join as fs_join) - - -#@TODO: Move to utils in 0.4.10 -def isiterable(obj): - return hasattr(obj, "__iter__") - - -#@TODO: Move to utils in 0.4.10 -def decode(string, encoding=None): - """Encoded string (default to UTF-8) -> unicode string""" - if type(string) is str: - try: - res = unicode(string, encoding or "utf-8") - - except UnicodeDecodeError, e: - if encoding: - raise UnicodeDecodeError(e) - - encoding = get_console_encoding(sys.stdout.encoding) - res = unicode(string, encoding) - - elif type(string) is unicode: - res = string - - else: - res = unicode(string) - - return res - - -#@TODO: Remove in 0.4.10 -def _decode(*args, **kwargs): - return decode(*args, **kwargs) - - -#@TODO: Move to utils in 0.4.10 -def encode(string, encoding=None, decoding=None): - """Unicode or decoded string -> encoded string (default to UTF-8)""" - if type(string) is unicode: - res = string.encode(encoding or "utf-8") - - elif type(string) is str: - res = encode(decode(string, decoding), encoding) - - else: - res = str(string) - - return res - - -#@TODO: Move to utils in 0.4.10 -def exists(path): - if os.path.exists(path): - if os.name is "nt": - dir, name = os.path.split(path.rstrip(os.sep)) - return name in os.listdir(dir) - else: - return True - else: - return False - - -def fixurl(url, unquote=None): - old = url - url = urllib.unquote(url) - - if unquote is None: - unquote = url is old - - url = html_unescape(decode(url).decode('unicode-escape')) - url = re.sub(r'(?<!:)/{2,}', '/', url).strip().lstrip('.') - - if not unquote: - url = urllib.quote(url) - - return url - - -def parse_name(string): - path = fixurl(decode(string), unquote=False) - url_p = urlparse.urlparse(path.rstrip('/')) - name = (url_p.path.split('/')[-1] or - url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or - url_p.netloc.split('.', 1)[0]) - - return urllib.unquote(name) - - -#@TODO: Move to utils in 0.4.10 -def str2int(string): - try: - return int(string) - except: - pass - - ones = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", - "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", - "sixteen", "seventeen", "eighteen", "nineteen"] - tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", - "eighty", "ninety"] - - o_tuple = [(w, i) for i, w in enumerate(ones)] - t_tuple = [(w, i * 10) for i, w in enumerate(tens)] - - numwords = dict(o_tuple + t_tuple) - tokens = re.split(r"[\s\-]+", string.lower()) - - try: - return sum(numwords[word] for word in tokens) - except: - return 0 - - -def parse_time(string): - if re.search("da(il)?y|today", string): - seconds = seconds_to_midnight() - - else: - regex = re.compile(r'(\d+| (?:this|an?) )\s*(hr|hour|min|sec|)', re.I) - seconds = sum((int(v) if v.strip() not in ("this", "a", "an") else 1) * - {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, '': 1}[u.lower()] - for v, u in regex.findall(string)) - return seconds - - -#@TODO: Move to utils in 0.4.10 -def timestamp(): - return int(time.time() * 1000) - - -#@TODO: Move to utils in 0.4.10 -def which(program): - """ - Works exactly like the unix command which - Courtesy of http://stackoverflow.com/a/377028/675646 - """ - isExe = lambda x: os.path.isfile(x) and os.access(x, os.X_OK) - - fpath, fname = os.path.split(program) - - if fpath: - if isExe(program): - return program - else: - for path in os.environ['PATH'].split(os.pathsep): - exe_file = os.path.join(path.strip('"'), program) - if isExe(exe_file): - return exe_file - - -#@TODO: Move to utils in 0.4.10 -def format_exc(frame=None): - """ - Format call-stack and display exception information (if availible) - """ - exception_info = sys.exc_info() - callstack_list = traceback.extract_stack(frame) - callstack_list = callstack_list[:-1] - - exception_desc = "" - if exception_info[0] is not None: - exception_callstack_list = traceback.extract_tb(exception_info[2]) - if callstack_list[-1][0] == exception_callstack_list[0][0]: #Does this exception belongs to us? - callstack_list = callstack_list[:-1] - callstack_list.extend(exception_callstack_list) - exception_desc = "".join(traceback.format_exception_only(exception_info[0], exception_info[1])) - - traceback_str = "Traceback (most recent call last):\n" - traceback_str += "".join(traceback.format_list(callstack_list)) - traceback_str += exception_desc - traceback_str = traceback_str[:-1] #Removing the last '\n' - return traceback_str - -def seconds_to_nexthour(strict=False): - now = datetime.datetime.today() - nexthour = now.replace(minute=0 if strict else 1, second=0, microsecond=0) + datetime.timedelta(hours=1) - return (nexthour - now).seconds - - -def seconds_to_midnight(utc=None, strict=False): - if utc is None: - now = datetime.datetime.today() - else: - now = datetime.datetime.utcnow() + datetime.timedelta(hours=utc) - - midnight = now.replace(hour=0, minute=0 if strict else 1, second=0, microsecond=0) + datetime.timedelta(days=1) - - return (midnight - now).seconds - - -def replace_patterns(string, ruleslist): - for r in ruleslist: - rf, rt = r - string = re.sub(rf, rt, string) - return string - - -#@TODO: Remove in 0.4.10 and fix CookieJar.setCookie -def set_cookie(cj, domain, name, value): - return cj.setCookie(domain, name, encode(value)) - - -def set_cookies(cj, cookies): - for cookie in cookies: - if isinstance(cookie, tuple) and len(cookie) == 3: - set_cookie(cj, *cookie) - - -def parse_html_tag_attr_value(attr_name, tag): - m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^>\s\"'][^>\s]*)\1" % attr_name, tag, re.I) - return m.group(2) if m else None - - -def parse_html_form(attr_str, html, input_names={}): - for form in re.finditer(r"(?P<TAG><form[^>]*%s[^>]*>)(?P<CONTENT>.*?)</?(form|body|html)[^>]*>" % attr_str, - html, re.I | re.S): - inputs = {} - action = parse_html_tag_attr_value("action", form.group('TAG')) - - for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.I | re.S): - name = parse_html_tag_attr_value("name", inputtag.group(1)) - if name: - value = parse_html_tag_attr_value("value", inputtag.group(1)) - if not value: - inputs[name] = inputtag.group(3) or "" - else: - inputs[name] = value - - if not input_names: - #: No attribute check - return action, inputs - else: - #: Check input attributes - for key, val in input_names.items(): - if key in inputs: - if isinstance(val, basestring) and inputs[key] is val: - continue - elif isinstance(val, tuple) and inputs[key] in val: - continue - elif hasattr(val, "search") and re.match(val, inputs[key]): - continue - else: - break #: Attibute value does not match - else: - break #: Attibute name does not match - else: - return action, inputs #: Passed attribute check - - return {}, None #: No matching form found +import pycurl +import module.plugins.internal.utils as utils -#@TODO: Move to utils in 0.4.10 -def chunks(iterable, size): - it = iter(iterable) - item = list(islice(it, size)) - while item: - yield item - item = list(islice(it, size)) +from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 +from module.plugins.internal.utils import * class Plugin(object): @@ -336,38 +71,37 @@ class Plugin(object): def log_debug(self, *args, **kwargs): self._log("debug", self.__type__, self.__name__, args) - if self.pyload.debug and kwargs.get('trace', False): - self.log_exc("debug") + if self.pyload.debug and kwargs.get('trace'): + self.print_exc() def log_info(self, *args, **kwargs): self._log("info", self.__type__, self.__name__, args) - if kwargs.get('trace', False): - self.log_exc("info") + if self.pyload.debug and kwargs.get('trace'): + self.print_exc() def log_warning(self, *args, **kwargs): self._log("warning", self.__type__, self.__name__, args) - if kwargs.get('trace', False): - self.log_exc("warning") + if self.pyload.debug and kwargs.get('trace'): + self.print_exc() def log_error(self, *args, **kwargs): self._log("error", self.__type__, self.__name__, args) - if kwargs.get('trace', False): - self.log_exc("error") + if self.pyload.debug and kwargs.get('trace', True): + self.print_exc() def log_critical(self, *args, **kwargs): self._log("critical", self.__type__, self.__name__, args) if kwargs.get('trace', True): - self.log_exc("critical") + self.print_exc() - def log_exc(self, level): + def print_exc(self): frame = inspect.currentframe() - log = getattr(self.pyload.log, level) - log(format_exc(frame.f_back)) + print format_exc(frame.f_back) del frame @@ -427,7 +161,7 @@ class Plugin(object): Saves a value persistently to the database """ value = map(decode, value) if isiterable(value) else decode(value) - entry = json_dumps(value).encode('base64') + entry = json.dumps(value).encode('base64') self.pyload.db.setStorage(self.classname, key, entry) @@ -441,12 +175,12 @@ class Plugin(object): if entry is None: value = default else: - value = json_loads(entry.decode('base64')) + value = json.loads(entry.decode('base64')) else: if not entry: value = default else: - value = dict((k, json_loads(v.decode('base64'))) for k, v in value.items()) + value = dict((k, json.loads(v.decode('base64'))) for k, v in value.items()) return value @@ -506,8 +240,8 @@ class Plugin(object): req.http.c.setopt(pycurl.FOLLOWLOCATION, 1) elif type(redirect) is int: - req.http.c.setopt(pycurl.MAXREDIRS, - self.get_config("maxredirs", 5, plugin="UserAgentSwitcher")) + maxredirs = self.get_config("maxredirs", default=5, plugin="UserAgentSwitcher") + req.http.c.setopt(pycurl.MAXREDIRS, maxredirs) #@TODO: Move to network in 0.4.10 if decode: @@ -515,7 +249,7 @@ class Plugin(object): #@TODO: Move to network in 0.4.10 if isinstance(decode, basestring): - html = _decode(html, decode) #@NOTE: Use `utils.decode()` in 0.4.10 + html = utils.decode(html, decode) self.last_html = html @@ -544,13 +278,15 @@ class Plugin(object): else: #@TODO: Move to network in 0.4.10 header = {'code': req.code} + for line in html.splitlines(): line = line.strip() if not line or ":" not in line: continue key, none, value = line.partition(":") - key = key.strip().lower() + + key = key.strip().lower() value = value.strip() if key in header: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index a4e01249e..4ab305f20 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -9,8 +9,10 @@ import time from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, encode, parse_name, parse_size, parse_time, replace_patterns, seconds_to_midnight, set_cookie, set_cookies -from module.utils import fixup, fs_encode +from module.plugins.internal.Plugin import Fail +from module.plugins.internal.utils import (encode, fixup, parse_name, parse_size, + parse_time, replace_patterns, seconds_to_midnight, + set_cookie, set_cookies) class SimpleHoster(Hoster): @@ -20,10 +22,11 @@ class SimpleHoster(Hoster): __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("activated" , "bool", "Activated" , True), - ("use_premium" , "bool", "Use premium account if available" , True), - ("fallback_premium", "bool", "Fallback to free download if premium fails", True), - ("chk_filesize" , "bool", "Check file size" , True)] + __config__ = [("activated" , "bool", "Activated" , True), + ("use_premium" , "bool", "Use premium account if available" , True), + ("fallback" , "bool", "Fallback to free download if premium fails" , True), + ("chk_filesize", "bool", "Check file size" , True), + ("max_wait" , "int" , "Reconnect if waiting time is greater than minutes", 10 )] __description__ = """Simple hoster plugin""" __license__ = "GPLv3" @@ -133,7 +136,6 @@ class SimpleHoster(Hoster): @classmethod def get_info(cls, url="", html=""): info = super(SimpleHoster, cls).get_info(url) - info.update(cls.api_info(url)) if not html and info['status'] is not 2: @@ -148,7 +150,7 @@ class SimpleHoster(Hoster): except BadHeader, e: info['error'] = "%d: %s" % (e.code, e.content) - if e.code is 404: + if e.code in (404, 410): info['status'] = 1 elif e.code is 503: @@ -200,7 +202,8 @@ class SimpleHoster(Hoster): def setup(self): - self.resume_download = self.multiDL = self.premium + self.multiDL = self.premium + self.resume_download = self.premium def prepare(self): @@ -249,7 +252,6 @@ class SimpleHoster(Hoster): def process(self, pyfile): self.prepare() - self.check_info() #@TODO: Remove in 0.4.10 if self.leech_dl: self.log_info(_("Processing as debrid download...")) @@ -271,8 +273,8 @@ class SimpleHoster(Hoster): if not self.link and not self.last_download: self.preload() - if self.info.get('status', 3) is 3: #@TODO: Recheck in 0.4.10 - self.check_info() + if self.info.get('status', 3) is not 2: + self.grab_info() if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as premium download...")) @@ -286,12 +288,15 @@ class SimpleHoster(Hoster): self.log_info(_("Downloading file...")) self.download(self.link, disposition=self.DISPOSITION) + + def _check_download(self): + super(SimpleHoster, self)._check_download() self.check_download() def check_download(self): - self.log_info(_("Checking downloaded file...")) - self.log_debug("Using default check rules...") + self.log_debug("Performing default check rules...") + for r, p in self.FILE_ERRORS: errmsg = self.check_file({r: re.compile(p)}) if errmsg is not None: @@ -308,12 +313,12 @@ class SimpleHoster(Hoster): self.restart(errmsg) else: if self.CHECK_FILE: - self.log_debug("Using custom check rules...") - with open(fs_encode(self.last_download), "rb") as f: + self.log_debug("Performing custom check rules...") + + with open(encode(self.last_download), "rb") as f: self.html = f.read(1048576) #@TODO: Recheck in 0.4.10 - self.check_errors() - self.log_info(_("No errors found")) + self.check_errors() def check_errors(self): @@ -346,7 +351,7 @@ class SimpleHoster(Hoster): self.log_warning(errmsg) wait_time = parse_time(errmsg) - self.wait(wait_time, reconnect=wait_time > 300) + self.wait(wait_time, reconnect=wait_time > self.get_config("max_wait", 10) * 60) self.restart(_("Download limit exceeded")) if self.HAPPY_HOUR_PATTERN and re.search(self.HAPPY_HOUR_PATTERN, self.html): @@ -369,7 +374,7 @@ class SimpleHoster(Hoster): if re.search('limit|wait|slot', errmsg, re.I): wait_time = parse_time(errmsg) - self.wait(wait_time, reconnect=wait_time > 300) + self.wait(wait_time, reconnect=wait_time > self.get_config("max_wait", 10) * 60) self.restart(_("Download limit exceeded")) elif re.search('country|ip|region|nation', errmsg, re.I): @@ -410,80 +415,20 @@ class SimpleHoster(Hoster): waitmsg = m.group(0).strip() wait_time = parse_time(waitmsg) - self.wait(wait_time, reconnect=wait_time > 300) + self.wait(wait_time, reconnect=wait_time > self.get_config("max_wait", 10) * 60) self.info.pop('error', None) - def check_status(self, getinfo=True): - if not self.info or getinfo: - self.log_info(_("Updating file info...")) - old_info = self.info.copy() - self.info.update(self.get_info(self.pyfile.url, self.html)) - self.log_debug("File info: %s" % self.info) - self.log_debug("Previous file info: %s" % old_info) - - try: - status = self.info['status'] or 14 - - if status is 1: - self.offline() - - elif status is 6: - self.temp_offline() - - elif status is 8: - self.fail() - - finally: - self.log_info(_("File status: ") + self.pyfile.getStatusName()) - - - def check_name_size(self, getinfo=True): - if not self.info or getinfo: - self.log_info(_("Updating file info...")) - old_info = self.info.copy() - self.info.update(self.get_info(self.pyfile.url, self.html)) - self.log_debug("File info: %s" % self.info) - self.log_debug("Previous file info: %s" % old_info) - - name = self.info.get('name') - size = self.info.get('size') - - if name and name is not self.info.get('url'): - self.pyfile.name = name - else: - name = self.pyfile.name - - if size > 0: - self.pyfile.size = int(self.info['size']) #@TODO: Fix int conversion in 0.4.10 - else: - size = self.pyfile.size - - self.log_info(_("File name: ") + name) - self.log_info(_("File size: %s bytes") % size or "N/D") - - - #@TODO: Rewrite in 0.4.10 - def check_info(self): - self.check_name_size() - - if self.html: - self.check_errors() - self.check_name_size() - - self.check_status(getinfo=False) - - #: Deprecated method (Remove in 0.4.10) def get_fileInfo(self): - self.info = {} - self.check_info() + self.info.clear() + self.grab_info() return self.info def handle_direct(self, pyfile): - self.link = self.direct_link(pyfile.url, self.resume_download) + self.link = self.is_download(pyfile.url) def handle_multi(self, pyfile): #: Multi-hoster handler @@ -492,7 +437,7 @@ class SimpleHoster(Hoster): def handle_free(self, pyfile): if not self.LINK_FREE_PATTERN: - self.log_error(_("Free download not implemented")) + self.log_warning(_("Free download not implemented")) m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: @@ -503,7 +448,7 @@ class SimpleHoster(Hoster): def handle_premium(self, pyfile): if not self.LINK_PREMIUM_PATTERN: - self.log_error(_("Premium download not implemented")) + self.log_warning(_("Premium download not implemented")) self.restart(premium=False) m = re.search(self.LINK_PREMIUM_PATTERN, self.html) |