diff options
Diffstat (limited to 'module/plugins/internal/Hoster.py')
| -rw-r--r-- | module/plugins/internal/Hoster.py | 502 |
1 files changed, 59 insertions, 443 deletions
diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index bc340e78f..d1b894c6f 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -2,19 +2,10 @@ from __future__ import with_statement -import inspect -import mimetypes import os -import random -import time -import urlparse - -from module.plugins.internal.Captcha import Captcha -from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, decode, encode, exists, fixurl, - parse_html_form, parse_html_tag_attr_value, parse_name, - replace_patterns, seconds_to_midnight, - set_cookie, set_cookies, timestamp) + +from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo +from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename @@ -24,40 +15,17 @@ def parse_fileInfo(klass, url="", html=""): return encode(info['name']), info['size'], info['status'], info['url'] -#@TODO: Remove in 0.4.10 -def getInfo(urls): - #: result = [ .. (name, size, status, url) .. ] - pass - - -#@TODO: Remove in 0.4.10 -def create_getInfo(klass): - def get_info(urls): - for url in urls: - if hasattr(klass, "URL_REPLACEMENTS"): - url = replace_patterns(url, klass.URL_REPLACEMENTS) - yield parse_fileInfo(klass, url) - - return get_info - - -#@NOTE: `check_abort` decorator -def check_abort(fn): - - def wrapper(self, *args, **kwargs): - self.check_abort() - return fn(self, *args, **kwargs) - - return wrapper - -class Hoster(Plugin): +class Hoster(Base): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.31" + __version__ = "0.32" __status__ = "testing" __pattern__ = r'^unmatchable$' + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("fallback_premium", "bool", "Fallback to free download if premium fails", True), + ("chk_filesize" , "bool", "Check file size" , True)] __description__ = """Base hoster plugin""" __license__ = "GPLv3" @@ -65,107 +33,39 @@ class Hoster(Plugin): def __init__(self, pyfile): - self._init(pyfile.m.core) - - #: Engage wan reconnection - self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10 + super(Base, self).__init__(pyfile) #: Enable simultaneous processing of multiple downloads - self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10 self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 - #: time.time() + wait in seconds - self.wait_until = 0 - self.waiting = False - - #: Account handler instance, see :py:class:`Account` - self.account = None - self.user = None #@TODO: Remove in 0.4.10 - - #: Associated pyfile instance, see `PyFile` - self.pyfile = pyfile - - self.thread = None #: Holds thread in future - #: Location where the last call to download was saved - self.last_download = "" + self.last_download = None #: Re match of the last call to `checkDownload` self.last_check = None - #: Js engine, see `JsEngine` - self.js = self.pyload.js - - #: Captcha stuff - self.captcha = Captcha(self) - - #: Some plugins store html code here - self.html = None - - #: Dict of the amount of retries already made - self.retries = {} - self.force_free = False #@TODO: Recheck in 0.4.10 + #: Restart flag + self.rst_free = False #@TODO: Recheck in 0.4.10 self._setup() self.init() - def _log(self, level, plugintype, pluginname, messages): - log = getattr(self.pyload.log, level) - msg = u" | ".join(decode(a).strip() for a in messages if a) - log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" - % {'plugintype': plugintype.upper(), - 'pluginname': pluginname, - 'id' : self.pyfile.id, - 'msg' : msg}) - - - @classmethod - def get_info(cls, url="", html=""): - url = fixurl(url, unquote=True) - return {'name' : parse_name(url), - 'size' : 0, - 'status': 3 if url else 8, - 'url' : url} - - - def init(self): - """ - Initialize the plugin (in addition to `__init__`) - """ - pass - - - def setup(self): - """ - Setup for enviroment and other things, called before downloading (possibly more than one time) - """ - pass + def _setup(self): + super(Base, self)._setup() + self.last_download = None + self.last_check = None + self.rst_free = False - def _setup(self): - #@TODO: Remove in 0.4.10 - self.html = "" - self.last_download = "" - self.pyfile.error = "" - try: - self.req.close() - except Exception: - pass - - if self.account: - self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) - self.chunk_limit = -1 #: -1 for unlimited - self.resume_download = True - self.premium = self.account.premium + def load_account(self): + if self.rst_free: + self.account = False + self.user = None #@TODO: Remove in 0.4.10 else: - self.req = self.pyload.requestFactory.getRequest(self.__name__) - self.chunk_limit = 1 - self.resume_download = False - self.premium = False - - return self.setup() + super(Base, self).load_account() + # self.rst_free = False def _process(self, thread): @@ -174,229 +74,44 @@ class Hoster(Plugin): """ self.thread = thread - if self.force_free: - self.account = False - else: - self.load_account() #@TODO: Move to PluginThread in 0.4.10 - self.force_free = False - self._setup() # self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 - self.pyfile.setStatus("starting") - self.check_abort() - self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) - return self.process(self.pyfile) - - - #: Deprecated method, use `_process` instead (Remove in 0.4.10) - def preprocessing(self, *args, **kwargs): - return self._process(*args, **kwargs) - - - def load_account(self): - if not self.account: - self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - - if not self.account: - self.account = False - self.user = None #@TODO: Remove in 0.4.10 - - else: - self.account.choose() - self.user = self.account.user #@TODO: Remove in 0.4.10 - if self.account.user is None: - self.account = False - - - def process(self, pyfile): - """ - The 'main' method of every plugin, you **have to** overwrite it - """ - raise NotImplementedError - - - def set_reconnect(self, reconnect): - self.log_debug("RECONNECT %s required" % ("" if reconnect else "not"), - "Previous wantReconnect: %s" % self.wantReconnect) - self.wantReconnect = bool(reconnect) - - - def set_wait(self, seconds, reconnect=None): - """ - Set a specific wait time later used with `wait` - - :param seconds: wait time in seconds - :param reconnect: True if a reconnect would avoid wait time - """ - wait_time = max(int(seconds), 1) - wait_until = time.time() + wait_time + 1 - - self.log_debug("WAIT set to %d seconds" % wait_time, - "Previous waitUntil: %f" % self.pyfile.waitUntil) - - self.pyfile.waitUntil = wait_until - - if reconnect is not None: - self.set_reconnect(reconnect) - - - def wait(self, seconds=None, reconnect=None): - """ - Waits the time previously set - """ - pyfile = self.pyfile - - if seconds is not None: - self.set_wait(seconds) - - if reconnect is not None: - self.set_reconnect(reconnect) - - self.waiting = True - - status = pyfile.status #@NOTE: Recheck in 0.4.10 - pyfile.setStatus("waiting") - - self.log_info(_("Waiting %d seconds...") % pyfile.waitUntil - time.time()) - - if self.wantReconnect: - self.log_info(_("Requiring reconnection...")) - if self.account: - self.log_warning("Ignore reconnection due logged account") - - if not self.wantReconnect or self.account: - while pyfile.waitUntil > time.time(): - self.check_abort() - time.sleep(2) - - else: - while pyfile.waitUntil > time.time(): - self.check_abort() - self.thread.m.reconnecting.wait(1) - - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - time.sleep(2) - - self.waiting = False - pyfile.status = status #@NOTE: Recheck in 0.4.10 - - - def skip(self, msg=""): - """ - Skip and give msg - """ - raise Skip(encode(msg or self.pyfile.error or self.pyfile.pluginname)) #@TODO: Remove `encode` in 0.4.10 - - - #@TODO: Remove in 0.4.10 - def fail(self, msg): - """ - Fail and give msg - """ - msg = msg.strip() - - if msg: - self.pyfile.error = msg - else: - msg = self.pyfile.error or (self.info['error'] if 'error' in self.info else self.pyfile.getStatusName()) - - raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - - - def error(self, msg="", type=_("Parse")): - type = _("%s error") % type.strip().capitalize() if type else _("Unknown") - msg = _("%(type)s: %(msg)s | Plugin may be out of date" - % {'type': type, 'msg': msg or self.pyfile.error}) - - self.fail(msg) - - - def abort(self, msg=""): - """ - Abort and give msg - """ - if msg: #@TODO: Remove in 0.4.10 - self.pyfile.error = encode(msg) - - raise Abort - - - #@TODO: Recheck in 0.4.10 - def offline(self, msg=""): - """ - Fail and indicate file is offline - """ - self.fail("offline") - - - #@TODO: Recheck in 0.4.10 - def temp_offline(self, msg=""): - """ - Fail and indicates file ist temporary offline, the core may take consequences - """ - self.fail("temp. offline") - + self.pyfile.setStatus("starting") - def retry(self, attemps=5, delay=1, msg=""): - """ - Retries and begin again from the beginning + try: + self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) #@TODO: Remove in 0.4.10 + self.process(self.pyfile) - :param attemps: number of maximum retries - :param delay: time to wait in seconds - :param msg: msg for retrying, will be passed to fail if attemps value was reached - """ - id = inspect.currentframe().f_back.f_lineno - if id not in self.retries: - self.retries[id] = 0 + self.check_abort() - if 0 < attemps <= self.retries[id]: - self.fail(msg or _("Max retries reached")) + self.log_debug("CHECK DOWNLOAD") #@TODO: Recheck in 0.4.10 + self._check_download() - self.wait(delay, False) + except Fail, e: #@TODO: Move to PluginThread in 0.4.10 + if self.get_config('fallback_premium', True) and self.premium: + self.log_warning(_("Premium download failed"), e) + self.restart() - self.retries[id] += 1 - raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 + else: + raise Fail(e) - def restart(self, msg=None, nopremium=False): + def restart(self, msg="", premium=False): if not msg: - msg = _("Fallback to free download") if nopremium else _("Restart") + msg = _("Simple restart") if premium else _("Fallback to free download") - if nopremium: + if not premium: if self.premium: - self.force_free = True + self.rst_free = True else: self.fail("%s | %s" % (msg, _("Download was already free"))) raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def fixurl(self, url, baseurl=None, unquote=None): - url = fixurl(url) - - if not baseurl: - baseurl = fixurl(self.pyfile.url) - - if not urlparse.urlparse(url).scheme: - url_p = urlparse.urlparse(baseurl) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - url = urlparse.urljoin(baseurl, url) - - return fixurl(url, unquote) - - - @check_abort - def load(self, *args, **kwargs): - return super(Hoster, self).load(*args, **kwargs) - - @check_abort def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ @@ -415,7 +130,7 @@ class Hoster(Plugin): self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) - url = self.fixurl(url, unquote=True) + url = self.fixurl(url) self.pyfile.name = parse_name(self.pyfile.name) #: Safe check @@ -475,27 +190,7 @@ class Hoster(Plugin): self.last_download = filename - return self.last_download - - - def check_abort(self): - if not self.pyfile.abort: - return - - if self.pyfile.status is 8: - self.fail() - - elif self.pyfile.status is 4: - self.skip(self.pyfile.statusname) - - elif self.pyfile.status is 1: - self.offline() - - elif self.pyfile.status is 6: - self.temp_offline() - - else: - self.abort() + return filename def check_filesize(self, file_size, size_tolerance=1024): @@ -537,7 +232,7 @@ class Hoster(Plugin): :return: dictionary key of the first rule that matched """ do_delete = False - last_download = fs_encode(self.last_download) + last_download = fs_encode(self.last_download) #@TODO: Recheck in 0.4.10 if not self.last_download or not exists(last_download): self.fail(self.pyfile.error or _("No file downloaded")) @@ -558,7 +253,7 @@ class Hoster(Plugin): elif hasattr(rule, "search"): m = rule.search(content) - if m: + if m is not None: do_delete = True self.last_check = m return name @@ -575,11 +270,25 @@ class Hoster(Plugin): self.last_download = "" #: Recheck in 0.4.10 + def _check_download(self): + if self.captcha.task and not self.last_download: + self.retry_captcha() + + elif self.check_file({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, + delete=True): + self.error(_("Empty file")) + + elif self.get_config('chk_filesize', False) and self.info.get('size'): + # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size + # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB + self.check_filesize(self.info['size'], size_tolerance=10485760) + + def check_traffic(self): if not self.account: return True - traffic = self.account.get_data(refresh=True)['trafficleft'] + traffic = self.account.get_data('trafficleft') if traffic is None: return False @@ -632,96 +341,3 @@ class Hoster(Plugin): def checkForSameFiles(self, *args, **kwargs): if self.pyload.config.get("download", "skip_existing"): return self.check_filedupe() - - - def direct_link(self, url, follow_location=None): - link = "" - - if follow_location is None: - redirect = 1 - - elif type(follow_location) is int: - redirect = max(follow_location, 1) - - else: - redirect = self.get_config("maxredirs", 10, "UserAgentSwitcher") - - for i in xrange(redirect): - try: - self.log_debug("Redirect #%d to: %s" % (i, url)) - header = self.load(url, just_header=True) - - except Exception: #: Bad bad bad... rewrite this part in 0.4.10 - res = self.load(url, - just_header=True, - req=self.pyload.requestFactory.getRequest(self.__name__)) - - header = {'code': req.code} - for line in res.splitlines(): - line = line.strip() - if not line or ":" not in line: - continue - - key, none, value = line.partition(":") - key = key.lower().strip() - value = value.strip() - - if key in header: - if type(header[key]) is list: - header[key].append(value) - else: - header[key] = [header[key], value] - else: - header[key] = value - - if 'content-disposition' in header: - link = url - - elif header.get('location'): - location = self.fixurl(header['location'], url) - - if header.get('code') == 302: - link = location - - if follow_location: - url = location - continue - - else: - extension = os.path.splitext(parse_name(url))[-1] - - if header.get('content-type'): - mimetype = header['content-type'].split(';')[0].strip() - - elif extension: - mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" - - else: - mimetype = "" - - if mimetype and (link or 'html' not in mimetype): - link = url - else: - link = "" - - break - - else: - try: - self.log_error(_("Too many redirects")) - - except Exception: - pass - - return link - - - def parse_html_form(self, attr_str="", input_names={}): - return parse_html_form(attr_str, self.html, input_names) - - - def get_password(self): - """ - Get the password the user provided in the package - """ - return self.pyfile.package().password or "" |