diff options
| -rw-r--r-- | module/plugins/internal/Hoster.py | 201 |
1 files changed, 140 insertions, 61 deletions
diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index b233f8755..333b9699f 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -5,9 +5,10 @@ from __future__ import with_statement import os import re -from module.plugins.internal.Base import Base, check_abort, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name -from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename +from module.network.HTTPRequest import BadHeader +from module.plugins.internal.Base import Base, create_getInfo, parse_fileInfo +from module.plugins.internal.Plugin import Fail, Retry +from module.plugins.internal.utils import encode, exists, fixurl, fs_join, parse_name class Hoster(Base): @@ -17,10 +18,10 @@ class Hoster(Base): __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("activated" , "bool", "Activated" , True), - ("use_premium" , "bool", "Use premium account if available" , True), - ("fallback_premium", "bool", "Fallback to free download if premium fails", True), - ("chk_filesize" , "bool", "Check file size" , True)] + __config__ = [("activated" , "bool", "Activated" , True), + ("use_premium" , "bool", "Use premium account if available" , True), + ("fallback" , "bool", "Fallback to free download if premium fails", True), + ("chk_filesize", "bool", "Check file size" , True)] __description__ = """Base hoster plugin""" __license__ = "GPLv3" @@ -44,13 +45,13 @@ class Hoster(Base): self.last_check = None #: Restart flag - self.rst_free = False #@TODO: Recheck in 0.4.10 + self.restart_free = False #@TODO: Recheck in 0.4.10 def setup_base(self): self.last_download = None self.last_check = None - self.rst_free = False + self.restart_free = False if self.account: self.chunk_limit = -1 #: -1 for unlimited @@ -61,39 +62,39 @@ class Hoster(Base): def load_account(self): - if self.rst_free: + if self.restart_free: self.account = False self.user = None #@TODO: Remove in 0.4.10 else: super(Hoster, self).load_account() - # self.rst_free = False + # self.restart_free = False def _process(self, thread): """ Handles important things to do before starting """ - self.thread = thread + self.log_debug("Plugin version: " + self.__version__) + self.log_debug("Plugin status: " + self.__status__) + + if self.__status__ is "broken": + self.fail(_("Plugin is currently broken")) + self.thread = thread self._setup() # self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 - self.check_abort() + self.check_status() self.pyfile.setStatus("starting") try: - self.log_debug("PROCESS URL " + self.pyfile.url, - "PLUGIN VERSION %s" % self.__version__) #@TODO: Remove in 0.4.10 self.process(self.pyfile) - - self.check_abort() - - self.log_debug("CHECK DOWNLOAD") #@TODO: Recheck in 0.4.10 + self.check_status() self._check_download() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 - if self.get_config('fallback_premium', True) and self.premium: + if self.get_config('fallback', True) and self.premium: self.log_warning(_("Premium download failed"), e) self.restart(premium=False) @@ -101,7 +102,62 @@ class Hoster(Base): raise Fail(e) - @check_abort + def is_download(self, url, resume=None, redirect=True): + link = False + maxredirs = 10 + + if resume is None: + resume = self.resume_download + + if type(redirect) is int: + maxredirs = max(redirect, 1) + + elif redirect: + maxredirs = self.get_config("maxredirs", default=maxredirs, plugin="UserAgentSwitcher") + + for i in xrange(maxredirs): + self.log_debug("Redirect #%d to: %s" % (i, url)) + + header = self.load(url, just_header=True) + + if 'content-disposition' in header: + link = url + + elif header.get('location'): + location = self.fixurl(header.get('location'), url) + code = header.get('code') + + if code is 302: + link = location + + elif code is 301: + url = location + if redirect: + continue + + if resume: + url = location + continue + + else: + mimetype = "" + contenttype = header.get('content-type') + extension = os.path.splitext(parse_name(url))[-1] + + if contenttype: + mimetype = contenttype.split(';')[0].strip() + + elif extension: + mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" + + if mimetype and (link or 'html' not in mimetype): + link = url + else: + link = False + + return link + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True, resume=None, chunks=None): """ Downloads the content at url to download folder @@ -115,13 +171,19 @@ class Hoster(Base): the filename will be changed if needed :return: The location where the file was saved """ + self.check_status() + if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) - url = self.fixurl(url) + dl_url = self.is_download(url, resume) + dl_basename = parse_name(self.pyfile.name) - self.pyfile.name = parse_name(self.pyfile.name) #: Safe check + self.pyfile.name = dl_basename + + if not dl_url: + self.error("Invalid download url") self.captcha.correct() @@ -130,63 +192,75 @@ class Hoster(Base): self.pyfile.setStatus("downloading") - download_folder = self.pyload.config.get("general", "download_folder") - download_location = fs_join(download_folder, self.pyfile.package().folder) + dl_folder = self.pyload.config.get("general", "download_folder") + dl_dirname = os.path.join(dl_folder, self.pyfile.package().folder) + dl_filename = os.path.join(dl_dirname, dl_basename) + + dl_dir = encode(dl_dirname) + dl_file = encode(dl_filename) #@TODO: Move safe-filename check to HTTPDownload in 0.4.10 - if not exists(download_location): + if not exists(dl_dir): try: - os.makedirs(download_location) + os.makedirs(dl_dir) except Exception, e: self.fail(e) - self.set_permissions(download_location) + self.set_permissions(dl_dir) - location = fs_decode(download_location) - filename = os.path.join(location, safe_filename(self.pyfile.name)) #@TODO: Move `safe_filename` check to HTTPDownload in 0.4.10 + self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, dl_url, dl_filename) + self.check_status() - self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename) + dl_chunks = self.pyload.config.get("download", "chunks") + chunk_limit = chunks or self.chunk_limit or -1 - self.check_abort() + if dl_chunks is -1 or chunk_limit is -1: + chunks = max(dl_chunks, chunk_limit) + else: + chunks = min(dl_chunks, chunk_limit) - chunks = min(self.pyload.config.get("download", "chunks"), chunks or self.chunk_limit or -1) - - if resume is None: - resume = self.resume_download + resume = self.resume_download if resume is None else bool(resume) try: - newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, + newname = self.req.httpDownload(dl_url, dl_file, get=get, post=post, ref=ref, cookies=cookies, chunks=chunks, resume=resume, progressNotify=self.pyfile.setProgress, disposition=disposition) + except BadHeader, e: + if e.code in (404, 410): + self.pyfile.setStatus("offline") + raise BadHeader(e) + finally: self.pyfile.size = self.req.size #@TODO: Recheck in 0.4.10 if disposition and newname: - finalname = parse_name(newname).split(' filename*=')[0] + safename = parse_name(newname.split(' filename*=')[0]) - if finalname != newname: + if safename != newname: try: - oldname_enc = fs_join(download_location, newname) - newname_enc = fs_join(download_location, finalname) - os.rename(oldname_enc, newname_enc) + old_file = fs_join(dl_dirname, newname) + new_file = fs_join(dl_dirname, safename) + os.rename(old_file, new_file) except OSError, e: self.log_warning(_("Error renaming `%s` to `%s`") - % (newname, finalname), e) - finalname = newname + % (newname, safename), e) + safename = newname - self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname)) + self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, safename)) - self.pyfile.name = finalname - filename = os.path.join(location, finalname) + self.pyfile.name = safename - self.set_permissions(fs_encode(filename)) + dl_filename = os.path.join(dl_dirname, safename) + dl_file = encode(dl_filename) - self.last_download = filename + self.set_permissions(dl_file) - return filename + self.last_download = dl_filename + + return dl_filename def check_filesize(self, file_size, size_tolerance=1024): @@ -199,18 +273,18 @@ class Hoster(Base): if not self.last_download: return - download_location = fs_encode(self.last_download) - download_size = os.stat(download_location).st_size + dl_location = encode(self.last_download) + dl_size = os.stat(dl_location).st_size - if download_size < 1: + if dl_size < 1: self.fail(_("Empty file")) elif file_size > 0: - diff = abs(file_size - download_size) + diff = abs(file_size - dl_size) if diff > size_tolerance: self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") - % (file_size, download_size)) + % (file_size, dl_size)) elif diff != 0: self.log_warning(_("File size is not equal to expected size")) @@ -228,7 +302,7 @@ class Hoster(Base): :return: dictionary key of the first rule that matched """ do_delete = False - last_download = fs_encode(self.last_download) #@TODO: Recheck in 0.4.10 + last_download = encode(self.last_download) #@TODO: Recheck in 0.4.10 if not self.last_download or not exists(last_download): self.fail(self.pyfile.error or _("No file downloaded")) @@ -267,6 +341,8 @@ class Hoster(Base): def _check_download(self): + self.log_info(_("Checking downloaded file...")) + if self.captcha.task and not self.last_download: self.retry_captcha() @@ -279,6 +355,9 @@ class Hoster(Base): # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB self.check_filesize(self.info['size'], size_tolerance=10485760) + else: + self.log_info(_("File is OK")) + def check_traffic(self): if not self.account: @@ -319,18 +398,18 @@ class Hoster(Base): if pyfile.status in (0, 5, 7, 12): #: (finished, waiting, starting, downloading) self.skip(pyfile.pluginname) - download_folder = self.pyload.config.get("general", "download_folder") - package_folder = pack.folder if self.pyload.config.get("general", "folder_per_package") else "" - download_location = fs_join(download_folder, package_folder, self.pyfile.name) + dl_folder = self.pyload.config.get("general", "download_folder") + package_folder = pack.folder if self.pyload.config.get("general", "folder_per_package") else "" + dl_location = fs_join(dl_folder, package_folder, self.pyfile.name) - if not exists(download_location): + if not exists(dl_location): return pyfile = self.pyload.db.findDuplicates(self.pyfile.id, package_folder, self.pyfile.name) if pyfile: self.skip(pyfile[0]) - size = os.stat(download_location).st_size + size = os.stat(dl_location).st_size if size >= self.pyfile.size: self.skip(_("File exists")) |