[Hoster] Fix download routine + new method is_download

1 files changed, 140 insertions, 61 deletions

diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py
index b233f8755..333b9699f 100644
--- a/module/plugins/internal/Hoster.py
+++ b/module/plugins/internal/Hoster.py
@@ -5,9 +5,10 @@ from __future__ import with_statement
 import os
 import re
 
-from module.plugins.internal.Base import Base, check_abort, create_getInfo, parse_fileInfo
-from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name
-from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename
+from module.network.HTTPRequest import BadHeader
+from module.plugins.internal.Base import Base, create_getInfo, parse_fileInfo
+from module.plugins.internal.Plugin import Fail, Retry
+from module.plugins.internal.utils import encode, exists, fixurl, fs_join, parse_name
 
 
 class Hoster(Base):
@@ -17,10 +18,10 @@ class Hoster(Base):
     __status__  = "testing"
 
     __pattern__ = r'^unmatchable$'
-    __config__  = [("activated"       , "bool", "Activated"                                 , True),
-                   ("use_premium"     , "bool", "Use premium account if available"          , True),
-                   ("fallback_premium", "bool", "Fallback to free download if premium fails", True),
-                   ("chk_filesize"    , "bool", "Check file size"                           , True)]
+    __config__  = [("activated"   , "bool", "Activated"                                 , True),
+                   ("use_premium" , "bool", "Use premium account if available"          , True),
+                   ("fallback"    , "bool", "Fallback to free download if premium fails", True),
+                   ("chk_filesize", "bool", "Check file size"                           , True)]
 
     __description__ = """Base hoster plugin"""
     __license__     = "GPLv3"
@@ -44,13 +45,13 @@ class Hoster(Base):
         self.last_check = None
 
         #: Restart flag
-        self.rst_free = False  #@TODO: Recheck in 0.4.10
+        self.restart_free = False  #@TODO: Recheck in 0.4.10
 
 
     def setup_base(self):
         self.last_download = None
         self.last_check    = None
-        self.rst_free      = False
+        self.restart_free  = False
 
         if self.account:
             self.chunk_limit     = -1  #: -1 for unlimited
@@ -61,39 +62,39 @@ class Hoster(Base):
 
 
     def load_account(self):
-        if self.rst_free:
+        if self.restart_free:
             self.account = False
             self.user    = None  #@TODO: Remove in 0.4.10
         else:
             super(Hoster, self).load_account()
-            # self.rst_free = False
+            # self.restart_free = False
 
 
     def _process(self, thread):
         """
         Handles important things to do before starting
         """
-        self.thread = thread
+        self.log_debug("Plugin version: " + self.__version__)
+        self.log_debug("Plugin status: " + self.__status__)
+
+        if self.__status__ is "broken":
+            self.fail(_("Plugin is currently broken"))
 
+        self.thread = thread
         self._setup()
 
         # self.pyload.hookManager.downloadPreparing(self.pyfile)  #@TODO: Recheck in 0.4.10
-        self.check_abort()
+        self.check_status()
 
         self.pyfile.setStatus("starting")
 
         try:
-            self.log_debug("PROCESS URL " + self.pyfile.url,
-                           "PLUGIN VERSION %s" % self.__version__)  #@TODO: Remove in 0.4.10
             self.process(self.pyfile)
-
-            self.check_abort()
-
-            self.log_debug("CHECK DOWNLOAD")  #@TODO: Recheck in 0.4.10
+            self.check_status()
             self._check_download()
 
         except Fail, e:  #@TODO: Move to PluginThread in 0.4.10
-            if self.get_config('fallback_premium', True) and self.premium:
+            if self.get_config('fallback', True) and self.premium:
                 self.log_warning(_("Premium download failed"), e)
                 self.restart(premium=False)
 
@@ -101,7 +102,62 @@ class Hoster(Base):
                 raise Fail(e)
 
 
-    @check_abort
+    def is_download(self, url, resume=None, redirect=True):
+        link      = False
+        maxredirs = 10
+
+        if resume is None:
+            resume = self.resume_download
+
+        if type(redirect) is int:
+            maxredirs = max(redirect, 1)
+
+        elif redirect:
+            maxredirs = self.get_config("maxredirs", default=maxredirs, plugin="UserAgentSwitcher")
+
+        for i in xrange(maxredirs):
+            self.log_debug("Redirect #%d to: %s" % (i, url))
+
+            header = self.load(url, just_header=True)
+
+            if 'content-disposition' in header:
+                link = url
+
+            elif header.get('location'):
+                location = self.fixurl(header.get('location'), url)
+                code     = header.get('code')
+
+                if code is 302:
+                    link = location
+
+                elif code is 301:
+                    url = location
+                    if redirect:
+                        continue
+
+                if resume:
+                    url = location
+                    continue
+
+            else:
+                mimetype    = ""
+                contenttype = header.get('content-type')
+                extension   = os.path.splitext(parse_name(url))[-1]
+
+                if contenttype:
+                    mimetype = contenttype.split(';')[0].strip()
+
+                elif extension:
+                    mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream"
+
+                if mimetype and (link or 'html' not in mimetype):
+                    link = url
+                else:
+                    link = False
+
+            return link
+
+
     def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True, resume=None, chunks=None):
         """
         Downloads the content at url to download folder
@@ -115,13 +171,19 @@ class Hoster(Base):
         the filename will be changed if needed
         :return: The location where the file was saved
         """
+        self.check_status()
+
         if self.pyload.debug:
             self.log_debug("DOWNLOAD URL " + url,
                            *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")])
 
-        url = self.fixurl(url)
+        dl_url      = self.is_download(url, resume)
+        dl_basename = parse_name(self.pyfile.name)
 
-        self.pyfile.name = parse_name(self.pyfile.name)  #: Safe check
+        self.pyfile.name = dl_basename
+
+        if not dl_url:
+            self.error("Invalid download url")
 
         self.captcha.correct()
 
@@ -130,63 +192,75 @@ class Hoster(Base):
 
         self.pyfile.setStatus("downloading")
 
-        download_folder   = self.pyload.config.get("general", "download_folder")
-        download_location = fs_join(download_folder, self.pyfile.package().folder)
+        dl_folder   = self.pyload.config.get("general", "download_folder")
+        dl_dirname  = os.path.join(dl_folder, self.pyfile.package().folder)
+        dl_filename = os.path.join(dl_dirname, dl_basename)
+
+        dl_dir  = encode(dl_dirname)
+        dl_file = encode(dl_filename)  #@TODO: Move safe-filename check to HTTPDownload in 0.4.10
 
-        if not exists(download_location):
+        if not exists(dl_dir):
             try:
-                os.makedirs(download_location)
+                os.makedirs(dl_dir)
 
             except Exception, e:
                 self.fail(e)
 
-        self.set_permissions(download_location)
+        self.set_permissions(dl_dir)
 
-        location = fs_decode(download_location)
-        filename = os.path.join(location, safe_filename(self.pyfile.name))  #@TODO: Move `safe_filename` check to HTTPDownload in 0.4.10
+        self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, dl_url, dl_filename)
+        self.check_status()
 
-        self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename)
+        dl_chunks   = self.pyload.config.get("download", "chunks")
+        chunk_limit = chunks or self.chunk_limit or -1
 
-        self.check_abort()
+        if dl_chunks is -1 or chunk_limit is -1:
+            chunks = max(dl_chunks, chunk_limit)
+        else:
+            chunks = min(dl_chunks, chunk_limit)
 
-        chunks = min(self.pyload.config.get("download", "chunks"), chunks or self.chunk_limit or -1)
-
-        if resume is None:
-            resume = self.resume_download
+        resume = self.resume_download if resume is None else bool(resume)
 
         try:
-            newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref,
+            newname = self.req.httpDownload(dl_url, dl_file, get=get, post=post, ref=ref,
                                             cookies=cookies, chunks=chunks, resume=resume,
                                             progressNotify=self.pyfile.setProgress,
                                             disposition=disposition)
+        except BadHeader, e:
+            if e.code in (404, 410):
+                self.pyfile.setStatus("offline")
+            raise BadHeader(e)
+
         finally:
             self.pyfile.size = self.req.size
 
         #@TODO: Recheck in 0.4.10
         if disposition and newname:
-            finalname = parse_name(newname).split(' filename*=')[0]
+            safename = parse_name(newname.split(' filename*=')[0])
 
-            if finalname != newname:
+            if safename != newname:
                 try:
-                    oldname_enc = fs_join(download_location, newname)
-                    newname_enc = fs_join(download_location, finalname)
-                    os.rename(oldname_enc, newname_enc)
+                    old_file = fs_join(dl_dirname, newname)
+                    new_file = fs_join(dl_dirname, safename)
+                    os.rename(old_file, new_file)
 
                 except OSError, e:
                     self.log_warning(_("Error renaming `%s` to `%s`")
-                                     % (newname, finalname), e)
-                    finalname = newname
+                                     % (newname, safename), e)
+                    safename = newname
 
-                self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname))
+                self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, safename))
 
-            self.pyfile.name = finalname
-            filename = os.path.join(location, finalname)
+            self.pyfile.name = safename
 
-        self.set_permissions(fs_encode(filename))
+            dl_filename = os.path.join(dl_dirname, safename)
+            dl_file = encode(dl_filename)
 
-        self.last_download = filename
+        self.set_permissions(dl_file)
 
-        return filename
+        self.last_download = dl_filename
+
+        return dl_filename
 
 
     def check_filesize(self, file_size, size_tolerance=1024):
@@ -199,18 +273,18 @@ class Hoster(Base):
         if not self.last_download:
             return
 
-        download_location = fs_encode(self.last_download)
-        download_size     = os.stat(download_location).st_size
+        dl_location = encode(self.last_download)
+        dl_size     = os.stat(dl_location).st_size
 
-        if download_size < 1:
+        if dl_size < 1:
             self.fail(_("Empty file"))
 
         elif file_size > 0:
-            diff = abs(file_size - download_size)
+            diff = abs(file_size - dl_size)
 
             if diff > size_tolerance:
                 self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s")
-                          % (file_size, download_size))
+                          % (file_size, dl_size))
 
             elif diff != 0:
                 self.log_warning(_("File size is not equal to expected size"))
@@ -228,7 +302,7 @@ class Hoster(Base):
         :return: dictionary key of the first rule that matched
         """
         do_delete = False
-        last_download = fs_encode(self.last_download)  #@TODO: Recheck in 0.4.10
+        last_download = encode(self.last_download)  #@TODO: Recheck in 0.4.10
 
         if not self.last_download or not exists(last_download):
             self.fail(self.pyfile.error or _("No file downloaded"))
@@ -267,6 +341,8 @@ class Hoster(Base):
 
 
     def _check_download(self):
+        self.log_info(_("Checking downloaded file..."))
+
         if self.captcha.task and not self.last_download:
             self.retry_captcha()
 
@@ -279,6 +355,9 @@ class Hoster(Base):
             # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB
             self.check_filesize(self.info['size'], size_tolerance=10485760)
 
+        else:
+            self.log_info(_("File is OK"))
+
 
     def check_traffic(self):
         if not self.account:
@@ -319,18 +398,18 @@ class Hoster(Base):
             if pyfile.status in (0, 5, 7, 12):  #: (finished, waiting, starting, downloading)
                 self.skip(pyfile.pluginname)
 
-        download_folder   = self.pyload.config.get("general", "download_folder")
-        package_folder    = pack.folder if self.pyload.config.get("general", "folder_per_package") else ""
-        download_location = fs_join(download_folder, package_folder, self.pyfile.name)
+        dl_folder      = self.pyload.config.get("general", "download_folder")
+        package_folder = pack.folder if self.pyload.config.get("general", "folder_per_package") else ""
+        dl_location    = fs_join(dl_folder, package_folder, self.pyfile.name)
 
-        if not exists(download_location):
+        if not exists(dl_location):
             return
 
         pyfile = self.pyload.db.findDuplicates(self.pyfile.id, package_folder, self.pyfile.name)
         if pyfile:
             self.skip(pyfile[0])
 
-        size = os.stat(download_location).st_size
+        size = os.stat(dl_location).st_size
         if size >= self.pyfile.size:
             self.skip(_("File exists"))