[SimpleHoster] Improve logging and file checking

8 files changed, 100 insertions, 67 deletions

diff --git a/module/plugins/accounts/ShareonlineBiz.py b/module/plugins/accounts/ShareonlineBiz.py
index b9509b466..9269cf2a1 100644
--- a/module/plugins/accounts/ShareonlineBiz.py
+++ b/module/plugins/accounts/ShareonlineBiz.py
@@ -64,5 +64,5 @@ class ShareonlineBiz(Account):
         html = self.api_response(user, req)
         err  = re.search(r'\*\*(.+?)\*\*', html)
         if err:
-            self.logError(err.group(1))
+            self.logError(err.group(1).strip())
             self.wrongPassword()
diff --git a/module/plugins/crypter/Dereferer.py b/module/plugins/crypter/Dereferer.py
index 85b84a242..cdc8d2d53 100644
--- a/module/plugins/crypter/Dereferer.py
+++ b/module/plugins/crypter/Dereferer.py
@@ -8,7 +8,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter
 class Dereferer(SimpleCrypter):
     __name__    = "Dereferer"
     __type__    = "crypter"
-    __version__ = "0.14"
+    __version__ = "0.15"
 
     __pattern__ = r'https?://(?:www\.)?(?:\w+\.)*?(?P<DOMAIN>(?:[\d.]+|[\w\-]{3,}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)/.*?(?P<LINK>(?:ht|f)tps?://.+)'
     __config__  = [("use_subfolder"     , "bool", "Save package to subfolder"          , True),
@@ -24,9 +24,7 @@ class Dereferer(SimpleCrypter):
 
 
     def _log(self, type, args):
-        msg = " | ".join(str(a).strip() for a in args if a)
-        logger = getattr(self.log, type)
-        logger("%s: %s: %s" % (self.__name__, self.HOSTER_NAME, msg or _("%s MARK" % type.upper())))
+        return super(Dereferer, self)._log(type, [self.HOSTER_NAME] + args)
 
 
     def init(self):
diff --git a/module/plugins/crypter/XFileSharingProFolder.py b/module/plugins/crypter/XFileSharingProFolder.py
index 2e1a2f022..8043a1183 100644
--- a/module/plugins/crypter/XFileSharingProFolder.py
+++ b/module/plugins/crypter/XFileSharingProFolder.py
@@ -8,7 +8,7 @@ from module.plugins.internal.XFSCrypter import XFSCrypter, create_getInfo
 class XFileSharingProFolder(XFSCrypter):
     __name__    = "XFileSharingProFolder"
     __type__    = "crypter"
-    __version__ = "0.06"
+    __version__ = "0.07"
 
     __pattern__ = r'https?://(?:www\.)?(?:\w+\.)*?(?P<DOMAIN>(?:[\d.]+|[\w\-^_]{3,}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)/(?:user|folder)s?/\w+'
     __config__  = [("use_subfolder"     , "bool", "Save package to subfolder"          , True),
@@ -20,9 +20,7 @@ class XFileSharingProFolder(XFSCrypter):
 
 
     def _log(self, type, args):
-        msg = " | ".join(str(a).strip() for a in args if a)
-        logger = getattr(self.log, type)
-        logger("%s: %s: %s" % (self.__name__, self.HOSTER_NAME, msg or _("%s MARK" % type.upper())))
+        return super(XFileSharingProFolder, self)._log(type, [self.HOSTER_NAME] + args)
 
 
     def init(self):
diff --git a/module/plugins/hoster/SolidfilesCom.py b/module/plugins/hoster/SolidfilesCom.py
index d359577d6..184329a40 100644
--- a/module/plugins/hoster/SolidfilesCom.py
+++ b/module/plugins/hoster/SolidfilesCom.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Test links:
-# http://www.solidfiles.com/d/609cdb4b1b
+#   http://www.solidfiles.com/d/609cdb4b1b
 
 from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo
 
diff --git a/module/plugins/hoster/XFileSharingPro.py b/module/plugins/hoster/XFileSharingPro.py
index 169fbc8d3..c62a42779 100644
--- a/module/plugins/hoster/XFileSharingPro.py
+++ b/module/plugins/hoster/XFileSharingPro.py
@@ -8,7 +8,7 @@ from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo
 class XFileSharingPro(XFSHoster):
     __name__    = "XFileSharingPro"
     __type__    = "hoster"
-    __version__ = "0.46"
+    __version__ = "0.47"
 
     __pattern__ = r'https?://(?:www\.)?(?:\w+\.)*?(?P<DOMAIN>(?:[\d.]+|[\w\-^_]{3,}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)/(?:embed-)?\w{12}(?:\W|$)'
 
@@ -21,9 +21,7 @@ class XFileSharingPro(XFSHoster):
 
 
     def _log(self, type, args):
-        msg = " | ".join(str(a).strip() for a in args if a)
-        logger = getattr(self.log, type)
-        logger("%s: %s: %s" % (self.__name__, self.HOSTER_NAME, msg or _("%s MARK" % type.upper())))
+        return super(XFileSharingPro, self)._log(type, [self.HOSTER_NAME] + args)
 
 
     def init(self):
diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py
index 2b4ceb9b0..2b3eb8941 100644
--- a/module/plugins/internal/MultiHoster.py
+++ b/module/plugins/internal/MultiHoster.py
@@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r
 class MultiHoster(SimpleHoster):
     __name__    = "MultiHoster"
     __type__    = "hoster"
-    __version__ = "0.41"
+    __version__ = "0.42"
 
     __pattern__ = r'^unmatchable$'
     __config__  = [("use_premium" , "bool", "Use premium account if available"    , True),
@@ -32,8 +32,8 @@ class MultiHoster(SimpleHoster):
     def prepare(self):
         self.info     = {}
         self.html     = ""
-        self.link     = ""     #@TODO: Move to hoster class in 0.4.10
-        self.directDL = False  #@TODO: Move to hoster class in 0.4.10
+        self.link     = ""     #@TODO: Move to Hoster in 0.4.10
+        self.directDL = False  #@TODO: Move to Hoster in 0.4.10
 
         if not self.getConfig('use_premium', True):
             self.retryFree()
@@ -77,7 +77,7 @@ class MultiHoster(SimpleHoster):
                     self.logDebug("Handled as free download")
                     self.handleFree(pyfile)
 
-            self.downloadLink(self.link, True)
+            self.download(self.link, ref=False, disposition=True)
             self.checkFile()
 
         except Fail, e:  #@TODO: Move to PluginThread in 0.4.10
diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py
index 6279be1bd..00b872b69 100644
--- a/module/plugins/internal/SimpleCrypter.py
+++ b/module/plugins/internal/SimpleCrypter.py
@@ -11,7 +11,7 @@ from module.utils import fixup, html_unescape
 class SimpleCrypter(Crypter, SimpleHoster):
     __name__    = "SimpleCrypter"
     __type__    = "crypter"
-    __version__ = "0.50"
+    __version__ = "0.51"
 
     __pattern__ = r'^unmatchable$'
     __config__  = [("use_subfolder"     , "bool", "Save package to subfolder"          , True),  #: Overrides core.config['general']['folder_per_package']
@@ -74,8 +74,8 @@ class SimpleCrypter(Crypter, SimpleHoster):
 
         self.info  = {}
         self.html  = ""
-        self.link  = ""  #@TODO: Move to hoster class in 0.4.10
-        self.links = []  #@TODO: Move to hoster class in 0.4.10
+        self.link  = ""  #@TODO: Move to Hoster in 0.4.10
+        self.links = []  #@TODO: Move to Hoster in 0.4.10
 
         if self.LOGIN_PREMIUM and not self.premium:
             self.fail(_("Required premium account not found"))
@@ -121,6 +121,7 @@ class SimpleCrypter(Crypter, SimpleHoster):
             self.logDebug("Package has %d links" % len(self.links))
 
         if self.links:
+            self.links    = [html_unescape(l.decode('unicode-escape').strip()) for l in links]  #@TODO: Move to Crypter in 0.4.10
             self.packages = [(self.info['name'], self.links, self.info['folder'])]
 
         elif not self.urls and not self.packages:  #@TODO: Remove in 0.4.10
@@ -160,10 +161,8 @@ class SimpleCrypter(Crypter, SimpleHoster):
         url_p   = urlparse.urlparse(self.pyfile.url)
         baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
 
-        links = [urlparse.urljoin(baseurl, link) if not urlparse.urlparse(link).scheme else link \
-                 for link in re.findall(self.LINK_PATTERN, self.html)]
-
-        return [html_unescape(l.decode('unicode-escape').strip()) for l in links]
+        return [urlparse.urljoin(baseurl, link) if not urlparse.urlparse(link).scheme else link \
+                for link in re.findall(self.LINK_PATTERN, self.html)]
 
 
     def handlePages(self, pyfile):
diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py
index da477ad79..a1a1a4a67 100644
--- a/module/plugins/internal/SimpleHoster.py
+++ b/module/plugins/internal/SimpleHoster.py
@@ -133,7 +133,7 @@ def timestamp():
     return int(time.time() * 1000)
 
 
-#@TODO: Move to hoster class in 0.4.10
+#@TODO: Move to Hoster in 0.4.10
 def getFileURL(self, url, follow_location=None):
     link     = ""
     redirect = 1
@@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0):
 class SimpleHoster(Hoster):
     __name__    = "SimpleHoster"
     __type__    = "hoster"
-    __version__ = "1.56"
+    __version__ = "1.57"
 
     __pattern__ = r'^unmatchable$'
     __config__  = [("use_premium", "bool", "Use premium account if available"          , True),
@@ -308,6 +308,11 @@ class SimpleHoster(Hoster):
     SIZE_REPLACEMENTS = []
     URL_REPLACEMENTS  = []
 
+    FILE_ERRORS = [('Html error'   , r'\A(?:\s*<.+>)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)'),
+                   ('Request error', r'([Aa]n error occured while processing your request)'                   ),
+                   ('Html file'    , r'\A\s*<!DOCTYPE html'                                                   )]
+
+    CHECK_FILE    = True   #: Set to False to not check the last downloaded file with declared error patterns
     CHECK_TRAFFIC = False  #: Set to True to force checking traffic left for premium account
     COOKIES       = True   #: or False or list of tuples [(domain, name, value)]
     DIRECT_LINK   = None   #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False
@@ -416,18 +421,55 @@ class SimpleHoster(Hoster):
         return info
 
 
+    #@TODO: Move to Hoster in 0.4.10
+    def _log(self, type, args):
+        msg = " | ".join([(a.encode("utf-8", "replace") if type(a) is unicode else str(a)).strip() for a in args if a])
+        logger = getattr(self.core.log, type)
+        logger("%(plugin)s[%(id)s]: %(msg)s" % {'plugin' : self.__name,
+                                                'id'     : self.pyfile.id
+                                                'msg'    : msg or _("%s MARK" % type.upper())})
+
+
+    #@TODO: Move to Hoster in 0.4.10
+    def logDebug(self, *args):
+        if self.core.debug:
+            return self._log("debug", args)
+
+
+    #@TODO: Move to Hoster in 0.4.10
+    def logInfo(self, *args):
+        return self._log("info", args)
+
+
+    #@TODO: Move to Hoster in 0.4.10
+    def logWarning(self, *args):
+        return self._log("warning", args)
+
+
+    #@TODO: Move to Hoster in 0.4.10
+    def logError(self, *args):
+        return self._log("error", args)
+
+
+    #@TODO: Move to Hoster in 0.4.10
+    def logCritical(self, *args):
+        return self._log("critical", args)
+
+
+    #@TODO: Move to Hoster in 0.4.10
     def setup(self):
         self.resumeDownload = self.multiDL = self.premium
 
 
+    #@TODO: Move to Hoster in 0.4.10
     def prepare(self):
         self.pyfile.error = ""  #@TODO: Remove in 0.4.10
 
         self.info      = {}
         self.html      = ""
-        self.link      = ""     #@TODO: Move to hoster class in 0.4.10
-        self.directDL  = False  #@TODO: Move to hoster class in 0.4.10
-        self.multihost = False  #@TODO: Move to hoster class in 0.4.10
+        self.link      = ""     #@TODO: Move to Hoster in 0.4.10
+        self.directDL  = False  #@TODO: Move to Hoster in 0.4.10
+        self.multihost = False  #@TODO: Move to Hoster in 0.4.10
 
         if not self.getConfig('use_premium', True):
             self.retryFree()
@@ -493,7 +535,7 @@ class SimpleHoster(Hoster):
                     self.logDebug("Handled as free download")
                     self.handleFree(pyfile)
 
-            self.downloadLink(self.link, self.DISPOSITION)
+            self.download(self.link, ref=False, disposition=self.DISPOSITION)
             self.checkFile()
 
         except Fail, e:  #@TODO: Move to PluginThread in 0.4.10
@@ -510,23 +552,23 @@ class SimpleHoster(Hoster):
                 raise Fail(err)
 
 
-    def downloadLink(self, link, disposition=True):
-        if not link or not isinstance(link, basestring):
+    def download(self, url, *args, **kwargs):
+        if not url or not isinstance(url, basestring):
             return
 
         self.correctCaptcha()
 
-        link = html_unescape(link.decode('unicode-escape').strip())  #@TODO: Move this check to plugin `load` method in 0.4.10
+        url = html_unescape(url.decode('unicode-escape').strip())  #@TODO: Move to Hoster in 0.4.10
 
-        if not urlparse.urlparse(link).scheme:
+        if not urlparse.urlparse(url).scheme:
             url_p   = urlparse.urlparse(self.pyfile.url)
             baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
-            link    = urlparse.urljoin(baseurl, link)
+            url     = urlparse.urljoin(baseurl, url)
 
-        self.download(link, ref=False, disposition=disposition)
+        return super(SimpleHoster, self).download(url, *args, **kwargs)
 
 
-    def checkFile(self, rules={}):
+    def checkFile(self):
         if self.cTask and not self.lastDownload:
             self.invalidCaptcha()
             self.retry(10, reason=_("Wrong captcha"))
@@ -536,38 +578,36 @@ class SimpleHoster(Hoster):
             self.error(self.pyfile.error or _("No file downloaded"))
 
         else:
-            errmsg = self.checkDownload({'Empty file': re.compile(r'\A\s*\Z'),
-                                         'Html error': re.compile(r'\A(?:\s*<.+>)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)')})
-
-            if not errmsg:
-                for r, p in [('Html file'    , re.compile(r'\A\s*<!DOCTYPE html')                                ),
-                             ('Request error', re.compile(r'([Aa]n error occured while processing your request)'))]:
-                    if r not in rules:
-                        rules[r] = p
-
-                for r, a in [("IP blocked"    , "IP_BLOCKED_PATTERN"  ),
-                             ("Download limit", "DL_LIMIT_PATTERN"    ),
-                             ("Size limit"    , "SIZE_LIMIT_PATTERN"  ),
-                             ("Error"         , "ERROR_PATTERN"       ),
-                             ("Premium only"  , "PREMIUM_ONLY_PATTERN"),
-                             ("Wait error"    , "WAIT_PATTERN"        )]:
-                    if r not in rules and hasattr(self, a):
-                        rules[r] = getattr(self, a)
-
-                errmsg = self.checkDownload(rules)
-
-            if not errmsg:
+            self.logDebug(_("Checking last downloaded file with built-in rules"))
+            errmsg = self.checkDownload({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')})
+
+            if errmsg:
                 return
 
-            errmsg = errmsg.strip().capitalize()
+            self.logDebug(_("Checking last downloaded file with custom rules"))
+
+            if self.CHECK_FILE:
+                rules = [r, getattr(self, a) for r, a in (("IP blocked"    , "IP_BLOCKED_PATTERN"  ),
+                                                          ("Download limit", "DL_LIMIT_PATTERN"    ),
+                                                          ("Size limit"    , "SIZE_LIMIT_PATTERN"  ),
+                                                          ("Error"         , "ERROR_PATTERN"       ),
+                                                          ("Premium only"  , "PREMIUM_ONLY_PATTERN"),
+                                                          ("Wait error"    , "WAIT_PATTERN"        ))]
+                self.FILE_ERRORS.extend(rules)
+
+            for r, p in self.FILE_ERRORS:
+                errmsg = self.checkDownload({r: re.compile(p)}).strip().capitalize()
 
-            try:
-                errmsg += " | " + self.lastCheck.group(1).strip()
-            except Exception:
-                pass
+                if not errmsg:
+                    continue
+
+                try:
+                    errmsg += " | " + self.lastCheck.group(1).strip()
+                except Exception:
+                    pass
 
-            self.logWarning("Check result: " + errmsg, "Waiting 1 minute and retry")
-            self.retry(3, 60, errmsg)
+                self.logWarning("Check result: " + errmsg, "Waiting 1 minute and retry")
+                self.retry(3, 60, errmsg)
 
 
     def checkErrors(self):
@@ -636,7 +676,7 @@ class SimpleHoster(Hoster):
                     self.retry(10, reason=_("Wrong captcha"))
 
                 elif re.search('countdown|expired', errmsg, re.I):
-                    self.retry(10, wait_time=60, reason=_("Link expired"))
+                    self.retry(10, 60, _("Link expired"))
 
                 elif re.search('maintenance|maintainance|temp', errmsg, re.I):
                     self.tempOffline()