[SimpleHoster] Auto-detect offline & temp. offline status and hashsum info by default + Update check_download stuff + Spare code cosmetics and optimizations

1 files changed, 43 insertions, 26 deletions

diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py
index 4d7697d57..86d72ba48 100644
--- a/module/plugins/internal/SimpleHoster.py
+++ b/module/plugins/internal/SimpleHoster.py
@@ -7,9 +7,9 @@ import time
 
 from module.network.HTTPRequest import BadHeader
 from module.network.RequestFactory import getURL as get_url
-from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo
+from module.plugins.internal.Hoster import Hoster
 from module.plugins.internal.Plugin import Fail
-from module.plugins.internal.utils import (encode, parse_name, parse_size,
+from module.plugins.internal.misc import (encode, parse_name, parse_size,
                                            parse_time, replace_patterns)
 
 
@@ -43,7 +43,7 @@ class SimpleHoster(Hoster):
           example: SIZE_PATTERN = r'(?P<S>file_size) (?P<U>size_unit)'
 
       HASHSUM_PATTERN: (optional) Hash code and type of the file
-        example: HASHSUM_PATTERN = r'(?P<H>hash_code) (?P<T>MD5)'
+        example: HASHSUM_PATTERN = r'(?P<D>hash_digest) (?P<H>MD5)'
 
       OFFLINE_PATTERN: (mandatory) Check if the page is unreachable
         example: OFFLINE_PATTERN = r'File (deleted|not found)'
@@ -109,9 +109,9 @@ class SimpleHoster(Hoster):
     INFO_PATTERN         = None
     NAME_PATTERN         = None
     SIZE_PATTERN         = None
-    HASHSUM_PATTERN      = None
-    OFFLINE_PATTERN      = None
-    TEMP_OFFLINE_PATTERN = None
+    HASHSUM_PATTERN      = r'[^\w](?P<H>(CRC|crc)(-?32)?|(MD|md)-?5|(SHA|sha)-?(1|224|256|384|512)).*(:|=|>)[ ]*(?P<D>(?:[a-z0-9]|[A-Z0-9]){8,})'
+    OFFLINE_PATTERN      = r'[^\w](404\s|[Ii]nvalid|[Oo]ffline|[Dd]elet|[Rr]emov|([Nn]o(t|thing)?|sn\'t) (found|(longer )?(available|exist)))'
+    TEMP_OFFLINE_PATTERN = r'[^\w](503\s|[Mm]aint(e|ai)nance|[Tt]emp([.-]|orarily)|[Mm]irror)'
 
     WAIT_PATTERN         = None
     PREMIUM_ONLY_PATTERN = None
@@ -187,8 +187,8 @@ class SimpleHoster(Hoster):
             info['size'] = parse_size(info['size'], unit)
 
         if 'H' in info['pattern']:
-            hashtype = info['pattern']['T'] if 'T' in info['pattern'] else "hash"
-            info[hashtype] = info['pattern']['H']
+            type = info['pattern']['H'].strip('-').upper()
+            info['hash'][type] = info['pattern']['D']
 
         return info
 
@@ -254,20 +254,22 @@ class SimpleHoster(Hoster):
                 self.log_info(_("Failed to leech url"))
 
         else:
-            if not self.link and self.direct_dl and not self.last_download:
+            if not self.link and self.direct_dl:
                 self.log_info(_("Looking for direct download link..."))
                 self.handle_direct(pyfile)
 
-                if self.link or self.last_download:
+                if self.link:
                     self.log_info(_("Direct download link detected"))
                 else:
                     self.log_info(_("Direct download link not found"))
 
-            if not self.link and not self.last_download:
+            if not self.link:
                 self.preload()
 
                 if self.info.get('status', 3) is not 2:
                     self.grab_info()
+                    self.check_status()
+                    self.check_duplicates()
 
                 if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()):
                     self.log_info(_("Processing as premium download..."))
@@ -285,12 +287,17 @@ class SimpleHoster(Hoster):
             self.download(self.link, disposition=self.DISPOSITION)
 
 
+    def _check_download(self):
+        super(SimpleHoster, self)._check_download()
+        self.check_download()
+
+
     def check_download(self):
         super(SimpleHoster, self).check_download()
 
         self.log_info(_("Checking downloaded file with built-in rules..."))
         for r, p in self.FILE_ERRORS:
-            errmsg = self.check_file({r: re.compile(p)})
+            errmsg = self.scan_download({r: re.compile(p)})
             if errmsg is not None:
                 errmsg = errmsg.strip().capitalize()
 
@@ -312,12 +319,14 @@ class SimpleHoster(Hoster):
 
                 self.check_errors()
 
-        self.log_info(_("File is OK"))
+        self.log_info(_("No errors found"))
 
 
     def check_errors(self):
+        self.log_info(_("Checking for link errors..."))
+
         if not self.data:
-            self.log_debug("No data to check")
+            self.log_warning(_("No data to check"))
             return
 
         if self.IP_BLOCKED_PATTERN and re.search(self.IP_BLOCKED_PATTERN, self.data):
@@ -345,7 +354,7 @@ class SimpleHoster(Hoster):
                 self.log_warning(errmsg)
 
                 wait_time = parse_time(errmsg)
-                self.wait(wait_time, reconnect=wait_time > self.get_config("max_wait", 10) * 60)
+                self.wait(wait_time, reconnect=wait_time > self.config.get("max_wait", 10) * 60)
                 self.restart(_("Download limit exceeded"))
 
         if self.HAPPY_HOUR_PATTERN and re.search(self.HAPPY_HOUR_PATTERN, self.data):
@@ -366,33 +375,40 @@ class SimpleHoster(Hoster):
                 self.info['error'] = errmsg
                 self.log_warning(errmsg)
 
-                if re.search('limit|wait|slot', errmsg, re.I):
+                if re.search(self.TEMP_OFFLINE_PATTERN, errmsg):
+                    self.temp_offline()
+
+                elif re.search(self.OFFLINE_PATTERN, errmsg):
+                    self.offline()
+
+                elif re.search(r'limit|wait|slot', errmsg, re.I):
                     wait_time = parse_time(errmsg)
-                    self.wait(wait_time, reconnect=wait_time > self.get_config("max_wait", 10) * 60)
+                    self.wait(wait_time, reconnect=wait_time > self.config.get("max_wait", 10) * 60)
                     self.restart(_("Download limit exceeded"))
 
-                elif re.search('country|ip|region|nation', errmsg, re.I):
+                elif re.search(r'country|ip|region|nation', errmsg, re.I):
                     self.fail(_("Connection from your current IP address is not allowed"))
 
-                elif re.search('captcha|code', errmsg, re.I):
+                elif re.search(r'captcha|code', errmsg, re.I):
                     self.retry_captcha()
 
-                elif re.search('countdown|expired', errmsg, re.I):
+                elif re.search(r'countdown|expired', errmsg, re.I):
                     self.retry(10, 60, _("Link expired"))
 
-                elif re.search('maint(e|ai)nance|temp', errmsg, re.I):
+                elif re.search(r'503|maint(e|ai)nance|temp|mirror', errmsg, re.I):
                     self.temp_offline()
 
-                elif re.search('up to|size', errmsg, re.I):
+                elif re.search(r'up to|size', errmsg, re.I):
                     self.fail(_("File too large for free download"))
 
-                elif re.search('offline|delet|remov|not? (found|(longer)? available)', errmsg, re.I):
+                elif re.search(r'404|sorry|offline|delet|remov|(no(t|thing)?|sn\'t) (found|(longer )?(available|exist))',
+                               errmsg, re.I):
                     self.offline()
 
-                elif re.search('filename', errmsg, re.I):
+                elif re.search(r'filename', errmsg, re.I):
                     self.fail(_("Invalid url"))
 
-                elif re.search('premium', errmsg, re.I):
+                elif re.search(r'premium', errmsg, re.I):
                     self.fail(_("File can be downloaded by premium users only"))
 
                 else:
@@ -409,8 +425,9 @@ class SimpleHoster(Hoster):
                     waitmsg = m.group(0).strip()
 
                 wait_time = parse_time(waitmsg)
-                self.wait(wait_time, reconnect=wait_time > self.get_config("max_wait", 10) * 60)
+                self.wait(wait_time, reconnect=wait_time > self.config.get("max_wait", 10) * 60)
 
+        self.log_info(_("No errors found"))
         self.info.pop('error', None)