simple checksum and download scheduler hooks; keep config options ordered

6 files changed, 226 insertions, 41 deletions

diff --git a/module/plugins/accounts/ShareonlineBiz.py b/module/plugins/accounts/ShareonlineBiz.py
index 1aac2282c..cdc4ebb63 100644
--- a/module/plugins/accounts/ShareonlineBiz.py
+++ b/module/plugins/accounts/ShareonlineBiz.py
@@ -29,8 +29,6 @@ class ShareonlineBiz(Account):
     __author_name__ = ("mkaay", "zoidberg")
     __author_mail__ = ("mkaay@mkaay.de", "zoidberg@mujmail.cz")
     
-    info_threshold = 60
-    
     def getUserAPI(self, user, req):
         return req.load("http://api.share-online.biz/account.php?username=%s&password=%s&act=userDetails" % (user, self.accounts[user]["password"]))                
 
diff --git a/module/plugins/hooks/Checksum.py b/module/plugins/hooks/Checksum.py
new file mode 100644
index 000000000..199316cd8
--- /dev/null
+++ b/module/plugins/hooks/Checksum.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+    @author: zoidberg
+"""
+from __future__ import with_statement
+import hashlib, zlib
+
+from module.utils import save_join, fs_encode
+from module.plugins.Hook import Hook
+
+def computeChecksum(local_file, algorithm): 
+    if algorithm in getattr(hashlib, "algorithms", ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")):
+        h = getattr(hashlib, algorithm)()
+        chunk_size = 128 * h.block_size
+        
+        with open(local_file, 'rb') as f: 
+            for chunk in iter(lambda: f.read(chunk_size), b''): 
+                 h.update(chunk)
+        
+        return h.hexdigest()
+         
+    elif algorithm in ("adler32", "crc32"):
+        hf = getattr(zlib, algorithm)
+        last = 0
+        
+        with open(local_file, 'rb') as f: 
+            for chunk in iter(lambda: f.read(8192), b''): 
+                last = hf(chunk, last)
+        
+        return "%x" % last
+    
+    else:
+        return None      
+
+class Checksum(Hook):
+    __name__ = "Checksum"
+    __version__ = "0.02"
+    __description__ = "Check downloaded file hash"
+    __config__ = [("activated", "bool", "Activated", True),
+                  ("action", "fail;retry;nothing", "What to do if check fails?", "retry"),
+                  ("max_tries", "int", "Number of retries", 2)]
+    __author_name__ = ("zoidberg")
+    __author_mail__ = ("zoidberg@mujmail.cz")
+             
+    def downloadFinished(self, pyfile):
+        """ 
+        Compute checksum for the downloaded file and compare it with the hash provided by the hoster.
+        pyfile.plugin.file_check should be a dictionary which can contain:
+        a) if known, the exact filesize in bytes (e.g. "size": 123456789)
+        b) hexadecimal hash string with algorithm name as key (e.g. "md5": "d76505d0869f9f928a17d42d66326307")    
+        """
+        if hasattr(pyfile.plugin, "file_check") and (isinstance(pyfile.plugin.file_check, dict)):
+            
+            download_folder = self.config['general']['download_folder']
+            local_file = fs_encode(save_join(download_folder, pyfile.package().folder, pyfile.name))
+            
+            for key, value in sorted(pyfile.plugin.file_check.items(), reverse = True):                          
+                if key == "size":
+                    if value and value != pyfile.size:
+                        self.logWarning("File %s has incorrect size: %d B (%d expected)" % (pyfile.size, value))
+                        self.checkFailed(pyfile, "Incorrect file size")
+                else:
+                    checksum = computeChecksum(local_file, key.replace("-","").lower())                    
+                    if checksum:
+                        if checksum == value:
+                            self.logInfo('File integrity of "%s" verified by %s checksum (%s).' % (pyfile.name, key.upper() , checksum))
+                            return
+                        else:
+                            self.logWarning("%s checksum for file %s does not match (%s != %s)" % (key.upper(), pyfile.name, checksum, value))    
+                            self.checkFailed(pyfile, "Checksums do not match")
+                    else:
+                        self.logWarning("Unsupported hashing algorithm: %s" % key.upper())  
+            else:
+                self.logWarning("Unable to validate checksum for file %s" % (pyfile.name))
+    
+    def checkFailed(self, pyfile, msg):
+        action = self.getConfig("action")
+        if action == "fail":
+            pyfile.plugin.fail(reason = msg)
+        elif action == "retry":
+            pyfile.plugin.retry(reason = msg, max_tries = self.getConfig("max_tries"))
\ No newline at end of file
diff --git a/module/plugins/hooks/DownloadScheduler.py b/module/plugins/hooks/DownloadScheduler.py
new file mode 100644
index 000000000..2d7d2ffb0
--- /dev/null
+++ b/module/plugins/hooks/DownloadScheduler.py
@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+    
+    @author: zoidberg
+    Original idea by new.cze
+"""
+
+import re
+from time import localtime
+from module.plugins.Hook import Hook
+
+class DownloadScheduler(Hook):
+    __name__ = "DownloadScheduler"
+    __version__ = "0.20"
+    __description__ = """Download Scheduler"""
+    __config__ = [("activated", "bool", "Activated", "False"),                  
+                  ("timetable", "str", "List time periods as hh:mm full or number(kB/s)", "0:00 full, 7:00 250, 10:00 0, 17:00 150")]
+    __author_name__ = ("zoidberg")
+    __author_mail__ = ("zoidberg@mujmail.cz")
+    
+    def setup(self):
+        self.cb = None # callback to scheduler job; will be by removed hookmanager when hook unloaded       
+    
+    def coreReady(self):
+        self.updateSchedule()
+        
+    def updateSchedule(self, schedule = None):        
+        if schedule is None: 
+            schedule = self.getConfig("timetable")                   
+        
+        schedule = re.findall("(\d{1,2}):(\d{2})[\s]*(-?\d+)", schedule.lower().replace("full", "-1").replace("none", "0"))         
+        if not schedule:
+            self.logError("Invalid schedule")
+            return
+        
+        t0 = localtime()
+        now = (t0.tm_hour, t0.tm_min, t0.tm_sec, "X")
+        schedule = sorted([(int(x[0]), int(x[1]), 0, int(x[2])) for x in schedule] + [now])
+    
+        self.logDebug("Schedule", schedule)      
+      
+        for i, v in enumerate(schedule):
+            if v[3] == "X":
+                last, next = schedule[i-1], schedule[(i+1) % len(schedule)]
+                self.logDebug("Now/Last/Next", now, last, next)
+                 
+                self.setDownloadSpeed(last[3]) 
+                
+                next_time = (((24 + next[0] - now[0])* 60 + next[1] - now[1]) * 60 + next[2] - now[2]) % 86400
+                self.core.scheduler.removeJob(self.cb)
+                self.cb = self.core.scheduler.addJob(next_time, self.updateSchedule, threaded=False)                                     
+                
+    def setDownloadSpeed(self, speed):     
+        if speed == 0:
+            self.logInfo("Stopping download server. Current downloads will not be interrupted.")
+            self.core.api.pauseServer()
+        else:
+            self.core.api.unpauseServer()
+            
+            if speed > 0:
+                self.logInfo("Setting download speed to %d kB/s" % speed)
+                self.core.api.setConfigValue("download","limit_speed",1)
+                self.core.api.setConfigValue("download","max_speed",speed)
+            else:
+                self.logInfo("Setting download speed to FULL")
+                self.core.api.setConfigValue("download","limit_speed",0)
+                self.core.api.setConfigValue("download","max_speed",-1)
\ No newline at end of file
diff --git a/module/plugins/hooks/XFileSharingPro.py b/module/plugins/hooks/XFileSharingPro.py
index f4461f8c6..3981db2d0 100644
--- a/module/plugins/hooks/XFileSharingPro.py
+++ b/module/plugins/hooks/XFileSharingPro.py
@@ -5,41 +5,44 @@ import re
 
 class XFileSharingPro(Hook):
     __name__ = "XFileSharingPro"
-    __version__ = "0.02"
+    __version__ = "0.03"
     __type__ = "hook"
-    __config__ = [ ("activated", "bool", "Activated" , "True"),
-                   ("loadDefault", "bool", "Load default hoster list" , "True"),
+    __config__ = [ ("activated" , "bool" , "Activated"  , "True"),
+                   ("loadDefault", "bool", "Include default (built-in) hoster list" , "True"),
                    ("includeList", "str", "Include hosters (comma separated)", ""),
-                   ("excludeList", "str", "Exclude hosters (comma separated)", "")]
-    __description__ = """MultiShare.cz hook plugin"""
+                   ("excludeList", "str", "Exclude hosters (comma separated)", "") ]
+    __description__ = """Hoster URL pattern loader for the generic XFileSharingPro plugin"""
     __author_name__ = ("zoidberg")
-    __author_mail__ = ("zoidberg@mujmail.cz")    
-    
+    __author_mail__ = ("zoidberg@mujmail.cz")
+          
     def coreReady(self):
+        self.loadPattern()
+      
+    def loadPattern(self):         
         hosterList = self.getConfigSet('includeList')
         excludeList = self.getConfigSet('excludeList')         
         
         if self.getConfig('loadDefault'):        
             hosterList |= set((
             #WORKING HOSTERS:
-            "azsharing\.com", "banashare\.com", "fileband\.com", "kingsupload\.com", "migahost\.com", "ryushare.com", "xfileshare\.eu",
+            "azsharing.com", "banashare.com", "fileband.com", "kingsupload.com", "migahost.com", "ryushare.com", "xfileshare.eu",
             #NOT TESTED:
-            "aieshare\.com", "amonshare\.com", "asixfiles\.com",   
-            "bebasupload\.com", "boosterking\.com", "buckshare\.com", "bulletupload\.com", "crocshare\.com", "ddlanime\.com", "divxme\.com", 
-            "dopeshare\.com", "downupload\.com", "eyesfile\.com", "eyvx\.com", "fik1\.com", "file4safe\.com", "file4sharing\.com", 
-            "fileforth\.com", "filemade\.com", "filemak\.com", "fileplaygroud\.com", "filerace\.com", "filestrack\.com", 
-            "fileupper\.com", "filevelocity\.com", "fooget\.com", "4bytez\.com", "freefilessharing\.com", "glumbouploads\.com", "grupload\.com", 
-            "heftyfile\.com", "hipfile\.com", "host4desi\.com", "hulkshare\.com", "idupin\.com", "imageporter\.com", "isharefast\.com", 
-            "jalurcepat\.com", "laoupload\.com", "linkzhost\.com", "loombo\.com", "maknyos\.com", 
-            "mlfat4arab\.com", "movreel\.com", "netuploaded\.com", "ok2upload\.com", "180upload\.com", "1hostclick\.com", "ovfile\.com", 
-            "putshare\.com", "pyramidfiles\.com", "q4share\.com", "queenshare\.com", "ravishare\.com", "rockdizfile\.com", "sendmyway\.com", 
-            "share76\.com", "sharebeast\.com", "sharehut\.com", "sharerun\.com", "shareswift\.com", "sharingonline\.com", "6ybh-upload\.com", 
-            "skipfile\.com", "spaadyshare\.com", "space4file\.com", "speedoshare\.com", "uploadbaz\.com", "uploadboost\.com", "uploadc\.com", 
-            "uploaddot\.com", "uploadfloor\.com", "uploadic\.com", "uploadville\.com", "uptobox\.com", "vidbull\.com", "zalaa\.com", 
-            "zomgupload\\.com\.com", "kupload\.org", "movbay\.org", "multishare\.org", "omegave\.org", "toucansharing\.org", "uflinq\.org",
-            "banicrazy\.info", "flowhot\.info", "upbrasil\.info", "shareyourfilez\.biz", "bzlink\.us", "cloudcache\.cc", "fileserver\.cc"
-            "farshare\.to", "kingshare\.to", "filemaze\.ws", "filehost\.ws", "goldfile\.eu", "filestock\.ru", "moidisk\.ru"
-            "4up\.me", "kfiles\.kz", "odsiebie\.pl", "upchi\.co\.il", "upit\.in", "verzend\.be"
+            "aieshare.com", "amonshare.com", "asixfiles.com",   
+            "bebasupload.com", "boosterking.com", "buckshare.com", "bulletupload.com", "crocshare.com", "ddlanime.com", "divxme.com", 
+            "dopeshare.com", "downupload.com", "eyesfile.com", "eyvx.com", "fik1.com", "file4safe.com", "file4sharing.com", 
+            "fileforth.com", "filemade.com", "filemak.com", "fileplaygroud.com", "filerace.com", "filestrack.com", 
+            "fileupper.com", "filevelocity.com", "fooget.com", "4bytez.com", "freefilessharing.com", "glumbouploads.com", "grupload.com", 
+            "heftyfile.com", "hipfile.com", "host4desi.com", "hulkshare.com", "idupin.com", "imageporter.com", "isharefast.com", 
+            "jalurcepat.com", "laoupload.com", "linkzhost.com", "loombo.com", "maknyos.com", 
+            "mlfat4arab.com", "movreel.com", "netuploaded.com", "ok2upload.com", "180upload.com", "1hostclick.com", "ovfile.com", 
+            "putshare.com", "pyramidfiles.com", "q4share.com", "queenshare.com", "ravishare.com", "rockdizfile.com", "sendmyway.com", 
+            "share76.com", "sharebeast.com", "sharehut.com", "sharerun.com", "shareswift.com", "sharingonline.com", "6ybh-upload.com", 
+            "skipfile.com", "spaadyshare.com", "space4file.com", "speedoshare.com", "uploadbaz.com", "uploadboost.com", "uploadc.com", 
+            "uploaddot.com", "uploadfloor.com", "uploadic.com", "uploadville.com", "uptobox.com", "vidbull.com", "zalaa.com", 
+            "zomgupload.com", "kupload.org", "movbay.org", "multishare.org", "omegave.org", "toucansharing.org", "uflinq.org",
+            "banicrazy.info", "flowhot.info", "upbrasil.info", "shareyourfilez.biz", "bzlink.us", "cloudcache.cc", "fileserver.cc"
+            "farshare.to", "kingshare.to", "filemaze.ws", "filehost.ws", "goldfile.eu", "filestock.ru", "moidisk.ru"
+            "4up.me", "kfiles.kz", "odsiebie.pl", "upchi.co.il", "upit.in", "verzend.be"
             ))
             
             #NOT WORKING:
@@ -50,17 +53,22 @@ class XFileSharingPro(Hook):
         hosterList -= set(('', u''))
         
         if not hosterList:
-            self.logError("Hoster list is empty" % len(hosterList))
+            self.unload()
             return
                                             
-        regexp = r"http://(?:[^/]*\.)?(%s)/\w{12}" % "|".join(sorted(hosterList))
-        self.logDebug("Added %d hosters" % len(hosterList))
+        regexp = r"http://(?:[^/]*\.)?(%s)/\w{12}" % ("|".join(sorted(hosterList)).replace('.','\.'))
         #self.logDebug(regexp)
         
         dict = self.core.pluginManager.hosterPlugins['XFileSharingPro']
         dict["pattern"] = regexp
         dict["re"] = re.compile(regexp)
+        self.logDebug("Pattern loaded - handling %d hosters" % len(hosterList))
         
     def getConfigSet(self, option):
-        s = self.getConfig(option).lower().replace('|',',').replace(';',',').replace('.','\.')
-        return set([x.strip() for x in s.split('|')])
\ No newline at end of file
+        s = self.getConfig(option).lower().replace('|',',').replace(';',',')
+        return set([x.strip() for x in s.split(',')])
+        
+    def unload(self):
+        dict = self.core.pluginManager.hosterPlugins['XFileSharingPro']
+        dict["pattern"] = r"^unmatchable$"
+        dict["re"] = re.compile(r"^unmatchable$")
\ No newline at end of file
diff --git a/module/plugins/hoster/ShareonlineBiz.py b/module/plugins/hoster/ShareonlineBiz.py
index 719235565..fbec82b7e 100644
--- a/module/plugins/hoster/ShareonlineBiz.py
+++ b/module/plugins/hoster/ShareonlineBiz.py
@@ -43,7 +43,7 @@ class ShareonlineBiz(Hoster):
     __name__ = "ShareonlineBiz"
     __type__ = "hoster"
     __pattern__ = r"http://[\w\.]*?(share\-online\.biz|egoshare\.com)/(download.php\?id\=|dl/)[\w]+"
-    __version__ = "0.29"
+    __version__ = "0.30"
     __description__ = """Shareonline.biz Download Hoster"""
     __author_name__ = ("spoob", "mkaay", "zoidberg")
     __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de", "zoidberg@mujmail.cz")
@@ -57,6 +57,8 @@ class ShareonlineBiz(Hoster):
 
         self.resumeDownload = self.multiDL = self.premium
         #self.chunkLimit = 1
+        
+        self.file_check = None
 
     def process(self, pyfile):       
         if self.premium:
@@ -74,9 +76,8 @@ class ShareonlineBiz(Hoster):
                 self.account.getAccountInfo(self.user, True)
             self.retry(reason=_("Invalid download ticket"))
             
-        self.logDebug('DOWNLOAD SIZE: %d B (%d expected)' % (self.pyfile.size , self.exp_size))
-        if self.pyfile.size != self.exp_size:
-            self.retry(reason="Incorrect file size: %d B" % self.pyfile.size)
+        if self.api_data:
+            self.file_check = {"size": int(self.api_data['size']), "md5": self.api_data['md5']}
 
     def downloadAPIData(self):
         api_url_base = "http://api.share-online.biz/linkcheck.php?md5=1"
@@ -95,7 +96,7 @@ class ShareonlineBiz(Hoster):
     def handleFree(self):       
         self.downloadAPIData()
         self.pyfile.name = self.api_data["filename"]
-        self.pyfile.size = self.exp_size = int(self.api_data["size"])
+        self.pyfile.size = int(self.api_data["size"])
         
         self.html = self.load(self.pyfile.url, cookies = True) #refer, stuff
         self.setWait(3)
@@ -145,7 +146,7 @@ class ShareonlineBiz(Hoster):
             self.offline()
         
         self.pyfile.name = dlinfo["name"]
-        self.pyfile.size = self.exp_size = int(dlinfo["size"])
+        self.pyfile.size = int(dlinfo["size"])
                
         dlLink = dlinfo["url"]
         if dlLink == "server_under_maintenance":
diff --git a/module/plugins/hoster/XFileSharingPro.py b/module/plugins/hoster/XFileSharingPro.py
index 6b98b4d08..ad96cc3b0 100644
--- a/module/plugins/hoster/XFileSharingPro.py
+++ b/module/plugins/hoster/XFileSharingPro.py
@@ -33,8 +33,8 @@ class XFileSharingPro(SimpleHoster):
     """
     __name__ = "XFileSharingPro"
     __type__ = "hoster"
-    __pattern__ = r"http://(?:\w*\.)*((aieshare|amonshare|asixfiles|azsharing|banashare|batubia|bebasupload|boosterking|buckshare|bulletupload|crocshare|ddlanime|divxme|dopeshare|downupload|eyesfile|eyvx|fik1|file(4safe|4sharing|band|beep|bit|box|dove|fat|forth|made|mak|planet|playgroud|race|rio|strack|upper|velocity)|fooget|4bytez|freefilessharing|glumbouploads|grupload|heftyfile|hipfile|host4desi|hulkshare.com|idupin|imageporter|isharefast|jalurcepat|kingsupload|laoupload|linkzhost|loombo|maknyos|migahost|mlfat4arab|movreel|netuploaded|ok2upload|180upload|1hostclick|ovfile|putshare|pyramidfiles|q4share|queenshare|ravishare|rockdizfile|sendmyway|share(76|beast|hut|run|swift)|sharingonline|6ybh-upload|skipfile|spaadyshare|space4file|speedoshare|upload(baz|boost|c|dot|floor|ic|dville)|uptobox|vidbull|zalaa|zomgupload)\.com|(kupload|movbay|multishare|omegave|toucansharing|uflinq)\.org|(annonhost|fupload|muchshare|supashare|tusfiles|usershare|xuploading)\.net|(banicrazy|flowhot|upbrasil)\.info|(shareyourfilez)|.biz|(bzlink|)\.us|(cloudcache|fileserver)\.cc|(farshare|kingshare)\.to|(filemaze|filehost)\.ws|(goldfile|xfileshare)\.eu|(filestock|moidisk)\.ru|4up\.me|kfiles\.kz|odsiebie\.pl|upchi\.co\.il|upit\.in|verzend\.be)/\w{12}" 
-    __version__ = "0.05"
+    __pattern__ = r"^unmatchable$"
+    __version__ = "0.07"
     __description__ = """XFileSharingPro common hoster base"""
     __author_name__ = ("zoidberg")
     __author_mail__ = ("zoidberg@mujmail.cz")
@@ -86,6 +86,11 @@ class XFileSharingPro(SimpleHoster):
                 self.handleFree()
 
     def handleFree(self):
+        url = self.getDownloadLink()
+        self.logDebug("Download URL: %s" % url)
+        self.startDownload(url)
+        
+    def getDownloadLink(self):
         for i in range(5):
             data = self.getPostParameters()
             
@@ -104,7 +109,7 @@ class XFileSharingPro(SimpleHoster):
 
         else: self.fail("No valid captcha code entered")
         
-        self.startDownload(found.group(1))
+        return found.group(1)
 
     def handlePremium(self):
         self.html = self.load(self.pyfile.url, post = self.getPostParameters())
@@ -146,7 +151,7 @@ class XFileSharingPro(SimpleHoster):
         found = re.search(self.ERROR_PATTERN, self.html)
         if found:
             self.errmsg = found.group(1)
-            self.logWarning(self.errmsg)
+            self.logWarning(re.sub(self.errmsg, "<.*?>"," "))
 
             if 'wait' in self.errmsg:
                 wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in re.findall('(\d+)\s*(hour|minute|second)?', self.errmsg)])