checksum methods for RapidshareCom and UploadedTo

5 files changed, 139 insertions, 21 deletions

diff --git a/module/download_thread.py b/module/download_thread.py
index 98e4915c5..9c7fe1a42 100644
--- a/module/download_thread.py
+++ b/module/download_thread.py
@@ -107,9 +107,23 @@ class Download_Thread(threading.Thread):
             status.url = pyfile.plugin.get_file_url()
 
         status.type = "downloading"
+        
+        local_file = pyfile.download_folder + "/" + status.filename
+        pyfile.plugin.proceed(status.url, local_file)
 
-        pyfile.plugin.proceed(status.url, pyfile.download_folder + "/" + status.filename)
-
+        status.type = "checking"
+        
+        check, code = pyfile.plugin.check_file(local_file)
+        """
+        return codes:
+        0  - checksum ok
+        1  - checksum wrong
+        5  - can't get checksum
+        10 - not implemented
+        20 - unknown error
+        """
+        #print "checksum check returned: %s, %s" % (check, code)
+        
         status.type = "finished"
 
         #startet downloader
diff --git a/module/network/Request.py b/module/network/Request.py
index 092e75a01..6799563ac 100755
--- a/module/network/Request.py
+++ b/module/network/Request.py
@@ -117,7 +117,7 @@ class Request:
         else:
             get = ""
 
-        url = url + get
+        url = "%s?%s" % (url, get)
 
 
         if self.curl:
@@ -222,7 +222,7 @@ class Request:
         else:
             get = ""
 
-        url = url + get
+        url = "%s?%s" % (url, get)
 
         if self.curl:
 
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index 011f0e8f5..7b726c3ac 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -30,12 +30,12 @@ class Plugin():
         self.config = {}
         props = {}
         props['name'] = "BasePlugin"
-        props['version'] = "0.1"
+        props['version'] = "0.2"
         props['pattern'] = None
         props['type'] = "hoster"
         props['description'] = """Base Plugin"""
-        props['author_name'] = ("RaNaN", "spoob")
-        props['author_mail'] = ("RaNaN@pyload.org", "spoob@pyload.org")
+        props['author_name'] = ("RaNaN", "spoob", "mkaay")
+        props['author_mail'] = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de")
         self.props = props
         self.parent = parent
         self.req = Request()
@@ -121,3 +121,14 @@ class Plugin():
 
     def __call__(self):
         return self.props['name']
+    
+    def check_file(self, local_file):
+        """
+        return codes:
+        0  - checksum ok
+        1  - checksum wrong
+        5  - can't get checksum
+        10 - not implemented
+        20 - unknown error
+        """
+    	return (True, 10)
diff --git a/module/plugins/RapidshareCom.py b/module/plugins/RapidshareCom.py
index 9270102df..53e47b518 100644
--- a/module/plugins/RapidshareCom.py
+++ b/module/plugins/RapidshareCom.py
@@ -5,6 +5,7 @@ import re
 from time import time
 
 from Plugin import Plugin
+import hashlib
 
 class RapidshareCom(Plugin):
 
@@ -14,16 +15,18 @@ class RapidshareCom(Plugin):
         props['name'] = "RapidshareCom"
         props['type'] = "hoster"
         props['pattern'] = r"http://(?:www.)?(rs\d*.)?rapidshare.com/files/"
-        props['version'] = "0.4"
+        props['version'] = "0.5"
         props['description'] = """Rapidshare.com Download Plugin"""
-        props['author_name'] = ("spoob", "RaNaN")
-        props['author_mail'] = ("spoob@pyload.org", "ranan@pyload.org")
+        props['author_name'] = ("spoob", "RaNaN", "mkaay")
+        props['author_mail'] = ("spoob@pyload.org", "ranan@pyload.org", "mkaay@mkaay.de")
         self.props = props
         self.parent = parent
         self.html = [None, None]
         self.html_old = None         #time() where loaded the HTML
         self.time_plus_wait = None   #time() + wait in seconds
         self.want_reconnect = False
+        
+        self.urlRegex = re.compile(r'http://[\w\.]*?rapidshare\.com/files/([\d]{3,9})/?(.+)') # regex from jdownloader
 
         self.read_config()
         if self.config['premium']:
@@ -51,6 +54,8 @@ class RapidshareCom(Plugin):
             if not pyfile.status.exists:
                 raise Exception, "The file was not found on the server."
             
+            self.download_api_data()
+            
             pyfile.status.filename = self.get_file_name()
             
             if self.config['premium']:
@@ -71,6 +76,42 @@ class RapidshareCom(Plugin):
 
         return True
 
+    def download_api_data(self):
+        """
+        http://images.rapidshare.com/apidoc.txt
+        """
+        url = self.parent.url
+        api_url_base = "http://api.rapidshare.com/cgi-bin/rsapi.cgi"
+        api_param = {"sub": "checkfiles_v1", "files": "", "filenames": "", "incmd5": "1"}
+        m = self.urlRegex.search(url)
+        if m:
+            api_param["files"] = m.group(1)
+            api_param["filenames"] = m.group(2)
+            src = self.req.load(api_url_base, cookies=False, get=api_param)
+            if not src.find("ERROR"):
+                return
+            fields = src.split(",")
+            self.api_data = {}
+            self.api_data["fileid"] = fields[0]
+            self.api_data["filename"] = fields[1]
+            self.api_data["size"] = fields[2] # in bytes
+            self.api_data["serverid"] = fields[3]
+            self.api_data["status"] = fields[4]
+            """
+            status codes:
+                0=File not found
+                1=File OK (Downloading possible without any logging)
+                2=File OK (TrafficShare direct download without any logging)
+                3=Server down
+                4=File marked as illegal
+                5=Anonymous file locked, because it has more than 10 downloads already
+                6=File OK (TrafficShare direct download with enabled logging)
+            """
+            self.api_data["shorthost"] = fields[5]
+            self.api_data["checksum"] = fields[6].strip().lower() # md5
+            
+            self.api_data["mirror"] = "http://rs%(serverid)s%(shorthost)s.rapidshare.com/files/%(fileid)s/%(filename)s" % self.api_data
+
     def download_html(self):
         """ gets the url from self.parent.url saves html in self.html and parses
         """
@@ -87,7 +128,6 @@ class RapidshareCom(Plugin):
         self.get_wait_time()
 
     def get_wait_time(self):
-
         if re.search(r"is already downloading", self.html[1]) != None:
             self.time_plus_wait = time() + 10 * 60
         try:
@@ -116,6 +156,8 @@ class RapidshareCom(Plugin):
         """
         if self.config['premium']:
             self.start_dl = True
+            if self.api_data and self.api_data["mirror"]:
+                return self.api_data["mirror"]
             return self.parent.url
 
         #if (self.html_old + 5 * 60) < time(): # nach einiger zeit ist die file_url nicht mehr aktuell
@@ -123,6 +165,8 @@ class RapidshareCom(Plugin):
 
 
         try:
+            if self.api_data and self.api_data["mirror"]:
+                return self.api_data["mirror"]
             if self.config['server'] == "":
                 file_url_pattern = r".*name=\"dlf\" action=\"(.*)\" method=.*"
             else:
@@ -137,13 +181,25 @@ class RapidshareCom(Plugin):
             #raise Exception, "Error when retrieving download url"
 
     def get_file_name(self):
-            
+        if self.api_data and self.api_data["filename"]:
+            return self.api_data["filename"]
         file_name_pattern = r"<p class=\"downloadlink\">.+/(.+) <font"
         return re.findall(file_name_pattern, self.html[0])[0]
 
     def proceed(self, url, location):
-
         if self.config['premium']:
             self.req.add_auth(self.config['username'], self.config['password'])
-
         self.req.download(url, location)
+
+    def check_file(self, local_file):
+        if self.api_data and self.api_data["checksum"]:
+            h = hashlib.md5()
+            with open(local_file, "rb") as f:
+                h.update(f.read())
+            hexd = h.hexdigest()
+            if hexd == self.api_data["checksum"]:
+                return (True, 0)
+            else:
+                return (False, 1)
+        else:
+        	return (True, 5)
diff --git a/module/plugins/UploadedTo.py b/module/plugins/UploadedTo.py
index 50ea01459..bf5582d7c 100644
--- a/module/plugins/UploadedTo.py
+++ b/module/plugins/UploadedTo.py
@@ -4,6 +4,7 @@
 import re
 from time import time
 from Plugin import Plugin
+import hashlib
 
 class UploadedTo(Plugin):
 
@@ -13,15 +14,19 @@ class UploadedTo(Plugin):
         props['name'] = "UploadedTo"
         props['type'] = "hoster"
         props['pattern'] = r"http://(?:www\.)?u(?:p)?l(?:oaded)?\.to/"
-        props['version'] = "0.2"
+        props['version'] = "0.3"
         props['description'] = """Uploaded.to Download Plugin"""
         props['author_name'] = ("spoob", "mkaay")
         props['author_mail'] = ("spoob@pyload.org", "mkaay@mkaay.de")
         self.props = props
         self.parent = parent
         self.html = None
-        self.html_old = None         #time() where loaded the HTML
-        self.time_plus_wait = None   #time() + wait in seconds
+        self.html_old = None		#time() where loaded the HTML
+        self.time_plus_wait = None	#time() + wait in seconds
+        self.api_data = None
+        
+        self.longUrlRegex = re.compile(r"uploaded.to/file/(.*?)/")
+        self.shortUrlRegex = re.compile(r"ul.to/(.*)")
         
         self.want_reconnect = False
 
@@ -48,7 +53,9 @@ class UploadedTo(Plugin):
 
             if not pyfile.status.exists:
                 raise Exception, "The file was not found on the server."
-
+            
+            self.download_api_data()
+            
             pyfile.status.filename = self.get_file_name()
             
             if self.config['premium']:
@@ -70,6 +77,21 @@ class UploadedTo(Plugin):
                 raise Exception, "Error while preparing DL, HTML dump: %s" % self.html
 
         return True
+        
+    def download_api_data(self):
+        url = self.parent.url
+        match = self.longUrlRegex.search(url)
+        if not match:
+            match = self.shortUrlRegex.search(url)
+        if match:
+            src = self.req.load("http://uploaded.to/api/file", cookies=False, get={"id": match.group(1)})
+            if not src.find("404 Not Found"):
+                return
+            self.api_data = {}
+            lines = src.split("\r\n")
+            self.api_data["filename"] = lines[0]
+            self.api_data["size"] = lines[1] # in kbytes
+            self.api_data["checksum"] = lines[2] #sha1
 	
     def download_html(self):
         if self.config['premium']:
@@ -102,6 +124,8 @@ class UploadedTo(Plugin):
 
     def get_file_name(self):
         try:
+            if self.api_data and self.api_data["filename"]:
+                return self.api_data["filename"]
             file_name = re.search(r"<td><b>\s+(.+)\s", self.html).group(1)
             file_suffix = re.search(r"</td><td>(\..+)</td></tr>", self.html)
             if not file_suffix:
@@ -120,7 +144,20 @@ class UploadedTo(Plugin):
             return True
 
     def proceed(self, url, location):
-    	if self.config['premium']:
-        	self.req.download(url, location, cookies=True)
+        if self.config['premium']:
+            self.req.download(url, location, cookies=True)
+        else:
+            self.req.download(url, location, cookies=False, post={"download_submit": "Free Download"})
+
+    def check_file(self, local_file):
+        if self.api_data and self.api_data["checksum"]:
+            h = hashlib.sha1()
+            with open(local_file, "rb") as f:
+                h.update(f.read())
+            hexd = h.hexdigest()
+            if hexd == self.api_data["checksum"]:
+                return (True, 0)
+            else:
+                return (False, 1)
         else:
-        	self.req.download(url, location, cookies=False, post={"download_submit": "Free Download"})
+        	return (True, 5)