Several Crypters and Hosters fixes, improvements, python2.5 issues...

3 files changed, 180 insertions, 111 deletions

diff --git a/module/plugins/hoster/MegauploadCom.py b/module/plugins/hoster/MegauploadCom.py
index 8aa37c47d..4ab476d1e 100644
--- a/module/plugins/hoster/MegauploadCom.py
+++ b/module/plugins/hoster/MegauploadCom.py
@@ -8,56 +8,66 @@ from module.plugins.Hoster import Hoster
 from module.network.RequestFactory import getURL
 
 from module.unescape import unescape
+from module.PyFile import statusMap
 
 from pycurl import error
 
 def getInfo(urls):
-    url = "http://megaupload.com/mgr_linkcheck.php"
-    
-    ids = [x.split("=")[-1] for x in urls]
+
+    result = []
     
-    i = 0
+    # MU API request 
     post = {}
-    for id in ids:
-        post["id%i"%i] = id
-        i += 1
-        
-    api = getURL(url, {}, post)
-    api = [re.split(r"&(?!amp;|#\d+;)", x) for x in re.split(r"&?(?=id[\d]+=)", api)]
+    fileIds = [x.split("=")[-1] for x in urls]  # Get ids from urls
+    for i, fileId in enumerate(fileIds):
+        post["id%i" % i] = fileId
+    response = getURL(MegauploadCom.API_URL, post=post)
     
-    result = []
-    i=0
-    for data in api:
-        if data[0].startswith("id"):
-            tmp = [x.split("=") for x in data]
-            if tmp[0][1] == "0":
-                status = 2
-            elif tmp[0][1] == "1":
-                status = 1
-            elif tmp[2][1] == "3":
-                status = 3
-            else:
-                status = 3
-
-            name = None
-            size = 0
-            if status != 1:
-                name = unescape(tmp[3][1])
-                size = tmp[1][1]
+    # Process API response
+    parts = [re.split(r"&(?!amp;|#\d+;)", x) for x in re.split(r"&?(?=id[\d]+=)", response)]
+    apiHosterMap = dict([elem.split('=') for elem in parts[0]])
+    for entry in parts[1:]:
+        apiFileDataMap = dict([elem.split('=') for elem in entry])
+        apiFileId = [key for key in apiFileDataMap.keys() if key.startswith('id')][0]
+        i = int(apiFileId.replace('id', ''))
             
-            result.append( (name, size, status, urls[i] ) )
-            i += 1
+        # File info
+        fileInfo = _translateAPIFileInfo(apiFileId, apiFileDataMap, apiHosterMap)
+        url = urls[i]
+        name = fileInfo.get('name', url)
+        size = fileInfo.get('size', 0)
+        status = fileInfo.get('status', statusMap['queued'])
+        
+        # Add result
+        result.append( (name, size, status, url ) )
     
     yield result
+    
+def _translateAPIFileInfo(apiFileId, apiFileDataMap, apiHosterMap):
+    
+    # Translate
+    fileInfo = {}
+    try:
+        fileInfo['status'] = MegauploadCom.API_STATUS_MAPPING[apiFileDataMap[apiFileId]]
+        fileInfo['name'] = apiFileDataMap['n'] 
+        fileInfo['size'] = apiFileDataMap['s']
+        fileInfo['hoster'] = apiHosterMap[apiFileDataMap['d']]        
+    except:
+        pass
+
+    return fileInfo
 
 class MegauploadCom(Hoster):
     __name__ = "MegauploadCom"
     __type__ = "hoster"
     __pattern__ = r"http://[\w\.]*?(megaupload)\.com/.*?(\?|&)d=[0-9A-Za-z]+"
-    __version__ = "0.22"
+    __version__ = "0.23"
     __description__ = """Megaupload.com Download Hoster"""
     __author_name__ = ("spoob")
     __author_mail__ = ("spoob@pyload.org")
+    
+    API_URL = "http://megaupload.com/mgr_linkcheck.php"
+    API_STATUS_MAPPING = {"0": statusMap['online'], "1": statusMap['offline'], "3": statusMap['temp. offline']} 
 
     def init(self):
         self.html = [None, None]
@@ -167,29 +177,30 @@ class MegauploadCom(Hoster):
 
     def download_api(self):
 
-        url = "http://megaupload.com/mgr_linkcheck.php"
-
-        id = self.pyfile.url.split("=")[-1]
-
-
-        post = {"id0": id}
-
-        api = self.load(url, {}, post)
-        self.log.debug("MU API: %s" % api)
-        api = [re.split(r"&(?!amp;|#\d+;)", x) for x in re.split(r"&?(?=id[\d]+=)", api)]
-
-        for data in api:
-            if data[0].startswith("id"):
-                tmp = [x.split("=") for x in data]
-                if tmp[0][1] == "1":
-                    self.offline()
-
-                name = unescape(tmp[3][1])
-                #size = tmp[1][1]
-
-                self.api["name"] = name
-                self.pyfile.name = name
-
+        # MU API request 
+        fileId = self.pyfile.url.split("=")[-1] # Get file id from url
+        apiFileId = "id0"
+        post = {apiFileId: fileId}
+        response = getURL(self.API_URL, post=post)    
+        self.log.debug("%s: API response [%s]" % (self.__name__, response))
+        
+        # Translate API response
+        parts = [re.split(r"&(?!amp;|#\d+;)", x) for x in re.split(r"&?(?=id[\d]+=)", response)]
+        apiHosterMap = dict([elem.split('=') for elem in parts[0]])
+        apiFileDataMap = dict([elem.split('=') for elem in parts[1]])        
+        self.api = _translateAPIFileInfo(apiFileId, apiFileDataMap, apiHosterMap)
+
+        # File info
+        try:
+            self.pyfile.status = self.api['status']
+            self.pyfile.name = self.api['name'] 
+            self.pyfile.size = self.api['size']
+        except KeyError:
+            self.log.warn("%s: Cannot recover all file [%s] info from API response." % (self.__name__, fileId))
+        
+        # Fail if offline
+        if self.pyfile.status == statusMap['offline']:
+            self.offline()
 
     def get_file_url(self):
         file_url_pattern = 'id="downloadlink"><a href="(.*)"\s+(?:onclick|class)="'
@@ -197,11 +208,11 @@ class MegauploadCom(Hoster):
         return search.group(1).replace(" ", "%20")
 
     def get_file_name(self):
-        if not self.api:
+        try:
+            return self.api["name"]
+        except KeyError:
             file_name_pattern = 'id="downloadlink"><a href="(.*)" onclick="'
             return re.search(file_name_pattern, self.html[1]).group(1).split("/")[-1]
-        else:
-            return self.api["name"]
 
     def get_wait_time(self):
         time = re.search(r"count=(\d+);", self.html[1])
diff --git a/module/plugins/hoster/OneFichierCom.py b/module/plugins/hoster/OneFichierCom.py
index b1ce20fbd..69d6e81c0 100644
--- a/module/plugins/hoster/OneFichierCom.py
+++ b/module/plugins/hoster/OneFichierCom.py
@@ -4,15 +4,63 @@
 import re
 
 from module.plugins.Hoster import Hoster
+from module.network.RequestFactory import getURL
+
+
+def getInfo(urls):
+    result = []
+    
+    for url in urls:
+        
+        # Get file info html
+        id = re.match(OneFichierCom.__pattern__, url).group('id')
+        url = 'http://%s.1fichier.com/en' % id  # Force response in english
+        html = getURL(url) 
+        
+        # Offline?
+        if re.search(OneFichierCom.FILE_OFFLINE_PATTERN, html):
+            result.append((url, 0, 1, url))
+            continue
+        
+        # Name
+        for pattern in OneFichierCom.FILE_NAME_PATTERNS:
+            m = re.search(pattern, html)
+            if m is not None:
+                name = m.group('name').strip()
+        
+        # Size
+        m = re.search(OneFichierCom.FILE_SIZE_PATTERN, html)
+        value = float(m.group('size'))
+        units = m.group('units')[0].upper()
+        pow = {'K' : 1, 'M' : 2, 'G' : 3}[units] 
+        size = int(value*1024**pow)
+    
+        # Return info
+        result.append((name, size, 2, url))
+        
+    yield result
+
 
 class OneFichierCom(Hoster):
     __name__ = "OneFichierCom"
     __type__ = "hoster"
-    __pattern__ = r"http://[a-z0-9]+\.1fichier\.com/(.*)"
-    __version__ = "0.2"
+    __pattern__ = r"http://(?P<id>[a-z0-9]+)\.1fichier\.com(?P<remain>.*)"
+    __version__ = "0.3"
     __description__ = """1fichier.com download hoster"""
     __author_name__ = ("fragonib")
     __author_mail__ = ("fragonib[AT]yahoo[DOT]es")
+    
+    FILE_NAME_PATTERNS = (
+        r'">File name :</th>[\t\r\n ]+<td>(?P<name>.*?)</td>',
+        r">Click here to download (?P<name>.*?)</a>",
+        r"content=\"Download the file named (?P<name>.*?)\">", 
+        r"<title>Download the file\s*:\s*(?P<name>.*?)</title>"
+    )
+    FILE_SIZE_PATTERN = r"<th>File size :</th>\s+<td>(?P<size>[\d\.]*) (?P<units>\w+)</td>"
+    DOWNLOAD_LINK_PATTERN = r'<br/>&nbsp;<br/>&nbsp;<br/>&nbsp;\s+<a href="(?P<url>http://.*?)"'
+    FILE_OFFLINE_PATTERN = r"(The requested file could not be found|The file may has been deleted by its owner)"
+    PASSWORD_PROTECTED_TOKEN = "protected by password"
+    WAITING_TOKEN = "Please wait a few seconds"
 
     def setup(self):
         self.html = None
@@ -20,65 +68,75 @@ class OneFichierCom(Hoster):
 
     def process(self, pyfile):
 
-        self.download_html()
-
-        if not self.file_exists():
-            self.log.debug("OneFichierCom: File not yet available.")
-            self.offline()
+        # Get main page (english version)
+        url = self.getEnglishURL()
+        self.html = self.load(url)  
+        self.handleErrors()
         
-        pyfile.name = self.get_file_name()
-        pyfile.size = self.get_file_size()
+        # Get file info
+        pyfile.name = self.getFileName()
+        pyfile.size = self.getFileSize()
         
-        url = self.get_file_url()
-        self.download(url)
-
-    def download_html(self):
-        self.html = self.load(self.pyfile.url, cookies=False)
+        # Check for protection 
+        if self.isProtected():
+            password = pyfile.package().password
+            self.log.debug("%s: Submitting password [%s]" % (self.__name__, password))
+            self.download(url, post={"password" : password})
+        else:
+            downloadLink = self.getDownloadLink()
+            self.download(downloadLink)
         
-    def file_exists(self):
-        warnings = (r"The requested file could not be found",
-                    r"The file may has been deleted by its owner",
-                    r"Le fichier demandé n'existe pas\.",
-                    r"Il a pu être supprimé par son propriétaire\.")
-        pattern = '(' + '|'.join(warnings) + ')'
-        if re.search(pattern, self.html) is not None:
-            return False 
-        return True
-        
-    def get_file_url(self):
-        file_url_pattern = r"<br/>\&nbsp;<br/>\&nbsp;<br/>\&nbsp;[\t\n\r ]+<a href=\"(?P<url>http://.*?)\""
-        
-        m = re.search(file_url_pattern, self.html)
-        if m is not None:
-            url = m.group('url')
-            self.log.debug("OneFichierCom: Got file URL [%s]" % url)
-            return url
+        # Check download 
+        self.handleDownloadedFile()
 
-    def get_file_name(self):
-        file_name_patterns = (
-            r"\">(Nom du fichier :|File name :)</th>[\t\r\n ]+<td>(?P<name>.*?)</td>",
-            r"(>Cliquez ici pour télécharger|>Click here to download) (?P<name>.*?)</a>",
-            r"content=\"(Téléchargement du fichier |Download the file named )(?P<name>.*?)\">", 
-            r"<title>(Téléchargement du fichier|Download the file)\s*:\s*(?P<name>.*?)</title>"
-        )
-    
-        for pattern in file_name_patterns:
+    def getEnglishURL(self):
+        id = re.match(self.__pattern__, self.pyfile.url).group('id')
+        url = 'http://%s.1fichier.com/en' % id
+        return url
+
+    def getFileName(self):
+        for pattern in self.FILE_NAME_PATTERNS:
             m = re.search(pattern, self.html)
             if m is not None:
                 name = m.group('name').strip()
-                self.log.debug("OneFichierCom: Got file name [%s]" % name)
+                self.log.debug("%s: Got file name [%s]" % (self.__name__, name))
                 return name
             
-    def get_file_size(self):
-        file_size_pattern = r"<th>(Taille :|File size :)</th>[\t\n\r ]+<td>(?P<size>\d*)\s+(?P<units>.*?)</td>"        
-        m = re.search(file_size_pattern, self.html)
+    def getFileSize(self):
+        m = re.search(self.FILE_SIZE_PATTERN, self.html) 
         if m is not None:
-            size = int(m.group('size'))
+            size = float(m.group('size'))
             units = m.group('units')[0].upper()
             try:
                 multiplier = 1024 ** {"K":1, "M":2, "G":3}[units]
             except KeyError:
                 multiplier = 1
-            bytes = size * multiplier
-            self.log.debug("OneFichierCom: Got file size of [%s] bytes" % bytes)
-            return bytes
\ No newline at end of file
+            bytes = int(size * multiplier)
+            self.log.debug("%s: Got file size of [%s] bytes" % (self.__name__, bytes))
+            return bytes
+    
+    def isProtected(self):
+        if self.PASSWORD_PROTECTED_TOKEN in self.html:
+            self.log.debug("%s: Links are password protected" % self.__name__)
+            return True
+        return False
+        
+    def getDownloadLink(self):
+        m = re.search(self.DOWNLOAD_LINK_PATTERN, self.html)
+        if m is not None:
+            url = m.group('url')
+            self.log.debug("%s: Got file URL [%s]" % (self.__name__, url))
+            return url
+        
+    def handleErrors(self):
+        if re.search(self.FILE_OFFLINE_PATTERN, self.html) is not None:
+            self.log.debug("%s: File not yet available." % self.__name__)
+            self.offline()
+            
+    def handleDownloadedFile(self):
+        check = self.checkDownload({"wait": self.WAITING_TOKEN})
+        if check == "wait":
+            wait = 5
+            self.setWait(wait, True)
+            self.wait()
+            self.retry()
\ No newline at end of file
diff --git a/module/plugins/hoster/UploadStationCom.py b/module/plugins/hoster/UploadStationCom.py
index 32ab1972a..65a44b765 100644
--- a/module/plugins/hoster/UploadStationCom.py
+++ b/module/plugins/hoster/UploadStationCom.py
@@ -38,7 +38,7 @@ class UploadStationCom(Hoster):
     __name__ = "UploadStationCom"

     __type__ = "hoster"

     __pattern__ = r"http://(www\.)?uploadstation\.com/file/(?P<id>[A-Za-z0-9]+)"

-    __version__ = "0.3"

+    __version__ = "0.31"

     __description__ = """UploadStation.Com File Download Hoster"""

     __author_name__ = ("fragonib")

     __author_mail__ = ("fragonib[AT]yahoo[DOT]es")

@@ -84,8 +84,8 @@ class UploadStationCom(Hoster):
         # self.jsPage = self.load("http://uploadstation.com" + jsPage)

         

         # Check download

-        response = self.load(self.pyfile.url, post={"checkDownload" : "check"})

-        self.log.debug("%s: Checking download, response [%s]" % (self.__name__, response))

+        response = self.load(self.pyfile.url, post={"checkDownload" : "check"}, utf8=True)

+        self.log.debug("%s: Checking download, response [%s]" % (self.__name__, response.encode('ascii', 'ignore')))

         self.handleErrors(response)

         

         # We got a captcha?

@@ -98,7 +98,7 @@ class UploadStationCom(Hoster):
                                   post={'recaptcha_challenge_field' : challenge,

                                         'recaptcha_response_field' : code, 

                                         'recaptcha_shortencode_field' : self.fileId})

-            self.log.debug("%s: Result of captcha resolving [%s]" % (self.__name__, response))

+            self.log.debug("%s: Result of captcha resolving [%s]" % (self.__name__, response.encode('ascii', 'ignore')))

             self.handleCaptchaErrors(response)

 

         # Process waiting