diff options
Diffstat (limited to 'module/plugins/hoster')
| -rw-r--r-- | module/plugins/hoster/MegauploadCom.py | 127 | ||||
| -rw-r--r-- | module/plugins/hoster/OneFichierCom.py | 156 | ||||
| -rw-r--r-- | module/plugins/hoster/UploadStationCom.py | 8 |
3 files changed, 180 insertions, 111 deletions
diff --git a/module/plugins/hoster/MegauploadCom.py b/module/plugins/hoster/MegauploadCom.py index 8aa37c47d..4ab476d1e 100644 --- a/module/plugins/hoster/MegauploadCom.py +++ b/module/plugins/hoster/MegauploadCom.py @@ -8,56 +8,66 @@ from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL from module.unescape import unescape +from module.PyFile import statusMap from pycurl import error def getInfo(urls): - url = "http://megaupload.com/mgr_linkcheck.php" - - ids = [x.split("=")[-1] for x in urls] + + result = [] - i = 0 + # MU API request post = {} - for id in ids: - post["id%i"%i] = id - i += 1 - - api = getURL(url, {}, post) - api = [re.split(r"&(?!amp;|#\d+;)", x) for x in re.split(r"&?(?=id[\d]+=)", api)] + fileIds = [x.split("=")[-1] for x in urls] # Get ids from urls + for i, fileId in enumerate(fileIds): + post["id%i" % i] = fileId + response = getURL(MegauploadCom.API_URL, post=post) - result = [] - i=0 - for data in api: - if data[0].startswith("id"): - tmp = [x.split("=") for x in data] - if tmp[0][1] == "0": - status = 2 - elif tmp[0][1] == "1": - status = 1 - elif tmp[2][1] == "3": - status = 3 - else: - status = 3 - - name = None - size = 0 - if status != 1: - name = unescape(tmp[3][1]) - size = tmp[1][1] + # Process API response + parts = [re.split(r"&(?!amp;|#\d+;)", x) for x in re.split(r"&?(?=id[\d]+=)", response)] + apiHosterMap = dict([elem.split('=') for elem in parts[0]]) + for entry in parts[1:]: + apiFileDataMap = dict([elem.split('=') for elem in entry]) + apiFileId = [key for key in apiFileDataMap.keys() if key.startswith('id')][0] + i = int(apiFileId.replace('id', '')) - result.append( (name, size, status, urls[i] ) ) - i += 1 + # File info + fileInfo = _translateAPIFileInfo(apiFileId, apiFileDataMap, apiHosterMap) + url = urls[i] + name = fileInfo.get('name', url) + size = fileInfo.get('size', 0) + status = fileInfo.get('status', statusMap['queued']) + + # Add result + result.append( (name, size, status, url ) ) yield result + +def _translateAPIFileInfo(apiFileId, apiFileDataMap, apiHosterMap): + + # Translate + fileInfo = {} + try: + fileInfo['status'] = MegauploadCom.API_STATUS_MAPPING[apiFileDataMap[apiFileId]] + fileInfo['name'] = apiFileDataMap['n'] + fileInfo['size'] = apiFileDataMap['s'] + fileInfo['hoster'] = apiHosterMap[apiFileDataMap['d']] + except: + pass + + return fileInfo class MegauploadCom(Hoster): __name__ = "MegauploadCom" __type__ = "hoster" __pattern__ = r"http://[\w\.]*?(megaupload)\.com/.*?(\?|&)d=[0-9A-Za-z]+" - __version__ = "0.22" + __version__ = "0.23" __description__ = """Megaupload.com Download Hoster""" __author_name__ = ("spoob") __author_mail__ = ("spoob@pyload.org") + + API_URL = "http://megaupload.com/mgr_linkcheck.php" + API_STATUS_MAPPING = {"0": statusMap['online'], "1": statusMap['offline'], "3": statusMap['temp. offline']} def init(self): self.html = [None, None] @@ -167,29 +177,30 @@ class MegauploadCom(Hoster): def download_api(self): - url = "http://megaupload.com/mgr_linkcheck.php" - - id = self.pyfile.url.split("=")[-1] - - - post = {"id0": id} - - api = self.load(url, {}, post) - self.log.debug("MU API: %s" % api) - api = [re.split(r"&(?!amp;|#\d+;)", x) for x in re.split(r"&?(?=id[\d]+=)", api)] - - for data in api: - if data[0].startswith("id"): - tmp = [x.split("=") for x in data] - if tmp[0][1] == "1": - self.offline() - - name = unescape(tmp[3][1]) - #size = tmp[1][1] - - self.api["name"] = name - self.pyfile.name = name - + # MU API request + fileId = self.pyfile.url.split("=")[-1] # Get file id from url + apiFileId = "id0" + post = {apiFileId: fileId} + response = getURL(self.API_URL, post=post) + self.log.debug("%s: API response [%s]" % (self.__name__, response)) + + # Translate API response + parts = [re.split(r"&(?!amp;|#\d+;)", x) for x in re.split(r"&?(?=id[\d]+=)", response)] + apiHosterMap = dict([elem.split('=') for elem in parts[0]]) + apiFileDataMap = dict([elem.split('=') for elem in parts[1]]) + self.api = _translateAPIFileInfo(apiFileId, apiFileDataMap, apiHosterMap) + + # File info + try: + self.pyfile.status = self.api['status'] + self.pyfile.name = self.api['name'] + self.pyfile.size = self.api['size'] + except KeyError: + self.log.warn("%s: Cannot recover all file [%s] info from API response." % (self.__name__, fileId)) + + # Fail if offline + if self.pyfile.status == statusMap['offline']: + self.offline() def get_file_url(self): file_url_pattern = 'id="downloadlink"><a href="(.*)"\s+(?:onclick|class)="' @@ -197,11 +208,11 @@ class MegauploadCom(Hoster): return search.group(1).replace(" ", "%20") def get_file_name(self): - if not self.api: + try: + return self.api["name"] + except KeyError: file_name_pattern = 'id="downloadlink"><a href="(.*)" onclick="' return re.search(file_name_pattern, self.html[1]).group(1).split("/")[-1] - else: - return self.api["name"] def get_wait_time(self): time = re.search(r"count=(\d+);", self.html[1]) diff --git a/module/plugins/hoster/OneFichierCom.py b/module/plugins/hoster/OneFichierCom.py index b1ce20fbd..69d6e81c0 100644 --- a/module/plugins/hoster/OneFichierCom.py +++ b/module/plugins/hoster/OneFichierCom.py @@ -4,15 +4,63 @@ import re from module.plugins.Hoster import Hoster +from module.network.RequestFactory import getURL + + +def getInfo(urls): + result = [] + + for url in urls: + + # Get file info html + id = re.match(OneFichierCom.__pattern__, url).group('id') + url = 'http://%s.1fichier.com/en' % id # Force response in english + html = getURL(url) + + # Offline? + if re.search(OneFichierCom.FILE_OFFLINE_PATTERN, html): + result.append((url, 0, 1, url)) + continue + + # Name + for pattern in OneFichierCom.FILE_NAME_PATTERNS: + m = re.search(pattern, html) + if m is not None: + name = m.group('name').strip() + + # Size + m = re.search(OneFichierCom.FILE_SIZE_PATTERN, html) + value = float(m.group('size')) + units = m.group('units')[0].upper() + pow = {'K' : 1, 'M' : 2, 'G' : 3}[units] + size = int(value*1024**pow) + + # Return info + result.append((name, size, 2, url)) + + yield result + class OneFichierCom(Hoster): __name__ = "OneFichierCom" __type__ = "hoster" - __pattern__ = r"http://[a-z0-9]+\.1fichier\.com/(.*)" - __version__ = "0.2" + __pattern__ = r"http://(?P<id>[a-z0-9]+)\.1fichier\.com(?P<remain>.*)" + __version__ = "0.3" __description__ = """1fichier.com download hoster""" __author_name__ = ("fragonib") __author_mail__ = ("fragonib[AT]yahoo[DOT]es") + + FILE_NAME_PATTERNS = ( + r'">File name :</th>[\t\r\n ]+<td>(?P<name>.*?)</td>', + r">Click here to download (?P<name>.*?)</a>", + r"content=\"Download the file named (?P<name>.*?)\">", + r"<title>Download the file\s*:\s*(?P<name>.*?)</title>" + ) + FILE_SIZE_PATTERN = r"<th>File size :</th>\s+<td>(?P<size>[\d\.]*) (?P<units>\w+)</td>" + DOWNLOAD_LINK_PATTERN = r'<br/> <br/> <br/> \s+<a href="(?P<url>http://.*?)"' + FILE_OFFLINE_PATTERN = r"(The requested file could not be found|The file may has been deleted by its owner)" + PASSWORD_PROTECTED_TOKEN = "protected by password" + WAITING_TOKEN = "Please wait a few seconds" def setup(self): self.html = None @@ -20,65 +68,75 @@ class OneFichierCom(Hoster): def process(self, pyfile): - self.download_html() - - if not self.file_exists(): - self.log.debug("OneFichierCom: File not yet available.") - self.offline() + # Get main page (english version) + url = self.getEnglishURL() + self.html = self.load(url) + self.handleErrors() - pyfile.name = self.get_file_name() - pyfile.size = self.get_file_size() + # Get file info + pyfile.name = self.getFileName() + pyfile.size = self.getFileSize() - url = self.get_file_url() - self.download(url) - - def download_html(self): - self.html = self.load(self.pyfile.url, cookies=False) + # Check for protection + if self.isProtected(): + password = pyfile.package().password + self.log.debug("%s: Submitting password [%s]" % (self.__name__, password)) + self.download(url, post={"password" : password}) + else: + downloadLink = self.getDownloadLink() + self.download(downloadLink) - def file_exists(self): - warnings = (r"The requested file could not be found", - r"The file may has been deleted by its owner", - r"Le fichier demandé n'existe pas\.", - r"Il a pu être supprimé par son propriétaire\.") - pattern = '(' + '|'.join(warnings) + ')' - if re.search(pattern, self.html) is not None: - return False - return True - - def get_file_url(self): - file_url_pattern = r"<br/>\ <br/>\ <br/>\ [\t\n\r ]+<a href=\"(?P<url>http://.*?)\"" - - m = re.search(file_url_pattern, self.html) - if m is not None: - url = m.group('url') - self.log.debug("OneFichierCom: Got file URL [%s]" % url) - return url + # Check download + self.handleDownloadedFile() - def get_file_name(self): - file_name_patterns = ( - r"\">(Nom du fichier :|File name :)</th>[\t\r\n ]+<td>(?P<name>.*?)</td>", - r"(>Cliquez ici pour télécharger|>Click here to download) (?P<name>.*?)</a>", - r"content=\"(Téléchargement du fichier |Download the file named )(?P<name>.*?)\">", - r"<title>(Téléchargement du fichier|Download the file)\s*:\s*(?P<name>.*?)</title>" - ) - - for pattern in file_name_patterns: + def getEnglishURL(self): + id = re.match(self.__pattern__, self.pyfile.url).group('id') + url = 'http://%s.1fichier.com/en' % id + return url + + def getFileName(self): + for pattern in self.FILE_NAME_PATTERNS: m = re.search(pattern, self.html) if m is not None: name = m.group('name').strip() - self.log.debug("OneFichierCom: Got file name [%s]" % name) + self.log.debug("%s: Got file name [%s]" % (self.__name__, name)) return name - def get_file_size(self): - file_size_pattern = r"<th>(Taille :|File size :)</th>[\t\n\r ]+<td>(?P<size>\d*)\s+(?P<units>.*?)</td>" - m = re.search(file_size_pattern, self.html) + def getFileSize(self): + m = re.search(self.FILE_SIZE_PATTERN, self.html) if m is not None: - size = int(m.group('size')) + size = float(m.group('size')) units = m.group('units')[0].upper() try: multiplier = 1024 ** {"K":1, "M":2, "G":3}[units] except KeyError: multiplier = 1 - bytes = size * multiplier - self.log.debug("OneFichierCom: Got file size of [%s] bytes" % bytes) - return bytes
\ No newline at end of file + bytes = int(size * multiplier) + self.log.debug("%s: Got file size of [%s] bytes" % (self.__name__, bytes)) + return bytes + + def isProtected(self): + if self.PASSWORD_PROTECTED_TOKEN in self.html: + self.log.debug("%s: Links are password protected" % self.__name__) + return True + return False + + def getDownloadLink(self): + m = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) + if m is not None: + url = m.group('url') + self.log.debug("%s: Got file URL [%s]" % (self.__name__, url)) + return url + + def handleErrors(self): + if re.search(self.FILE_OFFLINE_PATTERN, self.html) is not None: + self.log.debug("%s: File not yet available." % self.__name__) + self.offline() + + def handleDownloadedFile(self): + check = self.checkDownload({"wait": self.WAITING_TOKEN}) + if check == "wait": + wait = 5 + self.setWait(wait, True) + self.wait() + self.retry()
\ No newline at end of file diff --git a/module/plugins/hoster/UploadStationCom.py b/module/plugins/hoster/UploadStationCom.py index 32ab1972a..65a44b765 100644 --- a/module/plugins/hoster/UploadStationCom.py +++ b/module/plugins/hoster/UploadStationCom.py @@ -38,7 +38,7 @@ class UploadStationCom(Hoster): __name__ = "UploadStationCom"
__type__ = "hoster"
__pattern__ = r"http://(www\.)?uploadstation\.com/file/(?P<id>[A-Za-z0-9]+)"
- __version__ = "0.3"
+ __version__ = "0.31"
__description__ = """UploadStation.Com File Download Hoster"""
__author_name__ = ("fragonib")
__author_mail__ = ("fragonib[AT]yahoo[DOT]es")
@@ -84,8 +84,8 @@ class UploadStationCom(Hoster): # self.jsPage = self.load("http://uploadstation.com" + jsPage)
# Check download
- response = self.load(self.pyfile.url, post={"checkDownload" : "check"})
- self.log.debug("%s: Checking download, response [%s]" % (self.__name__, response))
+ response = self.load(self.pyfile.url, post={"checkDownload" : "check"}, utf8=True)
+ self.log.debug("%s: Checking download, response [%s]" % (self.__name__, response.encode('ascii', 'ignore')))
self.handleErrors(response)
# We got a captcha?
@@ -98,7 +98,7 @@ class UploadStationCom(Hoster): post={'recaptcha_challenge_field' : challenge,
'recaptcha_response_field' : code,
'recaptcha_shortencode_field' : self.fileId})
- self.log.debug("%s: Result of captcha resolving [%s]" % (self.__name__, response))
+ self.log.debug("%s: Result of captcha resolving [%s]" % (self.__name__, response.encode('ascii', 'ignore')))
self.handleCaptchaErrors(response)
# Process waiting
|