diff options
| -rw-r--r-- | module/plugins/hooks/Checksum.py | 21 | ||||
| -rw-r--r-- | module/plugins/hoster/OneFichierCom.py | 13 | ||||
| -rw-r--r-- | module/plugins/hoster/TurbobitNet.py | 118 |
3 files changed, 110 insertions, 42 deletions
diff --git a/module/plugins/hooks/Checksum.py b/module/plugins/hooks/Checksum.py index cb6f4bfe8..aec4bd0d7 100644 --- a/module/plugins/hooks/Checksum.py +++ b/module/plugins/hooks/Checksum.py @@ -18,6 +18,7 @@ """ from __future__ import with_statement import hashlib, zlib +from os import remove from os.path import getsize, isfile from module.utils import save_join, fs_encode @@ -49,7 +50,7 @@ def computeChecksum(local_file, algorithm): class Checksum(Hook): __name__ = "Checksum" - __version__ = "0.05" + __version__ = "0.06" __description__ = "Verify downloaded file size and checksum (enable in general preferences)" __config__ = [("activated", "bool", "Activated", True), ("action", "fail;retry;nothing", "What to do if check fails?", "retry"), @@ -80,11 +81,15 @@ class Checksum(Hook): self.logDebug(data) - download_folder = self.config['general']['download_folder'] - local_file = fs_encode(save_join(download_folder, pyfile.package().folder, pyfile.name)) + if not pyfile.plugin.lastDownload: + self.checkFailed(pyfile, None, "No file downloaded") + + local_file = fs_encode(pyfile.plugin.lastDownload) + #download_folder = self.config['general']['download_folder'] + #local_file = fs_encode(save_join(download_folder, pyfile.package().folder, pyfile.name)) if not isfile(local_file): - self.checkFailed(pyfile, "File does not exist") + self.checkFailed(pyfile, None, "File does not exist") # validate file size if "size" in data: @@ -92,7 +97,7 @@ class Checksum(Hook): file_size = getsize(local_file) if api_size != file_size: self.logWarning("File %s has incorrect size: %d B (%d expected)" % (pyfile.name, file_size, api_size)) - self.checkFailed(pyfile, "Incorrect file size") + self.checkFailed(pyfile, local_file, "Incorrect file size") del data['size'] # validate checksum @@ -109,15 +114,17 @@ class Checksum(Hook): return else: self.logWarning("%s checksum for file %s does not match (%s != %s)" % (key.upper(), pyfile.name, checksum, data[key])) - self.checkFailed(pyfile, "Checksums do not match") + self.checkFailed(pyfile, local_file, "Checksums do not match") else: self.logWarning("Unsupported hashing algorithm: %s" % key.upper()) else: self.logWarning("Unable to validate checksum for file %s" % (pyfile.name)) - def checkFailed(self, pyfile, msg): + def checkFailed(self, pyfile, local_file, msg): action = self.getConfig("action") if action == "fail": pyfile.plugin.fail(reason = msg) elif action == "retry": + if local_file: + remove(local_file) pyfile.plugin.retry(reason = msg, max_tries = self.getConfig("max_tries"))
\ No newline at end of file diff --git a/module/plugins/hoster/OneFichierCom.py b/module/plugins/hoster/OneFichierCom.py index 128942b75..3a4ff7275 100644 --- a/module/plugins/hoster/OneFichierCom.py +++ b/module/plugins/hoster/OneFichierCom.py @@ -7,7 +7,7 @@ class OneFichierCom(SimpleHoster): __name__ = "OneFichierCom" __type__ = "hoster" __pattern__ = r"(http://(\w+)\.((1fichier|d(es)?fichiers|pjointe)\.(com|fr|net|org)|(cjoint|mesfichiers|piecejointe|oi)\.(org|net)|tenvoi\.(com|org|net)|dl4free\.com|alterupload\.com|megadl.fr))" - __version__ = "0.43" + __version__ = "0.44" __description__ = """1fichier.com download hoster""" __author_name__ = ("fragonib", "the-razer", "zoidberg") __author_mail__ = ("fragonib[AT]yahoo[DOT]es", "daniel_ AT gmx DOT net", "zoidberg@mujmail.cz") @@ -19,7 +19,7 @@ class OneFichierCom(SimpleHoster): DOWNLOAD_LINK_PATTERN = r'<br/> <br/> <br/> \s+<a href="(?P<url>http://.*?)"' PASSWORD_PROTECTED_TOKEN = "protected by password" - WAITING_TOKEN = "Please wait a few seconds" + WAITING_PATTERN = "you must wait (\d+) minutes" def process(self, pyfile): found = re.search(self.__pattern__, pyfile.url) @@ -27,8 +27,9 @@ class OneFichierCom(SimpleHoster): url = "http://%s.%s/en/" % (found.group(2), found.group(3)) self.html = self.load(url, decode = True) - if self.WAITING_TOKEN in self.html: - self.waitAndRetry(120) + found = re.search(self.WAITING_PATTERN, self.html) + if found: + self.waitAndRetry(int(found.group(1)) * 60) self.getFileInfo() @@ -46,9 +47,9 @@ class OneFichierCom(SimpleHoster): self.checkDownloadedFile() def checkDownloadedFile(self): - check = self.checkDownload({"wait": self.WAITING_TOKEN}) + check = self.checkDownload({"wait": self.WAITING_PATTERN}) if check == "wait": - self.waitAndRetry(60) + self.waitAndRetry(int(self.lastcheck.group(1)) * 60) def waitAndRetry(self, wait_time): self.setWait(wait_time, True) diff --git a/module/plugins/hoster/TurbobitNet.py b/module/plugins/hoster/TurbobitNet.py index 9de7f9bd0..2a65c1b00 100644 --- a/module/plugins/hoster/TurbobitNet.py +++ b/module/plugins/hoster/TurbobitNet.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- """ + Copyright (C) 2012 pyLoad team + Copyright (C) 2012 JD-Team support@jdownloader.org + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, @@ -17,8 +20,13 @@ """ import re +import random +from urllib import quote +from binascii import hexlify, unhexlify +from Crypto.Cipher import ARC4 -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from module.network.RequestFactory import getURL +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp from module.plugins.ReCaptcha import ReCaptcha from pycurl import HTTPHEADER @@ -27,29 +35,38 @@ class TurbobitNet(SimpleHoster): __name__ = "TurbobitNet" __type__ = "hoster" __pattern__ = r"http://(?:\w*\.)?(turbobit.net|unextfiles.com)/(?:download/free/)?(?P<ID>\w+).*" - __version__ = "0.05" + __version__ = "0.06" __description__ = """Turbobit.net plugin""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - + FILE_INFO_PATTERN = r"<span class='file-icon1[^>]*>(?P<N>[^<]+)</span>\s*\((?P<S>[^\)]+)\)\s*</h1>" #long filenames are shortened - FILE_NAME_PATTERN = r'<meta name="keywords" content="\s*(?P<N>[^,]+)' #full name but missing on page2 - FILE_OFFLINE_PATTERN = r'<h2>File Not Found</h2>' - FILE_URL_REPLACEMENTS = [(r"(?<=http://)([^/]+)", "turbobit.net")] + FILE_NAME_PATTERN = r'<meta name="keywords" content="\s+(?P<N>[^,]+)' #full name but missing on page2 + FILE_OFFLINE_PATTERN = r'<h2>File Not Found</h2>|html\(\'File was not found' + FILE_URL_REPLACEMENTS = [(r"http://(?:\w*\.)?(turbobit.net|unextfiles.com)/(?:download/free/)?(?P<ID>\w+).*", "http://turbobit.net/\g<ID>.html")] SH_COOKIES = [("turbobit.net", "user_lang", "en")] - + CAPTCHA_KEY_PATTERN = r'src="http://api\.recaptcha\.net/challenge\?k=([^"]+)"' DOWNLOAD_URL_PATTERN = r'(?P<url>/download/redirect/[^"\']+)' LIMIT_WAIT_PATTERN = r'<div id="time-limit-text">\s*.*?<span id=\'timeout\'>(\d+)</span>' - CAPTCHA_SRC_PATTERN = r'<img alt="Captcha" src="(.*?)"' + CAPTCHA_SRC_PATTERN = r'<img alt="Captcha" src="(.*?)"' - def handleFree(self): + def handleFree(self): self.url = "http://turbobit.net/download/free/%s" % self.file_info['ID'] - if not '/download/free/' in self.pyfile.url: - self.html = self.load(self.url) - - recaptcha = ReCaptcha(self) + self.html = self.load(self.url) + + rtUpdate = self.getRtUpdate() + + self.solveCaptcha() + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.url = self.getDownloadUrl(rtUpdate) + self.wait() + self.html = self.load(self.url) + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) + self.downloadFile() + + def solveCaptcha(self): for i in range(5): found = re.search(self.LIMIT_WAIT_PATTERN, self.html) if found: @@ -57,12 +74,13 @@ class TurbobitNet(SimpleHoster): self.setWait(wait_time, wait_time > 60) self.wait() self.retry() - + action, inputs = self.parseHtmlForm("action='#'") - if not inputs: self.parseError("inputs") + if not inputs: self.parseError("captcha form") self.logDebug(inputs) - + if inputs['captcha_type'] == 'recaptcha': + recaptcha = ReCaptcha(self) found = re.search(self.CAPTCHA_KEY_PATTERN, self.html) captcha_key = found.group(1) if found else '6LcTGLoSAAAAAHCWY9TTIrQfjUlxu6kZlTYP50_c' inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) @@ -70,36 +88,78 @@ class TurbobitNet(SimpleHoster): found = re.search(self.CAPTCHA_SRC_PATTERN, self.html) if not found: self.parseError('captcha') captcha_url = found.group(1) - inputs['captcha_response'] = self.decryptCaptcha(captcha_url) + inputs['captcha_response'] = self.decryptCaptcha(captcha_url) self.logDebug(inputs) self.html = self.load(self.url, post = inputs) - + if not "<div class='download-timer-header'>" in self.html: self.invalidCaptcha() else: self.correctCaptcha() break else: self.fail("Invalid captcha") - + + def getRtUpdate(self): + rtUpdate = self.getStorage("rtUpdate") + if not rtUpdate: + if self.getStorage("version") != self.__version__ or int(self.getStorage("timestamp", 0)) + 86400000 < timestamp(): + # that's right, we are even using jdownloader updates + rtUpdate = getURL("http://update0.jdownloader.org/pluginstuff/tbupdate.js") + rtUpdate = self.decrypt(rtUpdate.splitlines()[1]) + rtUpdate = rtUpdate.replace("for each(var ss in[2,4,5,6]){inn+=s[ss]}", "inn+=s[2]+s[4]+s[5]+s[6]") + + self.logDebug("rtUpdate") + self.setStorage("rtUpdate", rtUpdate) + self.setStorage("timestamp", timestamp()) + self.setStorage("version", self.__version__) + else: + self.logError("Unable to download, wait for update...") + self.tempOffline() + + return rtUpdate + + def getDownloadUrl(self, rtUpdate): self.req.http.lastURL = self.url - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) - - self.setWait(60, False) - self.wait() - - self.html = self.load("http://turbobit.net/download/getLinkTimeout/" + self.file_info['ID']) - self.downloadFile() + + found = re.search("(/\w+/timeout\.js\?\w+=)([^\"\'<>]+)", self.html) + url = "http://turbobit.net%s%s" % (found.groups() if found else ('/files/timeout.js?ver=', ''.join(random.choice('0123456789ABCDEF') for x in range(32)))) + fun = self.load(url) + + self.setWait(65, False) + + for b in [1,3]: + self.jscode = "var id = \'%s\';var b = %d;var inn = \'%s\';%sout" % (self.file_info['ID'], b, quote(fun), rtUpdate) + + try: + out = self.js.eval(self.jscode) + if out.startswith('/download/'): + return "http://turbobit.net%s" % out.strip() + except Exception, e: + self.logError(e) + else: + # retry with updated js + self.delStorage("rtUpdate") + self.retry() + + def decrypt(self, data): + cipher = ARC4.new(hexlify('E\x15\xa1\x9e\xa3M\xa0\xc6\xa0\x84\xb6H\x83\xa8o\xa0')) + return unhexlify(cipher.encrypt(unhexlify(data))) + def getLocalTimeString(): + lt = time.localtime() + tz = time.altzone if lt.tm_isdst else time.timezone + return "%s GMT%+03d%02d" % (time.strftime("%a %b %d %Y %H:%M:%S", lt), -tz // 3600, tz % 3600) + def handlePremium(self): self.logDebug("Premium download as user %s" % self.user) self.downloadFile() - + def downloadFile(self): found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if not found: self.parseError("download link") + if not found: self.parseError("download link") self.url = "http://turbobit.net" + found.group('url') self.logDebug(self.url) - self.download(self.url) + self.download(self.url) getInfo = create_getInfo(TurbobitNet)
\ No newline at end of file |