diff options
Diffstat (limited to 'module/plugins/internal')
| -rw-r--r-- | module/plugins/internal/AbstractExtractor.py | 102 | ||||
| -rw-r--r-- | module/plugins/internal/CaptchaService.py | 477 | ||||
| -rw-r--r-- | module/plugins/internal/DeadCrypter.py | 28 | ||||
| -rw-r--r-- | module/plugins/internal/DeadHoster.py | 32 | ||||
| -rw-r--r-- | module/plugins/internal/Extractor.py | 129 | ||||
| -rw-r--r-- | module/plugins/internal/MultiHook.py | 308 | ||||
| -rw-r--r-- | module/plugins/internal/MultiHoster.py | 219 | ||||
| -rw-r--r-- | module/plugins/internal/SevenZip.py | 163 | ||||
| -rw-r--r-- | module/plugins/internal/SimpleCrypter.py | 190 | ||||
| -rw-r--r-- | module/plugins/internal/SimpleDereferer.py | 98 | ||||
| -rw-r--r-- | module/plugins/internal/SimpleHoster.py | 757 | ||||
| -rw-r--r-- | module/plugins/internal/UnRar.py | 227 | ||||
| -rw-r--r-- | module/plugins/internal/UnZip.py | 71 | ||||
| -rw-r--r-- | module/plugins/internal/XFSAccount.py | 175 | ||||
| -rw-r--r-- | module/plugins/internal/XFSCrypter.py | 45 | ||||
| -rw-r--r-- | module/plugins/internal/XFSHoster.py | 350 | ||||
| -rw-r--r-- | module/plugins/internal/XFSPAccount.py | 82 |
17 files changed, 2635 insertions, 818 deletions
diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py deleted file mode 100644 index c7f54f847..000000000 --- a/module/plugins/internal/AbstractExtractor.py +++ /dev/null @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- - - -class ArchiveError(Exception): - pass - - -class CRCError(Exception): - pass - - -class WrongPassword(Exception): - pass - - -class AbtractExtractor: - __name__ = "AbtractExtractor" - __version__ = "0.1" - - __description__ = """Abtract extractor plugin""" - __author_name__ = "pyLoad Team" - __author_mail__ = "admin@pyload.org" - - - @staticmethod - def checkDeps(): - """ Check if system statisfy dependencies - :return: boolean - """ - return True - - @staticmethod - def getTargets(files_ids): - """ Filter suited targets from list of filename id tuple list - :param files_ids: List of filepathes - :return: List of targets, id tuple list - """ - raise NotImplementedError - - def __init__(self, m, file, out, fullpath, overwrite, excludefiles, renice): - """Initialize extractor for specific file - - :param m: ExtractArchive Hook plugin - :param file: Absolute filepath - :param out: Absolute path to destination directory - :param fullpath: extract to fullpath - :param overwrite: Overwrite existing archives - :param renice: Renice value - """ - self.m = m - self.file = file - self.out = out - self.fullpath = fullpath - self.overwrite = overwrite - self.excludefiles = excludefiles - self.renice = renice - self.files = [] #: Store extracted files here - - def init(self): - """ Initialize additional data structures """ - pass - - def checkArchive(self): - """Check if password if needed. Raise ArchiveError if integrity is - questionable. - - :return: boolean - :raises ArchiveError - """ - return False - - def checkPassword(self, password): - """ Check if the given password is/might be correct. - If it can not be decided at this point return true. - - :param password: - :return: boolean - """ - return True - - def extract(self, progress, password=None): - """Extract the archive. Raise specific errors in case of failure. - - :param progress: Progress function, call this to update status - :param password password to use - :raises WrongPassword - :raises CRCError - :raises ArchiveError - :return: - """ - raise NotImplementedError - - def getDeleteFiles(self): - """Return list of files to delete, do *not* delete them here. - - :return: List with paths of files to delete - """ - raise NotImplementedError - - def getExtractedFiles(self): - """Populate self.files at some point while extracting""" - return self.files diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index d325d7928..c6e83fe7d 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -1,95 +1,472 @@ # -*- coding: utf-8 -*- import re -from random import random +import time + +from base64 import b64encode +from random import random, randint +from urlparse import urljoin, urlparse + +from module.common.json_layer import json_loads class CaptchaService: - __name__ = "CaptchaService" - __version__ = "0.05" + __name__ = "CaptchaService" + __version__ = "0.20" + + __description__ = """Base captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + - __description__ = """Captcha service plugin""" - __author_name__ = "pyLoad Team" - __author_mail__ = "admin@pyload.org" + key = None #: last key detected def __init__(self, plugin): self.plugin = plugin -class ReCaptcha: - RECAPTCHA_KEY_PATTERN = r"https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P<key>\w+)" - RECAPTCHA_KEY_AJAX_PATTERN = r"Recaptcha\.create\s*\(\s*[\"'](?P<key>\w+)[\"']\s*," + def detect_key(self, html=None): + raise NotImplementedError - recaptcha_key = None + def challenge(self, key=None, html=None): + raise NotImplementedError - def __init__(self, plugin): - self.plugin = plugin - def detect_key(self, html): - m = re.search(self.RECAPTCHA_KEY_PATTERN, html) - if m is None: - m = re.search(self.RECAPTCHA_KEY_AJAX_PATTERN, html) + def result(self, server, challenge): + raise NotImplementedError + + +class ReCaptcha(CaptchaService): + __name__ = "ReCaptcha" + __version__ = "0.11" + + __description__ = """ReCaptcha captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com"), + ("zapp-brannigan", "fuerst.reinje@web.de")] + + + KEY_V2_PATTERN = r'(?:data-sitekey=["\']|["\']sitekey["\']:\s*["\'])([\w-]+)' + KEY_V1_PATTERN = r'(?:recaptcha(?:/api|\.net)/(?:challenge|noscript)\?k=|Recaptcha\.create\s*\(\s*["\'])([\w-]+)' + + + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = _("ReCaptcha html not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + m = re.search(self.KEY_V2_PATTERN, html) or re.search(self.KEY_V1_PATTERN, html) if m: - self.recaptcha_key = m.group('key') - return self.recaptcha_key + self.key = m.group(1).strip() + self.plugin.logDebug("ReCaptcha key: %s" % self.key) + return self.key else: + self.plugin.logDebug("ReCaptcha key not found") return None - def challenge(self, key=None): - if key is None and self.recaptcha_key: - key = self.recaptcha_key - else: - raise TypeError("ReCaptcha key not found") - js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={"k": key}, cookies=True) + def challenge(self, key=None, html=None, version=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = _("ReCaptcha html not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + challenge = "challenge_v%s" % (version if version in (1, 2) else + 2 if re.search(self.KEY_V2_PATTERN, html) else 1) + + return getattr(self, challenge)(key, html) + + + def challenge_v1(self, key=None, html=None): + if not key: + if self.detect_key(html): + key = self.key + else: + errmsg = _("ReCaptcha key not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + html = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}) try: - challenge = re.search("challenge : '(.*?)',", js).group(1) - server = re.search("server : '(.*?)',", js).group(1) - except: - self.plugin.fail("recaptcha error") - result = self.result(server, challenge) + challenge = re.search("challenge : '(.+?)',", html).group(1) + server = re.search("server : '(.+?)',", html).group(1) + + except AttributeError: + errmsg = _("ReCaptcha challenge pattern not found") + self.plugin.fail(errmsg) + raise AttributeError(errmsg) + + self.plugin.logDebug("ReCaptcha challenge: %s" % challenge) + + return self.result(server, challenge), challenge - return challenge, result def result(self, server, challenge): - return self.plugin.decryptCaptcha("%simage" % server, get={"c": challenge}, - cookies=True, forceUser=True, imgtype="jpg") + result = self.plugin.decryptCaptcha("%simage" % server, + get={'c': challenge}, + cookies=True, + forceUser=True, + imgtype="jpg") + + self.plugin.logDebug("ReCaptcha result: %s" % result) + + return result + + + def _collectApiInfo(self): + html = self.plugin.req.load("http://www.google.com/recaptcha/api.js") + a = re.search(r'po.src = \'(.*?)\';', html).group(1) + vers = a.split("/")[5] + + self.plugin.logDebug("ReCaptcha API version: %s" %vers) + + language = a.split("__")[1].split(".")[0] + + self.plugin.logDebug("ReCaptcha API language: %s" % language) + + html = self.plugin.req.load("https://apis.google.com/js/api.js") + b = re.search(r'"h":"(.*?)","', html).group(1) + jsh = b.decode('unicode-escape') + + self.plugin.logDebug("ReCaptcha API jsh-string: %s" % jsh) + + return vers, language, jsh + + + def _prepareTimeAndRpc(self): + self.plugin.req.load("http://www.google.com/recaptcha/api2/demo") + + millis = int(round(time.time() * 1000)) + + self.plugin.logDebug("ReCaptcha time: %s" % millis) + + rand = randint(1, 99999999) + a = "0.%s" % str(rand * 2147483647) + rpc = int(100000000 * float(a)) + + self.plugin.logDebug("ReCaptcha rpc-token: %s" % rpc) + + return millis, rpc + + + def challenge_v2(self, key=None, html=None): + if not key: + if self.detect_key(html): + key = self.key + else: + errmsg = _("ReCaptcha key not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + try: + parent = urljoin("http://", urlparse(self.plugin.pyfile.url).netloc) + + except Exception: + parent = "" + + botguardstring = "!A" + vers, language, jsh = self._collectApiInfo() + millis, rpc = self._prepareTimeAndRpc() + + html = self.plugin.req.load("https://www.google.com/recaptcha/api2/anchor", + get={'k' : key, + 'hl' : language, + 'v' : vers, + 'usegapi' : "1", + 'jsh' : "%s#id=IO_%s" % (jsh, millis), + 'parent' : parent, + 'pfname' : "", + 'rpctoken': rpc}) + + token1 = re.search(r'id="recaptcha-token" value="(.*?)">', html) + self.plugin.logDebug("ReCaptcha token #1: %s" % token1.group(1)) + + html = self.plugin.req.load("https://www.google.com/recaptcha/api2/frame", + get={'c' : token1.group(1), + 'hl' : language, + 'v' : vers, + 'bg' : botguardstring, + 'usegapi': "1", + 'jsh' : jsh}).decode('unicode-escape') + + token2 = re.search(r'"finput","(.*?)",', html) + self.plugin.logDebug("ReCaptcha token #2: %s" % token2.group(1)) + + token3 = re.search(r'."asconf".\s.\s,"(.*?)".', html) + self.plugin.logDebug("ReCaptcha token #3: %s" % token3.group(1)) + + html = self.plugin.req.load("https://www.google.com/recaptcha/api2/reload", + post={'c' : token2.group(1), + 'reason': "fi", + 'fbg' : token3.group(1)}) + + token4 = re.search(r'"rresp","(.*?)",', html) + self.plugin.logDebug("ReCaptcha token #4: %s" % token4.group(1)) + + millis_captcha_loading = int(round(time.time() * 1000)) + captcha_response = self.plugin.decryptCaptcha("https://www.google.com/recaptcha/api2/payload", + get={'c':token4.group(1)}, forceUser=True) + response = b64encode('{"response":"%s"}' % captcha_response) + + self.plugin.logDebug("ReCaptcha result: %s" % response) + + timeToSolve = int(round(time.time() * 1000)) - millis_captcha_loading + timeToSolveMore = timeToSolve + int(float("0." + str(randint(1, 99999999))) * 500) + + html = self.plugin.req.load("https://www.google.com/recaptcha/api2/userverify", + post={'c' : token4.group(1), + 'response': response, + 't' : timeToSolve, + 'ct' : timeToSolveMore, + 'bg' : botguardstring}) + + token5 = re.search(r'"uvresp","(.*?)",', html) + self.plugin.logDebug("ReCaptcha token #5: %s" % token5.group(1)) + + return token5.group(1), None class AdsCaptcha(CaptchaService): + __name__ = "AdsCaptcha" + __version__ = "0.07" + + __description__ = """AdsCaptcha captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + - def challenge(self, src): - js = self.plugin.req.load(src, cookies=True) + CAPTCHAID_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(\d+)' + PUBLICKEY_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*PublicKey=([\w-]+)' + + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = _("AdsCaptcha html not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + m = re.search(self.PUBLICKEY_PATTERN, html) + n = re.search(self.CAPTCHAID_PATTERN, html) + if m and n: + self.key = (m.group(1).strip(), n.group(1).strip()) #: key is the tuple(PublicKey, CaptchaId) + self.plugin.logDebug("AdsCaptcha key|id: %s | %s" % self.key) + return self.key + else: + self.plugin.logDebug("AdsCaptcha key or id not found") + return None + + + def challenge(self, key=None, html=None): + if not key: + if self.detect_key(html): + key = self.key + else: + errmsg = _("AdsCaptcha key not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + PublicKey, CaptchaId = key + + html = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}) try: - challenge = re.search("challenge: '(.*?)',", js).group(1) - server = re.search("server: '(.*?)',", js).group(1) - except: - self.plugin.fail("adscaptcha error") - result = self.result(server, challenge) + challenge = re.search("challenge: '(.+?)',", html).group(1) + server = re.search("server: '(.+?)',", html).group(1) + + except AttributeError: + errmsg = _("AdsCaptcha challenge pattern not found") + self.plugin.fail(errmsg) + raise AttributeError(errmsg) + + self.plugin.logDebug("AdsCaptcha challenge: %s" % challenge) + + return self.result(server, challenge), challenge - return challenge, result def result(self, server, challenge): - return self.plugin.decryptCaptcha("%sChallenge.aspx" % server, get={"cid": challenge, "dummy": random()}, - cookies=True, imgtype="jpg") + result = self.plugin.decryptCaptcha("%sChallenge.aspx" % server, + get={'cid': challenge, 'dummy': random()}, + cookies=True, + imgtype="jpg") + + self.plugin.logDebug("AdsCaptcha result: %s" % result) + + return result class SolveMedia(CaptchaService): + __name__ = "SolveMedia" + __version__ = "0.08" + + __description__ = """SolveMedia captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + - def challenge(self, src): - html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript?k=%s" % src, cookies=True) + KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']' + + + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = _("SolveMedia html not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + m = re.search(self.KEY_PATTERN, html) + if m: + self.key = m.group(1).strip() + self.plugin.logDebug("SolveMedia key: %s" % self.key) + return self.key + else: + self.plugin.logDebug("SolveMedia key not found") + return None + + + def challenge(self, key=None, html=None): + if not key: + if self.detect_key(html): + key = self.key + else: + errmsg = _("SolveMedia key not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}) try: challenge = re.search(r'<input type=hidden name="adcopy_challenge" id="adcopy_challenge" value="([^"]+)">', html).group(1) - except: - self.plugin.fail("solvemedia error") - result = self.result(challenge) + server = "http://api.solvemedia.com/papi/media" + + except AttributeError: + errmsg = _("SolveMedia challenge pattern not found") + self.plugin.fail(errmsg) + raise AttributeError(errmsg) + + self.plugin.logDebug("SolveMedia challenge: %s" % challenge) + + return self.result(server, challenge), challenge + + + def result(self, server, challenge): + result = self.plugin.decryptCaptcha(server, get={'c': challenge}, imgtype="gif") + + self.plugin.logDebug("SolveMedia result: %s" % result) + + return result + + +class AdYouLike(CaptchaService): + __name__ = "AdYouLike" + __version__ = "0.04" + + __description__ = """AdYouLike captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + AYL_PATTERN = r'Adyoulike\.create\s*\((.+?)\)' + CALLBACK_PATTERN = r'(Adyoulike\.g\._jsonp_\d+)' + + + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = _("AdYouLike html not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + m = re.search(self.AYL_PATTERN, html) + n = re.search(self.CALLBACK_PATTERN, html) + if m and n: + self.key = (m.group(1).strip(), n.group(1).strip()) + self.plugin.logDebug("AdYouLike ayl|callback: %s | %s" % self.key) + return self.key #: key is the tuple(ayl, callback) + else: + self.plugin.logDebug("AdYouLike ayl or callback not found") + return None + + + def challenge(self, key=None, html=None): + if not key: + if self.detect_key(html): + key = self.key + else: + errmsg = _("AdYouLike key not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + ayl, callback = key + + # {"adyoulike":{"key":"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, + # "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} + ayl = json_loads(ayl) + + html = self.plugin.req.load("http://api-ayl.appspot.com/challenge", + get={'key' : ayl['adyoulike']['key'], + 'env' : ayl['all']['env'], + 'callback': callback}) + try: + challenge = json_loads(re.search(callback + r'\s*\((.+?)\)', html).group(1)) + + except AttributeError: + errmsg = _("AdYouLike challenge pattern not found") + self.plugin.fail(errmsg) + raise AttributeError(errmsg) + + self.plugin.logDebug("AdYouLike challenge: %s" % challenge) + + return self.result(ayl, challenge), challenge + + + def result(self, server, challenge): + # Adyoulike.g._jsonp_5579316662423138 + # ({"translations":{"fr":{"instructions_visual":"Recopiez « Soonnight » ci-dessous :"}}, + # "site_under":true,"clickable":true,"pixels":{"VIDEO_050":[],"DISPLAY":[],"VIDEO_000":[],"VIDEO_100":[], + # "VIDEO_025":[],"VIDEO_075":[]},"medium_type":"image/adyoulike", + # "iframes":{"big":"<iframe src=\"http://www.soonnight.com/campagn.html\" scrolling=\"no\" + # height=\"250\" width=\"300\" frameborder=\"0\"></iframe>"},"shares":{},"id":256, + # "token":"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1","formats":{"small":{"y":300,"x":0,"w":300,"h":60}, + # "big":{"y":0,"x":0,"w":300,"h":250},"hover":{"y":440,"x":0,"w":300,"h":60}}, + # "tid":"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) + + if isinstance(server, basestring): + server = json_loads(server) + + if isinstance(challenge, basestring): + challenge = json_loads(challenge) + + try: + instructions_visual = challenge['translations'][server['all']['lang']]['instructions_visual'] + result = re.search(u'«(.+?)»', instructions_visual).group(1).strip() + + except AttributeError: + errmsg = _("AdYouLike result not found") + self.plugin.fail(errmsg) + raise AttributeError(errmsg) + + result = {'_ayl_captcha_engine' : "adyoulike", + '_ayl_env' : server['all']['env'], + '_ayl_tid' : challenge['tid'], + '_ayl_token_challenge': challenge['token'], + '_ayl_response' : response} - return challenge, result + self.plugin.logDebug("AdYouLike result: %s" % result) - def result(self, challenge): - return self.plugin.decryptCaptcha("http://api.solvemedia.com/papi/media?c=%s" % challenge, imgtype="gif") + return result diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index d19731cb5..866d177cf 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -1,19 +1,31 @@ # -*- coding: utf-8 -*- +from module.plugins.internal.SimpleCrypter import create_getInfo from module.plugins.Crypter import Crypter as _Crypter class DeadCrypter(_Crypter): - __name__ = "DeadCrypter" - __version__ = "0.01" - __type__ = "crypter" + __name__ = "DeadCrypter" + __type__ = "crypter" + __version__ = "0.04" - __pattern__ = None + __pattern__ = r'^unmatchable$' - __description__ = """Crypter is no longer available""" - __author_name__ = "stickell" - __author_mail__ = "l.stickell@yahoo.it" + __description__ = """ Crypter is no longer available """ + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + @classmethod + def apiInfo(cls, url="", get={}, post={}): + api = super(DeadCrypter, self).apiInfo(url, get, post) + api['status'] = 1 + return api def setup(self): - self.fail("Crypter is no longer available") + self.pyfile.error = "Crypter is no longer available" + self.offline() #@TODO: self.offline("Crypter is no longer available") + + +getInfo = create_getInfo(DeadCrypter) diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index a7cf1f3ea..a6ad92607 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -1,27 +1,31 @@ # -*- coding: utf-8 -*- +from module.plugins.internal.SimpleHoster import create_getInfo from module.plugins.Hoster import Hoster as _Hoster -def create_getInfo(plugin): +class DeadHoster(_Hoster): + __name__ = "DeadHoster" + __type__ = "hoster" + __version__ = "0.14" - def getInfo(urls): - yield [('#N/A: ' + url, 0, 1, url) for url in urls] + __pattern__ = r'^unmatchable$' - return getInfo + __description__ = """ Hoster is no longer available """ + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] -class DeadHoster(_Hoster): - __name__ = "DeadHoster" - __version__ = "0.11" - __type__ = "hoster" + @classmethod + def apiInfo(cls, url="", get={}, post={}): + api = super(DeadHoster, self).apiInfo(url, get, post) + api['status'] = 1 + return api - __pattern__ = None - __description__ = """Hoster is no longer available""" - __author_name__ = "zoidberg" - __author_mail__ = "zoidberg@mujmail.cz" + def setup(self): + self.pyfile.error = "Hoster is no longer available" + self.offline() #@TODO: self.offline("Hoster is no longer available") - def setup(self): - self.fail("Hoster is no longer available") +getInfo = create_getInfo(DeadHoster) diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py new file mode 100644 index 000000000..ddf0f8a85 --- /dev/null +++ b/module/plugins/internal/Extractor.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- + +import os + +from module.PyFile import PyFile +from module.utils import fs_encode + + +class ArchiveError(Exception): + pass + + +class CRCError(Exception): + pass + + +class PasswordError(Exception): + pass + + +class Extractor: + __name__ = "Extractor" + __version__ = "0.15" + + __description__ = """Base extractor plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "ranan@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + EXTENSIONS = [] + + + @classmethod + def isArchive(cls, filename): + name = os.path.basename(filename).lower() + return any(name.endswith(ext) for ext in cls.EXTENSIONS) + + + @classmethod + def checkDeps(cls): + """ Check if system statisfy dependencies + :return: boolean + """ + return True + + + @classmethod + def getTargets(cls, files_ids): + """ Filter suited targets from list of filename id tuple list + :param files_ids: List of filepathes + :return: List of targets, id tuple list + """ + raise NotImplementedError + + + def __init__(self, manager, filename, out, + fullpath=True, + overwrite=False, + excludefiles=[], + renice=0, + delete=False, + keepbroken=False, + fid=None): + """ Initialize extractor for specific file """ + self.manager = manager + self.target = fs_encode(filename) + self.out = out + self.fullpath = fullpath + self.overwrite = overwrite + self.excludefiles = excludefiles + self.renice = renice + self.delete = delete + self.keepbroken = keepbroken + self.files = [] #: Store extracted files here + + pyfile = self.manager.core.files.getFile(fid) if fid else None + self.notifyProgress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None + + + def init(self): + """ Initialize additional data structures """ + pass + + + def checkArchive(self): + """Check if password if needed. Raise ArchiveError if integrity is + questionable. + + :return: boolean + :raises ArchiveError + """ + return False + + + def checkPassword(self, password): + """ Check if the given password is/might be correct. + If it can not be decided at this point return true. + + :param password: + :return: boolean + """ + return True + + + def extract(self, password=None): + """Extract the archive. Raise specific errors in case of failure. + + :param progress: Progress function, call this to update status + :param password password to use + :raises PasswordError + :raises CRCError + :raises ArchiveError + :return: + """ + raise NotImplementedError + + + def getDeleteFiles(self): + """Return list of files to delete, do *not* delete them here. + + :return: List with paths of files to delete + """ + return [self.target] + + + def getExtractedFiles(self): + """Populate self.files at some point while extracting""" + return self.files diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py new file mode 100644 index 000000000..652443098 --- /dev/null +++ b/module/plugins/internal/MultiHook.py @@ -0,0 +1,308 @@ +# -*- coding: utf-8 -*- + +import re + +from time import sleep + +from module.plugins.Hook import Hook +from module.utils import decode, remove_chars + + +class MultiHook(Hook): + __name__ = "MultiHook" + __type__ = "hook" + __version__ = "0.37" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Hook plugin for multi hoster/crypter""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + MIN_INTERVAL = 1 * 60 * 60 + + DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"), + (r'1fichier\.com' , "onefichier.com" ), + (r'2shared\.com' , "twoshared.com" ), + (r'4shared\.com' , "fourshared.com" ), + (r'bayfiles\.net' , "bayfiles.com" ), + (r'cloudnator\.com' , "shragle.com" ), + (r'dfiles\.eu' , "depositfiles.com" ), + (r'easy-share\.com' , "crocko.com" ), + (r'freakshare\.net' , "freakshare.com" ), + (r'hellshare\.com' , "hellshare.cz" ), + (r'ifile\.it' , "filecloud.io" ), + (r'nowdownload\.\w+', "nowdownload.sx" ), + (r'nowvideo\.\w+' , "nowvideo.sx" ), + (r'putlocker\.com' , "firedrive.com" ), + (r'share-?rapid\.cz', "multishare.cz" ), + (r'ul\.to' , "uploaded.to" ), + (r'uploaded\.net' , "uploaded.to" ), + (r'uploadhero\.co' , "uploadhero.com" ), + (r'zshares\.net' , "zshare.net" ), + (r'(\d+.+)' , "X\1" )] + + + def setup(self): + self.plugins = [] + self.supported = [] + self.new_supported = [] + + self.account = None + self.pluginclass = None + self.pluginmodule = None + self.pluginname = None + self.plugintype = None + + self._initPlugin() + + + def _initPlugin(self): + plugin, type = self.core.pluginManager.findPlugin(self.__name__) + + if not plugin: + self.logWarning("Hook plugin will be deactivated due missing plugin reference") + self.setConfig('activated', False) + else: + self.pluginname = self.__name__ + self.plugintype = type + self.pluginmodule = self.core.pluginManager.loadModule(type, self.__name__) + self.pluginclass = getattr(self.pluginmodule, self.__name__) + + + def _loadAccount(self): + self.account = self.core.accountManager.getAccountPlugin(self.pluginname) + + if self.account and not self.account.canUse(): + self.account = None + + if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: + self.logWarning("Hook plugin will be deactivated due missing account reference") + self.setConfig('activated', False) + + + def coreReady(self): + self._loadAccount() + + + def getURL(self, *args, **kwargs): #@TODO: Remove in 0.4.10 + """ see HTTPRequest for argument list """ + h = pyreq.getHTTPRequest(timeout=120) + try: + if not 'decode' in kwargs: + kwargs['decode'] = True + rep = h.load(*args, **kwargs) + finally: + h.close() + + return rep + + + def getConfig(self, option, default=''): + """getConfig with default value - sublass may not implements all config options""" + try: + return self.getConf(option) + + except KeyError: + return default + + + def pluginsCached(self): + if self.plugins: + return self.plugins + + for _i in xrange(3): + try: + pluginset = self._pluginSet(self.getHosters() if self.plugintype == "hoster" else self.getCrypters()) + + except Exception, e: + self.logError(e, "Waiting 1 minute and retry") + sleep(60) + + else: + break + else: + return list() + + try: + configmode = self.getConfig("pluginmode", 'all') + if configmode in ("listed", "unlisted"): + pluginlist = self.getConfig("pluginlist", '').replace('|', ',').replace(';', ',').split(',') + configset = self._pluginSet(pluginlist) + + if configmode == "listed": + pluginset &= configset + else: + pluginset -= configset + + except Exception, e: + self.logError(e) + + self.plugins = list(pluginset) + + return self.plugins + + + def _pluginSet(self, plugins): + plugins = set((decode(x).strip().lower() for x in plugins if '.' in x)) + + for rf, rt in self.DOMAIN_REPLACEMENTS: + regex = re.compile(rf) + for p in filter(lambda x: regex.match(x), plugins): + plugins.remove(p) + plugins.add(re.sub(rf, rt, p)) + + plugins.discard('') + + return plugins + + + def getHosters(self): + """Load list of supported hoster + + :return: List of domain names + """ + raise NotImplementedError + + + def getCrypters(self): + """Load list of supported crypters + + :return: List of domain names + """ + raise NotImplementedError + + + def periodical(self): + """reload plugin list periodically""" + self.logInfo(_("Reloading supported %s list") % self.plugintype) + + old_supported = self.supported + + self.supported = [] + self.new_supported = [] + self.plugins = [] + + self.overridePlugins() + + old_supported = [plugin for plugin in old_supported if plugin not in self.supported] + + if old_supported: + self.logDebug("Unload: %s" % ", ".join(old_supported)) + for plugin in old_supported: + self.unloadPlugin(plugin) + + if self.getConfig("reload", True): + self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_INTERVAL) + else: + self.core.scheduler.removeJob(self.cb) + self.cb = None + + + def overridePlugins(self): + excludedList = [] + + if self.plugintype == "hoster": + pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) + accountList = [account.type.lower() for account in self.core.api.getAccounts(False) if account.valid and account.premium] + else: + pluginMap = {} + accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.core.pluginManager.crypterPlugins.iterkeys()] + + for plugin in self.pluginsCached(): + name = remove_chars(plugin, "-.") + + if name in accountList: + excludedList.append(plugin) + else: + if name in pluginMap: + self.supported.append(pluginMap[name]) + else: + self.new_supported.append(plugin) + + if not self.supported and not self.new_supported: + self.logError(_("No %s loaded") % self.plugintype) + return + + # inject plugin plugin + self.logDebug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) + + for plugin in self.supported: + hdict = self.core.pluginManager.plugins[self.plugintype][plugin] + hdict['new_module'] = self.pluginmodule + hdict['new_name'] = self.pluginname + + if excludedList: + self.logInfo(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList)))) + + if self.new_supported: + plugins = sorted(self.new_supported) + + self.logDebug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) + + # create new regexp + regexp = r'.*(?P<DOMAIN>%s).*' % "|".join([x.replace(".", "\.") for x in plugins]) + if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and '://' in self.pluginclass.__pattern__: + regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) + + self.logDebug("Regexp: %s" % regexp) + + hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] + hdict['pattern'] = regexp + hdict['re'] = re.compile(regexp) + + + def unloadPlugin(self, plugin): + hdict = self.core.pluginManager.plugins[self.plugintype][plugin] + if "module" in hdict: + del hdict['module'] + + if "new_module" in hdict: + del hdict['new_module'] + del hdict['new_name'] + + + def unload(self): + """Remove override for all plugins. Scheduler job is removed by hookmanager""" + for plugin in self.supported: + self.unloadPlugin(plugin) + + # reset pattern + hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] + + hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') + hdict['re'] = re.compile(hdict['pattern']) + + + def downloadFailed(self, pyfile): + """remove plugin override if download fails but not if file is offline/temp.offline""" + if pyfile.status != 8 or not self.getConfig("revertfailed", True): + return + + hdict = self.core.pluginManager.plugins[self.plugintype][pyfile.pluginname] + if "new_name" in hdict and hdict['new_name'] == self.pluginname: + if pyfile.error == "MultiHook": + self.logDebug("Unload MultiHook", pyfile.pluginname, hdict) + self.unloadPlugin(pyfile.pluginname) + pyfile.setStatus("queued") + pyfile.sync() + else: + retries = max(self.getConfig("retry", 10), 0) + wait_time = max(self.getConfig("retryinterval", 1), 0) + + if 0 < retries > pyfile.plugin.retries: + self.logInfo(_("Retrying: %s") % pyfile.name) + pyfile.setCustomStatus("MultiHook", "queued") + pyfile.sync() + + pyfile.plugin.retries += 1 + pyfile.plugin.setWait(wait_time) + pyfile.plugin.wait() diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 46d943bd7..ad5b6283e 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -2,190 +2,85 @@ import re -from module.utils import remove_chars -from module.plugins.Hook import Hook +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies -class MultiHoster(Hook): - __name__ = "AbtractExtractor" - __version__ = "0.19" +class MultiHoster(SimpleHoster): + __name__ = "MultiHoster" + __type__ = "hoster" + __version__ = "0.36" - __description__ = """Generic MultiHoster plugin""" - __author_name__ = "pyLoad Team" - __author_mail__ = "admin@pyload.org" + __pattern__ = r'^unmatchable$' - replacements = [("2shared.com", "twoshared.com"), ("4shared.com", "fourshared.com"), ("cloudnator.com", "shragle.com"), - ("ifile.it", "filecloud.io"), ("easy-share.com", "crocko.com"), ("freakshare.net", "freakshare.com"), - ("hellshare.com", "hellshare.cz"), ("share-rapid.cz", "sharerapid.com"), ("sharerapid.cz", "sharerapid.com"), - ("ul.to", "uploaded.to"), ("uploaded.net", "uploaded.to"), ("1fichier.com", "onefichier.com")] - ignored = [] - interval = 24 * 60 * 60 #: reload hosters daily + __description__ = """Multi hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + LOGIN_ACCOUNT = True def setup(self): - self.hosters = [] - self.supported = [] - self.new_supported = [] + self.chunkLimit = 1 + self.multiDL = bool(self.account) + self.resumeDownload = self.premium + - def getConfig(self, option, default=''): - """getConfig with default value - sublass may not implements all config options""" - try: - return self.getConf(option) - except KeyError: - return default + def prepare(self): + self.info = {} + self.html = "" + self.link = "" #@TODO: Move to hoster class in 0.4.10 + self.directDL = False #@TODO: Move to hoster class in 0.4.10 - def getHosterCached(self): - if not self.hosters: + if self.LOGIN_ACCOUNT and not self.account: + self.fail(_("Required account not found")) - try: - hosterSet = self.toHosterSet(self.getHoster()) - set(self.ignored) - except Exception, e: - self.logError("%s" % str(e)) - return [] + self.req.setOption("timeout", 120) - try: - configMode = self.getConfig('hosterListMode', 'all') - if configMode in ("listed", "unlisted"): - configSet = self.toHosterSet(self.getConfig('hosterList', '').replace('|', ',').replace(';', ',').split(',')) + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) + + if self.DIRECT_LINK is None: + self.directDL = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) + else: + self.directDL = self.DIRECT_LINK - if configMode == "listed": - hosterSet &= configSet - else: - hosterSet -= configSet + self.pyfile.url = replace_patterns(self.pyfile.url, + self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 - except Exception, e: - self.logError("%s" % str(e)) - self.hosters = list(hosterSet) + def process(self, pyfile): + self.prepare() - return self.hosters + if self.directDL: + self.checkInfo() + self.logDebug("Looking for direct download link...") + self.handleDirect(pyfile) - def toHosterSet(self, hosters): - hosters = set((str(x).strip().lower() for x in hosters)) + if not self.link and not self.lastDownload: + self.preload() - for rep in self.replacements: - if rep[0] in hosters: - hosters.remove(rep[0]) - hosters.add(rep[1]) + self.checkErrors() + self.checkStatus(getinfo=False) - hosters.discard('') - return hosters + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as premium download") + self.handlePremium(pyfile) - def getHoster(self): - """Load list of supported hoster + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as free download") + self.handleFree(pyfile) - :return: List of domain names - """ - raise NotImplementedError + self.downloadLink(self.link, True) + self.checkFile() - def coreReady(self): - if self.cb: - self.core.scheduler.removeJob(self.cb) - self.setConfig("activated", True) #: config not in sync after plugin reload + def handlePremium(self, pyfile): + return self.handleFree(pyfile) - cfg_interval = self.getConfig("interval", None) #: reload interval in hours - if cfg_interval is not None: - self.interval = cfg_interval * 60 * 60 - if self.interval: - self._periodical() + def handleFree(self, pyfile): + if self.premium: + raise NotImplementedError else: - self.periodical() - - def initPeriodical(self): - pass - - def periodical(self): - """reload hoster list periodically""" - self.logInfo("Reloading supported hoster list") - - old_supported = self.supported - self.supported, self.new_supported, self.hosters = [], [], [] - - self.overridePlugins() - - old_supported = [hoster for hoster in old_supported if hoster not in self.supported] - if old_supported: - self.logDebug("UNLOAD: %s" % ", ".join(old_supported)) - for hoster in old_supported: - self.unloadHoster(hoster) - - def overridePlugins(self): - pluginMap = {} - for name in self.core.pluginManager.hosterPlugins.keys(): - pluginMap[name.lower()] = name - - accountList = [name.lower() for name, data in self.core.accountManager.accounts.items() if data] - excludedList = [] - - for hoster in self.getHosterCached(): - name = remove_chars(hoster.lower(), "-.") - - if name in accountList: - excludedList.append(hoster) - else: - if name in pluginMap: - self.supported.append(pluginMap[name]) - else: - self.new_supported.append(hoster) - - if not self.supported and not self.new_supported: - self.logError(_("No Hoster loaded")) - return - - module = self.core.pluginManager.getPlugin(self.__name__) - klass = getattr(module, self.__name__) - - # inject plugin plugin - self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) - for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - dict['new_module'] = module - dict['new_name'] = self.__name__ - - if excludedList: - self.logInfo("The following hosters were not overwritten - account exists: %s" % ", ".join(sorted(excludedList))) - - if self.new_supported: - self.logDebug("New Hosters: %s" % ", ".join(sorted(self.new_supported))) - - # create new regexp - regexp = r".*(%s).*" % "|".join([x.replace(".", "\\.") for x in self.new_supported]) - if hasattr(klass, "__pattern__") and isinstance(klass.__pattern__, basestring) and '://' in klass.__pattern__: - regexp = r"%s|%s" % (klass.__pattern__, regexp) - - self.logDebug("Regexp: %s" % regexp) - - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict['pattern'] = regexp - dict['re'] = re.compile(regexp) - - def unloadHoster(self, hoster): - dict = self.core.pluginManager.hosterPlugins[hoster] - if "module" in dict: - del dict['module'] - - if "new_module" in dict: - del dict['new_module'] - del dict['new_name'] - - def unload(self): - """Remove override for all hosters. Scheduler job is removed by hookmanager""" - for hoster in self.supported: - self.unloadHoster(hoster) - - # reset pattern - klass = getattr(self.core.pluginManager.getPlugin(self.__name__), self.__name__) - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict['pattern'] = getattr(klass, "__pattern__", r'^unmatchable$') - dict['re'] = re.compile(dict['pattern']) - - def downloadFailed(self, pyfile): - """remove plugin override if download fails but not if file is offline/temp.offline""" - if pyfile.hasStatus("failed") and self.getConfig("unloadFailing", True): - hdict = self.core.pluginManager.hosterPlugins[pyfile.pluginname] - if "new_name" in hdict and hdict['new_name'] == self.__name__: - self.logDebug("Unload MultiHoster", pyfile.pluginname, hdict) - self.unloadHoster(pyfile.pluginname) - pyfile.setStatus("queued") + self.fail(_("Required premium account not found")) diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py new file mode 100644 index 000000000..508cf9c8d --- /dev/null +++ b/module/plugins/internal/SevenZip.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- + +import os +import re + +from subprocess import Popen, PIPE + +from module.plugins.internal.UnRar import UnRar, renice +from module.utils import save_join + + +class SevenZip(UnRar): + __name__ = "SevenZip" + __version__ = "0.02" + + __description__ = """7-Zip extractor plugin""" + __license__ = "GPLv3" + __authors__ = [("Michael Nowak", ""), + ("Walter Purcaro", "vuolter@gmail.com")] + + + CMD = "7z" + + EXTENSIONS = [".7z", ".xz", ".zip", ".gz", ".gzip", ".tgz", ".bz2", ".bzip2", + ".tbz2", ".tbz", ".tar", ".wim", ".swm", ".lzma", ".rar", ".cab", + ".arj", ".z", ".taz", ".cpio", ".rpm", ".deb", ".lzh", ".lha", + ".chm", ".chw", ".hxs", ".iso", ".msi", ".doc", ".xls", ".ppt", + ".dmg", ".xar", ".hfs", ".exe", ".ntfs", ".fat", ".vhd", ".mbr", + ".squashfs", ".cramfs", ".scap"] + + + #@NOTE: there are some more uncovered 7z formats + re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)') + re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password)', re.I) + re_wrongcrc = re.compile(r'Encrypted\s+\=\s+\+', re.I) + + + @classmethod + def checkDeps(cls): + if os.name == "nt": + cls.CMD = os.path.join(pypath, "7z.exe") + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + p.communicate() + else: + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) + p.communicate() + + return True + + + def checkArchive(self): + p = self.call_cmd("l", "-slt", self.target) + out, err = p.communicate() + + if p.returncode > 1: + raise ArchiveError("Process terminated") + + # check if output or error macthes the 'wrong password'-Regexp + if self.re_wrongpwd.search(out): + return True + + # check if output matches 'Encrypted = +' + if self.re_wrongcrc.search(out): + return True + + # check if archive is empty + self.files = self.list() + if not self.files: + raise ArchiveError("Empty Archive") + + return False + + + def checkPassword(self, password): + p = self.call_cmd("l", self.target, password=password) + p.communicate() + return p.returncode == 0 + + + def extract(self, password=None): + command = "x" if self.fullpath else "e" + + p = self.call_cmd(command, '-o' + self.out, self.target, password=password) + + renice(p.pid, self.renice) + + progressstring = "" + while True: + c = p.stdout.read(1) + # quit loop on eof + if not c: + break + # reading a percentage sign -> set progress and restart + if c == '%': + self.notifyProgress(int(progressstring)) + progressstring = "" + # not reading a digit -> therefore restart + elif c not in digits: + progressstring = "" + # add digit to progressstring + else: + progressstring += c + + # retrieve stderr + err = p.stderr.read() + + if self.re_wrongpwd.search(err): + raise PasswordError + + elif self.re_wrongcrc.search(err): + raise CRCError + + elif err.strip(): #: raise error if anything is on stderr + raise ArchiveError(err.strip()) + + if p.returncode > 1: + raise ArchiveError("Process terminated") + + if not self.files: + self.files = self.list(password) + + + def list(self, password=None): + command = "l" if self.fullpath else "l" + + p = self.call_cmd(command, self.target, password=password) + out, err = p.communicate() + code = p.returncode + + if "Can not open" in err: + raise ArchiveError("Cannot open file") + + if code != 0: + raise ArchiveError("Process terminated unsuccessful") + + result = set() + for groups in self.re_filelist.findall(out): + f = groups[-1].strip() + result.add(save_join(self.out, f)) + + return list(result) + + + def call_cmd(self, command, *xargs, **kwargs): + args = [] + + #overwrite flag + if self.overwrite: + args.append("-y") + + #set a password + if "password" in kwargs and kwargs["password"]: + args.append("-p%s" % kwargs["password"]) + else: + args.append("-p-") + + #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue + call = [self.cmd, command] + args + list(xargs) + + self.manager.logDebug(" ".join([decode(arg) for arg in call])) + + p = Popen(call, stdout=PIPE, stderr=PIPE) + return p diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index dc93bef4d..b2622b5e0 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -2,117 +2,169 @@ import re +from urlparse import urljoin, urlparse + from module.plugins.Crypter import Crypter -from module.utils import html_unescape -from module.plugins.internal.SimpleHoster import PluginParseError, replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies +from module.utils import fixup -class SimpleCrypter(Crypter): - __name__ = "SimpleCrypter" - __version__ = "0.10" - __type__ = "crypter" +class SimpleCrypter(Crypter, SimpleHoster): + __name__ = "SimpleCrypter" + __type__ = "crypter" + __version__ = "0.42" - __pattern__ = None + __pattern__ = r'^unmatchable$' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] __description__ = """Simple decrypter plugin""" - __author_name__ = ("stickell", "zoidberg", "Walter Purcaro") - __author_mail__ = ("l.stickell@yahoo.it", "zoidberg@mujmail.cz", "vuolter@gmail.com") + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it"), + ("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] + """ Following patterns should be defined by each crypter: - LINK_PATTERN: group(1) must be a download link or a regex to catch more links - example: LINK_PATTERN = r'<div class="link"><a href="(http://speedload.org/\w+)' + LINK_PATTERN: Download link or regex to catch links in group(1) + example: LINK_PATTERN = r'<div class="link"><a href="(.+?)"' - TITLE_PATTERN: (optional) The group defined by 'title' should be the title - example: TITLE_PATTERN = r'<title>Files of: (?P<title>[^<]+) folder</title>' + NAME_PATTERN: (optional) folder name or page title + example: NAME_PATTERN = r'<title>Files of: (?P<N>[^<]+) folder</title>' - OFFLINE_PATTERN: (optional) Checks if the file is yet available online + OFFLINE_PATTERN: (optional) Checks if the page is unreachable example: OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN: (optional) Checks if the file is temporarily offline + TEMP_OFFLINE_PATTERN: (optional) Checks if the page is temporarily unreachable example: TEMP_OFFLINE_PATTERN = r'Server maintainance' - If it's impossible to extract the links using the LINK_PATTERN only you can override the getLinks method. + You can override the getLinks method if you need a more sophisticated way to extract the links. + + + If the links are splitted on multiple pages you can define the PAGES_PATTERN regex: - If the links are disposed on multiple pages you need to define a pattern: + PAGES_PATTERN: (optional) group(1) should be the number of overall pages containing the links + example: PAGES_PATTERN = r'Pages: (\d+)' - PAGES_PATTERN: The group defined by 'pages' must be the total number of pages - example: PAGES_PATTERN = r'Pages: (?P<pages>\d+)' + and its loadPage method: - and a function: - loadPage(self, page_n): - return the html of the page number 'page_n' + def loadPage(self, page_n): + return the html of the page number page_n """ - URL_REPLACEMENTS = [] + LINK_PATTERN = None - SH_COOKIES = True # or False or list of tuples [(domain, name, value)] + NAME_REPLACEMENTS = [("&#?\w+;", fixup)] + URL_REPLACEMENTS = [] + TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + COOKIES = True #: or False or list of tuples [(domain, name, value)] - def setup(self): - if isinstance(self.SH_COOKIES, list): - set_cookies(self.req.cj, self.SH_COOKIES) + LOGIN_ACCOUNT = False + LOGIN_PREMIUM = False - def decrypt(self, pyfile): - pyfile.url = replace_patterns(pyfile.url, self.URL_REPLACEMENTS) - self.html = self.load(pyfile.url, decode=True) + #@TODO: Remove in 0.4.10 + def init(self): + account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") + account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name) + + if account and account.canUse(): + self.user, data = account.selectAccount() + self.req = account.getAccountRequest(self.user) + self.premium = account.isPremium(self.user) + + self.account = account + + + def prepare(self): + self.pyfile.error = "" #@TODO: Remove in 0.4.10 + + self.info = {} + self.html = "" + self.links = [] #@TODO: Move to hoster class in 0.4.10 + + if self.LOGIN_PREMIUM and not self.premium: + self.fail(_("Required premium account not found")) + + if self.LOGIN_ACCOUNT and not self.account: + self.fail(_("Required account not found")) + + self.req.setOption("timeout", 120) + + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) + + self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - self.checkOnline() - package_name, folder_name = self.getPackageNameAndFolder() + def decrypt(self, pyfile): + self.prepare() + + self.preload() + self.checkInfo() - self.package_links = self.getLinks() + self.links = self.getLinks() if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): - self.handleMultiPages() + self.handlePages(pyfile) + + self.logDebug("Package has %d links" % len(self.links)) + + if self.links: + self.packages = [(self.info['name'], self.links, self.info['folder'])] + + elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 + self.fail(_("No link grabbed")) + + + def checkNameSize(self, getinfo=True): + if not self.info or getinfo: + self.logDebug("File info (BEFORE): %s" % self.info) + self.info.update(self.getInfo(self.pyfile.url, self.html)) + self.logDebug("File info (AFTER): %s" % self.info) + + try: + url = self.info['url'] + name = self.info['name'] + if name and name != url: + self.pyfile.name = name - self.logDebug('Package has %d links' % len(self.package_links)) + except Exception: + pass + + try: + folder = self.info['folder'] = self.pyfile.name + + except Exception: + pass + + self.logDebug("File name: %s" % self.pyfile.name, + "File folder: %s" % self.pyfile.name) - if self.package_links: - self.packages = [(package_name, self.package_links, folder_name)] - else: - self.fail('Could not extract any links') def getLinks(self): """ Returns the links extracted from self.html You should override this only if it's impossible to extract links using only the LINK_PATTERN. """ - return re.findall(self.LINK_PATTERN, self.html) - - def checkOnline(self): - if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, self.html): - self.offline() - elif hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, self.html): - self.tempOffline() - - def getPackageNameAndFolder(self): - if hasattr(self, 'TITLE_PATTERN'): - m = re.search(self.TITLE_PATTERN, self.html) - if m: - name = folder = html_unescape(m.group('title').strip()) - self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) - return name, folder - - name = self.pyfile.package().name - folder = self.pyfile.package().folder - self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) - return name, folder - - def handleMultiPages(self): - pages = re.search(self.PAGES_PATTERN, self.html) - if pages: - pages = int(pages.group('pages')) - else: + url_p = urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + + return [urljoin(baseurl, link) if not urlparse(link).scheme else link \ + for link in re.findall(self.LINK_PATTERN, self.html)] + + + def handlePages(self, pyfile): + try: + pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) + except Exception: pages = 1 for p in xrange(2, pages + 1): self.html = self.loadPage(p) - self.package_links += self.getLinks() - - def parseError(self, msg): - raise PluginParseError(msg) + self.links += self.getLinks() diff --git a/module/plugins/internal/SimpleDereferer.py b/module/plugins/internal/SimpleDereferer.py new file mode 100644 index 000000000..bd00f5d25 --- /dev/null +++ b/module/plugins/internal/SimpleDereferer.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +import re + +from urllib import unquote + +from module.plugins.Crypter import Crypter +from module.plugins.internal.SimpleHoster import fileUrl, set_cookies + + +class SimpleDereferer(Crypter): + __name__ = "SimpleDereferer" + __type__ = "crypter" + __version__ = "0.07" + + __pattern__ = r'^unmatchable$' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Simple dereferer plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + """ + Following patterns should be defined by each crypter: + + LINK_PATTERN: Regex to catch the redirect url in group(1) + example: LINK_PATTERN = r'<div class="link"><a href="(.+?)"' + + OFFLINE_PATTERN: (optional) Checks if the page is unreachable + example: OFFLINE_PATTERN = r'File (deleted|not found)' + + TEMP_OFFLINE_PATTERN: (optional) Checks if the page is temporarily unreachable + example: TEMP_OFFLINE_PATTERN = r'Server maintainance' + + + You can override the getLinks method if you need a more sophisticated way to extract the redirect url. + """ + + LINK_PATTERN = None + + TEXT_ENCODING = False + COOKIES = True + + + def decrypt(self, pyfile): + link = fileUrl(self, pyfile.url) + + if not link: + try: + link = unquote(re.match(self.__pattern__, pyfile.url).group('LINK')) + + except Exception: + self.prepare() + self.preload() + self.checkStatus() + + link = self.getLink() + + if link.strip(): + self.urls = [link.strip()] #@TODO: Remove `.strip()` in 0.4.10 + + elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 + self.fail(_("No link grabbed")) + + + def prepare(self): + self.info = {} + self.html = "" + + self.req.setOption("timeout", 120) + + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) + + + def preload(self): + self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING) + + if isinstance(self.TEXT_ENCODING, basestring): + self.html = unicode(self.html, self.TEXT_ENCODING) + + + def checkStatus(self): + if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, self.html): + self.offline() + + elif hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, self.html): + self.tempOffline() + + + def getLink(self): + try: + return re.search(self.LINK_PATTERN, self.html).group(1) + + except Exception: + pass diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 0dfecf0fb..c74e33d59 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -1,34 +1,55 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. -""" -from urlparse import urlparse +import mimetypes +import os import re + +from datetime import datetime, timedelta +from inspect import isclass from time import time +from urllib import unquote +from urlparse import urljoin, urlparse -from module.plugins.Hoster import Hoster -from module.utils import html_unescape, fixup, parseFileSize -from module.network.RequestFactory import getURL +from module.PyFile import statusMap as _statusMap from module.network.CookieJar import CookieJar +from module.network.HTTPRequest import BadHeader +from module.network.RequestFactory import getURL +from module.plugins.Hoster import Hoster +from module.plugins.Plugin import Fail +from module.utils import fixup, fs_encode, parseFileSize + + +#@TODO: Adapt and move to PyFile in 0.4.10 +statusMap = dict((v, k) for k, v in _statusMap.iteritems()) + + +#@TODO: Remove in 0.4.10 and redirect to self.error instead +def _error(self, reason, type): + if not reason and not type: + type = "unknown" + + msg = _("%s error") % type.strip().capitalize() if type else _("Error") + msg += ": %s" % reason.strip() if reason else "" + msg += _(" | Plugin may be out of date") + + raise Fail(msg) + + +#@TODO: Remove in 0.4.10 +def _wait(self, seconds, reconnect): + if seconds: + self.setWait(int(seconds) + 1) + + if reconnect is not None: + self.wantReconnect = reconnect + + super(SimpleHoster, self).wait() def replace_patterns(string, ruleslist): for r in ruleslist: rf, rt = r string = re.sub(rf, rt, string) - #self.logDebug(rf, rt, string) return string @@ -44,12 +65,13 @@ def parseHtmlTagAttrValue(attr_name, tag): return m.group(2) if m else None -def parseHtmlForm(attr_str, html, input_names=None): - for form in re.finditer(r"(?P<tag><form[^>]*%s[^>]*>)(?P<content>.*?)</?(form|body|html)[^>]*>" % attr_str, +def parseHtmlForm(attr_str, html, input_names={}): + for form in re.finditer(r"(?P<TAG><form[^>]*%s[^>]*>)(?P<CONTENT>.*?)</?(form|body|html)[^>]*>" % attr_str, html, re.S | re.I): inputs = {} - action = parseHtmlTagAttrValue("action", form.group('tag')) - for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('content'), re.S | re.I): + action = parseHtmlTagAttrValue("action", form.group('TAG')) + + for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.S | re.I): name = parseHtmlTagAttrValue("name", inputtag.group(1)) if name: value = parseHtmlTagAttrValue("value", inputtag.group(1)) @@ -58,9 +80,9 @@ def parseHtmlForm(attr_str, html, input_names=None): else: inputs[name] = value - if isinstance(input_names, dict): + if input_names: # check input attributes - for key, val in input_names.items(): + for key, val in input_names.iteritems(): if key in inputs: if isinstance(val, basestring) and inputs[key] == val: continue @@ -68,238 +90,615 @@ def parseHtmlForm(attr_str, html, input_names=None): continue elif hasattr(val, "search") and re.match(val, inputs[key]): continue - break # attibute value does not match + break #: attibute value does not match else: - break # attibute name does not match + break #: attibute name does not match else: - return action, inputs # passed attribute check + return action, inputs #: passed attribute check else: # no attribute check return action, inputs - return {}, None # no matching form found + return {}, None #: no matching form found + + +#: Deprecated +def parseFileInfo(plugin, url="", html=""): + if hasattr(plugin, "getInfo"): + info = plugin.getInfo(url, html) + res = info['name'], info['size'], info['status'], info['url'] + else: + url = unquote(url) + res = ((urlparse(url).path.split('/')[-1] + or urlparse(url).query.split('=', 1)[::-1][0].split('&', 1)[0] + or _("Unknown")), + 0, + 3 if url else 8, + url) + + return res + +#@TODO: Remove in 0.4.10 +#@NOTE: Every plugin must have own parseInfos classmethod to work with 0.4.10 +def create_getInfo(plugin): + + def generator(list): + for x in list: + yield x + + if hasattr(plugin, "parseInfos"): + fn = lambda urls: generator((info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfos(urls)) + else: + fn = lambda urls: generator(parseFileInfo(url) for url in urls) -def parseFileInfo(self, url='', html=''): - info = {"name": url, "size": 0, "status": 3} + return fn - if hasattr(self, "pyfile"): - url = self.pyfile.url - if hasattr(self, "req") and self.req.http.code == '404': - info['status'] = 1 +def timestamp(): + return int(time() * 1000) + + +#@TODO: Move to hoster class in 0.4.10 +def fileUrl(self, url, follow_location=None): + link = "" + redirect = 1 + + if type(follow_location) is int: + redirect = max(follow_location, 1) else: - if not html and hasattr(self, "html"): - html = self.html - if isinstance(self.SH_BROKEN_ENCODING, (str, unicode)): - html = unicode(html, self.SH_BROKEN_ENCODING) - if hasattr(self, "html"): - self.html = html - - if (hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html)) or \ - (hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html)): - # File offline - info['status'] = 1 - else: - online = False - try: - info.update(re.match(self.__pattern__, url).groupdict()) - except: - pass + redirect = 5 - for pattern in ("FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): - try: - info.update(re.search(getattr(self, pattern), html).groupdict()) - online = True - except AttributeError: + for i in xrange(redirect): + try: + self.logDebug("Redirect #%d to: %s" % (i, url)) + header = self.load(url, ref=True, cookies=True, just_header=True, decode=True) + + except Exception: #: Bad bad bad... + req = pyreq.getHTTPRequest() + res = req.load(url, cookies=True, just_header=True, decode=True) + + req.close() + + header = {"code": req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: continue - if online: - # File online, return name and size - info['status'] = 2 - if 'N' in info: - info['name'] = replace_patterns(info['N'], self.FILE_NAME_REPLACEMENTS) - if 'S' in info: - size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], - self.FILE_SIZE_REPLACEMENTS) - info['size'] = parseFileSize(size) - elif isinstance(info['size'], (str, unicode)): - if 'units' in info: - info['size'] += info['units'] - info['size'] = parseFileSize(info['size']) + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() - if hasattr(self, "file_info"): - self.file_info = info + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value - return info['name'], info['size'], info['status'], url + if 'content-disposition' in header: + link = url + elif 'location' in header and header['location'].strip(): + location = header['location'] -def create_getInfo(plugin): + if not urlparse(location).scheme: + url_p = urlparse(url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + location = urljoin(baseurl, location) - def getInfo(urls): - for url in urls: - cj = CookieJar(plugin.__name__) - if isinstance(plugin.SH_COOKIES, list): - set_cookies(cj, plugin.SH_COOKIES) - file_info = parseFileInfo(plugin, url, getURL(replace_patterns(url, plugin.FILE_URL_REPLACEMENTS), - decode=not plugin.SH_BROKEN_ENCODING, cookies=cj)) - yield file_info + if 'code' in header and header['code'] == 302: + link = location - return getInfo + if follow_location: + url = location + continue + else: + extension = os.path.splitext(urlparse(url).path.split('/')[-1])[-1] -def timestamp(): - return int(time() * 1000) + if 'content-type' in header and header['content-type'].strip(): + mimetype = header['content-type'].split(';')[0].strip() + + elif extension: + mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" + else: + mimetype = "" + + if mimetype and (link or 'html' not in mimetype): + link = url + else: + link = "" + + break + + else: + try: + self.logError(_("Too many redirects")) + except Exception: + pass + + return link + + +def secondsToMidnight(gmt=0): + now = datetime.utcnow() + timedelta(hours=gmt) + + if now.hour is 0 and now.minute < 10: + midnight = now + else: + midnight = now + timedelta(days=1) -class PluginParseError(Exception): + td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now - def __init__(self, msg): - Exception.__init__(self) - self.value = 'Parse error (%s) - plugin may be out of date' % msg + if hasattr(td, 'total_seconds'): + res = td.total_seconds() + else: #@NOTE: work-around for python 2.5 and 2.6 missing timedelta.total_seconds + res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 - def __str__(self): - return repr(self.value) + return int(res) class SimpleHoster(Hoster): - __name__ = "SimpleHoster" - __version__ = "0.34" - __type__ = "hoster" + __name__ = "SimpleHoster" + __type__ = "hoster" + __version__ = "1.12" - __pattern__ = None + __pattern__ = r'^unmatchable$' __description__ = """Simple hoster plugin""" - __author_name__ = ("zoidberg", "stickell") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] + """ - Following patterns should be defined by each hoster: + Info patterns should be defined by each hoster: - FILE_INFO_PATTERN: Name and Size of the file - example: FILE_INFO_PATTERN = r'(?P<N>file_name) (?P<S>file_size) (?P<U>size_unit)' + INFO_PATTERN: (optional) Name and Size of the file + example: INFO_PATTERN = r'(?P<N>file_name) (?P<S>file_size) (?P<U>size_unit)' or - FILE_NAME_PATTERN: Name that will be set for the file - example: FILE_NAME_PATTERN = r'(?P<N>file_name)' - FILE_SIZE_PATTERN: Size that will be checked for the file - example: FILE_SIZE_PATTERN = r'(?P<S>file_size) (?P<U>size_unit)' + NAME_PATTERN: (optional) Name that will be set for the file + example: NAME_PATTERN = r'(?P<N>file_name)' + SIZE_PATTERN: (optional) Size that will be checked for the file + example: SIZE_PATTERN = r'(?P<S>file_size) (?P<U>size_unit)' + + HASHSUM_PATTERN: (optional) Hash code and type of the file + example: HASHSUM_PATTERN = r'(?P<H>hash_code) (?P<T>MD5)' - OFFLINE_PATTERN: Checks if the file is yet available online + OFFLINE_PATTERN: (optional) Check if the page is unreachable example: OFFLINE_PATTERN = r'File (deleted|not found)' - or: - FILE_OFFLINE_PATTERN: Deprecated - TEMP_OFFLINE_PATTERN: Checks if the file is temporarily offline - example: TEMP_OFFLINE_PATTERN = r'Server maintainance' + TEMP_OFFLINE_PATTERN: (optional) Check if the page is temporarily unreachable + example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' + + + Error handling patterns are all optional: + + WAIT_PATTERN: (optional) Detect waiting time + example: WAIT_PATTERN = r'' - PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account + PREMIUM_ONLY_PATTERN: (optional) Check if the file can be downloaded only with a premium account example: PREMIUM_ONLY_PATTERN = r'Premium account required' + + ERROR_PATTERN: (optional) Detect any error preventing download + example: ERROR_PATTERN = r'' + + + Instead overriding handleFree and handlePremium methods you can define the following patterns for direct download: + + LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download + example: LINK_FREE_PATTERN = r'<div class="link"><a href="(.+?)"' + + LINK_PREMIUM_PATTERN: (optional) group(1) should be the direct link for premium download + example: LINK_PREMIUM_PATTERN = r'<div class="link"><a href="(.+?)"' """ - FILE_NAME_REPLACEMENTS = [("&#?\w+;", fixup)] - FILE_SIZE_REPLACEMENTS = [] - FILE_URL_REPLACEMENTS = [] + NAME_REPLACEMENTS = [("&#?\w+;", fixup)] + SIZE_REPLACEMENTS = [] + URL_REPLACEMENTS = [] - SH_BROKEN_ENCODING = False # Set to True or encoding name if encoding in http header is not correct - SH_COOKIES = True # or False or list of tuples [(domain, name, value)] - SH_CHECK_TRAFFIC = False # True = force check traffic left for a premium account + TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct + COOKIES = True #: or False or list of tuples [(domain, name, value)] + CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + DIRECT_LINK = None #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False + MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) + LOGIN_ACCOUNT = False #: Set to True to require account login + directLink = fileUrl #@TODO: Remove in 0.4.10 + + + @classmethod + def parseInfos(cls, urls): #@TODO: Built-in in 0.4.10 core, then remove from plugins + for url in urls: + url = replace_patterns(url, cls.FILE_URL_REPLACEMENTS if hasattr(cls, "FILE_URL_REPLACEMENTS") else cls.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 + yield cls.getInfo(url) + + + @classmethod + def apiInfo(cls, url="", get={}, post={}): + url = unquote(url) + return {'name' : (urlparse(url).path.split('/')[-1] + or urlparse(url).query.split('=', 1)[::-1][0].split('&', 1)[0] + or _("Unknown")), + 'size' : 0, + 'status': 3 if url else 8, + 'url' : url} + + + @classmethod + def getInfo(cls, url="", html=""): + info = cls.apiInfo(url) + online = False if info['status'] != 2 else True + + try: + info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here + + except Exception: + info['pattern'] = {} + + if not html and not online: + if not url: + info['error'] = "missing url" + info['status'] = 1 + + elif info['status'] is 3 and not fileUrl(None, url): + try: + html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING) + + if isinstance(cls.TEXT_ENCODING, basestring): + html = unicode(html, cls.TEXT_ENCODING) + + except BadHeader, e: + info['error'] = "%d: %s" % (e.code, e.content) + + if e.code is 404: + info['status'] = 1 + + elif e.code is 503: + info['status'] = 6 + + if html: + if hasattr(cls, "OFFLINE_PATTERN") and re.search(cls.OFFLINE_PATTERN, html): + info['status'] = 1 + + elif hasattr(cls, "FILE_OFFLINE_PATTERN") and re.search(cls.FILE_OFFLINE_PATTERN, html): #@TODO: Remove in 0.4.10 + info['status'] = 1 + + elif hasattr(cls, "TEMP_OFFLINE_PATTERN") and re.search(cls.TEMP_OFFLINE_PATTERN, html): + info['status'] = 6 + + else: + for pattern in ("FILE_INFO_PATTERN", "INFO_PATTERN", + "FILE_NAME_PATTERN", "NAME_PATTERN", + "FILE_SIZE_PATTERN", "SIZE_PATTERN", + "HASHSUM_PATTERN"): #@TODO: Remove old patterns starting with "FILE_" in 0.4.10 + try: + attr = getattr(cls, pattern) + pdict = re.search(attr, html).groupdict() + + if all(True for k in pdict if k not in info['pattern']): + info['pattern'].update(pdict) + + except AttributeError: + continue + + else: + online = True + + if online: + info['status'] = 2 + + if 'N' in info['pattern']: + info['name'] = replace_patterns(unquote(info['pattern']['N'].strip()), + cls.FILE_NAME_REPLACEMENTS if hasattr(cls, "FILE_NAME_REPLACEMENTS") else cls.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 + + if 'S' in info['pattern']: + size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info['pattern'] else info['pattern']['S'], + cls.FILE_SIZE_REPLACEMENTS if hasattr(cls, "FILE_SIZE_REPLACEMENTS") else cls.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 + info['size'] = parseFileSize(size) + + elif isinstance(info['size'], basestring): + unit = info['units'] if 'units' in info else None + info['size'] = parseFileSize(info['size'], unit) + + if 'H' in info['pattern']: + hashtype = info['pattern']['T'] if 'T' in info['pattern'] else "hash" + info[hashtype] = info['pattern']['H'] + + if not info['pattern']: + info.pop('pattern', None) + + return info - def init(self): - self.file_info = {} def setup(self): self.resumeDownload = self.multiDL = self.premium - if isinstance(self.SH_COOKIES, list): - set_cookies(self.req.cj, self.SH_COOKIES) - def process(self, pyfile): - pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) + + def prepare(self): + self.pyfile.error = "" #@TODO: Remove in 0.4.10 + + self.info = {} + self.html = "" + self.link = "" #@TODO: Move to hoster class in 0.4.10 + self.directDL = False #@TODO: Move to hoster class in 0.4.10 + self.multihost = False #@TODO: Move to hoster class in 0.4.10 + + if self.LOGIN_ACCOUNT and not self.account: + self.fail(_("Required account not found")) + self.req.setOption("timeout", 120) - # Due to a 0.4.9 core bug self.load would keep previous cookies even if overridden by cookies parameter. - # Workaround using getURL. Can be reverted in 0.5 as the cookies bug has been fixed. - self.html = getURL(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) - premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only: # Usually premium only pages doesn't show the file information - self.getFileInfo() - - if self.premium and (not self.SH_CHECK_TRAFFIC or self.checkTrafficLeft()): - self.handlePremium() - elif premium_only: - self.fail("This link require a premium account") + + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) + + if (self.MULTI_HOSTER + and (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] + or re.match(self.__pattern__, self.pyfile.url) is None)): + self.multihost = True + return + + if self.DIRECT_LINK is None: + self.directDL = bool(self.account) + else: + self.directDL = self.DIRECT_LINK + + self.pyfile.url = replace_patterns(self.pyfile.url, + self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 + + + def preload(self): + self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING) + + if isinstance(self.TEXT_ENCODING, basestring): + self.html = unicode(self.html, self.TEXT_ENCODING) + + + def process(self, pyfile): + self.prepare() + self.checkInfo() + + if self.directDL: + self.logDebug("Looking for direct download link...") + self.handleDirect(pyfile) + + if self.multihost and not self.link and not self.lastDownload: + self.logDebug("Looking for leeched download link...") + self.handleMulti(pyfile) + + if not self.link and not self.lastDownload: + self.MULTI_HOSTER = False + self.retry(1, reason="Multi hoster fails") + + if not self.link and not self.lastDownload: + self.preload() + self.checkInfo() + + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as premium download") + self.handlePremium(pyfile) + + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as free download") + self.handleFree(pyfile) + + self.downloadLink(self.link) + self.checkFile() + + + def downloadLink(self, link, disposition=False): #@TODO: Set `disposition=True` in 0.4.10 + if link and isinstance(link, basestring): + self.correctCaptcha() + + if not urlparse(link).scheme: + url_p = urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + link = urljoin(baseurl, link) + + self.download(link, ref=False, disposition=disposition) + + + def checkFile(self): + if self.cTask and not self.lastDownload: + self.invalidCaptcha() + self.retry(10, reason=_("Wrong captcha")) + + elif not self.lastDownload or not os.path.exists(fs_encode(self.lastDownload)): + self.lastDownload = "" + self.error(self.pyfile.error or _("No file downloaded")) + else: - # This line is required due to the getURL workaround. Can be removed in 0.5 - self.html = self.load(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) - self.handleFree() + rules = {'empty file': re.compile(r'\A\Z'), + 'html file' : re.compile(r'\A\s*<!DOCTYPE html'), + 'html error': re.compile(r'\A\s*(<.+>)?\d{3}(\Z|\s+)')} + + if hasattr(self, 'ERROR_PATTERN'): + rules['error'] = re.compile(self.ERROR_PATTERN) + + check = self.checkDownload(rules) + if check: #@TODO: Move to hoster in 0.4.10 + errmsg = check.strip().capitalize() + if self.lastCheck: + errmsg += " | " + self.lastCheck.group(0).strip() - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): - if type(url) == unicode: - url = url.encode('utf8') - return Hoster.load(self, url=url, get=get, post=post, ref=ref, cookies=cookies, - just_header=just_header, decode=decode) + self.lastDownload = "" + self.retry(10, 60, errmsg) + + def checkErrors(self): + if not self.html: + self.logWarning(_("No html code to check")) + return + + if hasattr(self, 'PREMIUM_ONLY_PATTERN') and not self.premium and re.search(self.PREMIUM_ONLY_PATTERN, self.html): + self.fail(_("Link require a premium account to be handled")) + + elif hasattr(self, 'ERROR_PATTERN'): + m = re.search(self.ERROR_PATTERN, self.html) + if m: + errmsg = self.info['error'] = m.group(1) + self.error(errmsg) + + elif hasattr(self, 'WAIT_PATTERN'): + m = re.search(self.WAIT_PATTERN, self.html) + if m: + wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I)]) + self.wait(wait_time, wait_time > 300) + return + + self.info.pop('error', None) + + + def checkStatus(self, getinfo=True): + if not self.info or getinfo: + self.logDebug("File info (BEFORE): %s" % self.info) + self.info.update(self.getInfo(self.pyfile.url, self.html)) + + try: + status = self.info['status'] + + if status is 1: + self.offline() + + elif status is 6: + self.tempOffline() + + elif status is 8: + self.fail() + + finally: + self.logDebug("File status: %s" % statusMap[status], + "File info: %s" % self.info) + + + def checkNameSize(self, getinfo=True): + if not self.info or getinfo: + self.logDebug("File info (BEFORE): %s" % self.info) + self.info.update(self.getInfo(self.pyfile.url, self.html)) + self.logDebug("File info (AFTER): %s" % self.info) + + try: + url = self.info['url'] + name = self.info['name'] + if name and name != url: + self.pyfile.name = name + + except Exception: + pass + + try: + size = self.info['size'] + if size > 0: + self.pyfile.size = size + + except Exception: + pass + + self.logDebug("File name: %s" % self.pyfile.name, + "File size: %s" % self.pyfile.size if self.pyfile.size > 0 else "Unknown") + + + def checkInfo(self): + self.checkNameSize() + + if self.html: + self.checkErrors() + self.checkNameSize() + + self.checkStatus(getinfo=False) + + + #: Deprecated def getFileInfo(self): - self.logDebug("URL: %s" % self.pyfile.url) - if hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, self.html): - self.tempOffline() + self.info = {} + self.checkInfo() + return self.info + - name, size, status = parseFileInfo(self)[:3] + def handleDirect(self, pyfile): + link = self.directLink(pyfile.url, self.resumeDownload) - if status == 1: - self.offline() - elif status != 2: - self.logDebug(self.file_info) - self.parseError('File info') + if link: + self.logInfo(_("Direct download link detected")) - if name: - self.pyfile.name = name + self.link = link else: - self.pyfile.name = html_unescape(urlparse(self.pyfile.url).path.split("/")[-1]) + self.logDebug("Direct download link not found") - if size: - self.pyfile.size = size + + def handleMulti(self, pyfile): #: Multi-hoster handler + pass + + + def handleFree(self, pyfile): + if not hasattr(self, 'LINK_FREE_PATTERN'): + self.logError(_("Free download not implemented")) + + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m is None: + self.error(_("Free download link not found")) else: - self.logError("File size not parsed") + self.link = m.group(1) - self.logDebug("FILE NAME: %s FILE SIZE: %s" % (self.pyfile.name, self.pyfile.size)) - return self.file_info - def handleFree(self): - self.fail("Free download not implemented") + def handlePremium(self, pyfile): + if not hasattr(self, 'LINK_PREMIUM_PATTERN'): + self.logError(_("Premium download not implemented")) + self.logDebug("Handled as free download") + self.handleFree(pyfile) - def handlePremium(self): - self.fail("Premium download not implemented") + m = re.search(self.LINK_PREMIUM_PATTERN, self.html) + if m is None: + self.error(_("Premium download link not found")) + else: + self.link = m.group(1) - def parseError(self, msg): - raise PluginParseError(msg) def longWait(self, wait_time=None, max_tries=3): if wait_time and isinstance(wait_time, (int, long, float)): - time_str = "%dh %dm" % divmod(wait_time / 60, 60) + time_str = "%dh %dm" % divmod(wait_time / 60, 60) else: wait_time = 900 - time_str = "(unknown time)" + time_str = _("(unknown time)") max_tries = 100 - self.logInfo("Download limit reached, reconnect or wait %s" % time_str) + self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str) self.setWait(wait_time, True) self.wait() - self.retry(max_tries=max_tries, reason="Download limit reached") + self.retry(max_tries=max_tries, reason=_("Download limit reached")) - def parseHtmlForm(self, attr_str='', input_names=None): + + def parseHtmlForm(self, attr_str="", input_names={}): return parseHtmlForm(attr_str, self.html, input_names) + def checkTrafficLeft(self): + if not self.account: + return True + traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] - if traffic == -1: + + if traffic is None: + return False + elif traffic == -1: return True - size = self.pyfile.size / 1024 - self.logInfo("Filesize: %i KiB, Traffic left for user %s: %i KiB" % (size, self.user, traffic)) - return size <= traffic - - # TODO: Remove in 0.5 - def wait(self, seconds=False, reconnect=False): - if seconds: - self.setWait(seconds, reconnect) - super(SimpleHoster, self).wait() + else: + size = self.pyfile.size / 1024 + self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) + return size <= traffic + + + #@TODO: Remove in 0.4.10 + def wait(self, seconds=0, reconnect=None): + return _wait(self, seconds, reconnect) + + + def error(self, reason="", type="parse"): + return _error(self, reason, type) diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 0251620ab..7f1b08caf 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -1,135 +1,115 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. -""" - import os import re -from os.path import join + from glob import glob -from subprocess import Popen, PIPE from string import digits +from subprocess import Popen, PIPE +from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError from module.utils import save_join, decode -from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError -class UnRar(AbtractExtractor): - __name__ = "UnRar" - __version__ = "0.16" +def renice(pid, value): + if value and os.name != "nt": + try: + Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) + except Exception: + pass + + +class UnRar(Extractor): + __name__ = "UnRar" + __version__ = "1.04" __description__ = """Rar extractor plugin""" - __author_name__ = "RaNaN" - __author_mail__ = "RaNaN@pyload.org" - - # there are some more uncovered rar formats - re_version = re.compile(r"(UNRAR 5[\.\d]+(.*?)freeware)") - re_splitfile = re.compile(r"(.*)\.part(\d+)\.rar$", re.I) - re_partfiles = re.compile(r".*\.(rar|r[0-9]+)", re.I) - re_filelist = re.compile(r"(.+)\s+(\d+)\s+(\d+)\s+") - re_filelist5 = re.compile(r"(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)") - re_wrongpwd = re.compile("(Corrupt file or wrong password|password incorrect)", re.I) + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + CMD = "unrar" + EXTENSIONS = [".rar", ".zip", ".cab", ".arj", ".lzh", ".tar", ".gz", ".bz2", + ".ace", ".uue", ".jar", ".iso", ".7z", ".xz", ".z"] - @staticmethod - def checkDeps(): + #@NOTE: there are some more uncovered rar formats + re_rarpart1 = re.compile(r'\.part(\d+)\.rar$', re.I) + re_rarpart2 = re.compile(r'\.r(\d+)$', re.I) + + re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') + re_wrongpwd = re.compile(r'password', re.I) + re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) + + + @classmethod + def checkDeps(cls): if os.name == "nt": - UnRar.CMD = join(pypath, "UnRAR.exe") - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + cls.CMD = os.path.join(pypath, "UnRAR.exe") + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() else: try: - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() - except OSError: - # fallback to rar - UnRar.CMD = "rar" - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + except OSError: #: fallback to rar + cls.CMD = "rar" + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() return True - @staticmethod - def getTargets(files_ids): - result = [] - for file, id in files_ids: - if not file.endswith(".rar"): + @classmethod + def getTargets(cls, files_ids): + targets = [] + + for filename, id in files_ids: + if not cls.isArchive(filename): continue - match = UnRar.re_splitfile.findall(file) - if match: - # only add first parts - if int(match[0][1]) == 1: - result.append((file, id)) - else: - result.append((file, id)) + m = cls.re_rarpart1.match(filename) + if not m or int(m.group(1)) is 1: #@NOTE: only add first part file + targets.append((filename, id)) - return result + return targets - def init(self): - self.passwordProtected = False - self.headerProtected = False #: list files will not work without password - self.smallestFile = None #: small file to test passwords - self.password = "" #: save the correct password def checkArchive(self): - p = self.call_unrar("l", "-v", self.file) + p = self.call_cmd("l", "-v", self.target) out, err = p.communicate() + if self.re_wrongpwd.search(err): - self.passwordProtected = True - self.headerProtected = True return True # output only used to check if passworded files are present - if self.re_version.search(out): - for attr, size, name in self.re_filelist5.findall(out): - if attr.startswith("*"): - self.passwordProtected = True - return True - else: - for name, size, packed in self.re_filelist.findall(out): - if name.startswith("*"): - self.passwordProtected = True - return True + for attr in self.re_filelist.findall(out): + if attr[0].startswith("*"): + return True - self.listContent() + self.files = self.list() if not self.files: raise ArchiveError("Empty Archive") return False + def checkPassword(self, password): # at this point we can only verify header protected files - if self.headerProtected: - p = self.call_unrar("l", "-v", self.file, password=password) - out, err = p.communicate() - if self.re_wrongpwd.search(err): - return False + p = self.call_cmd("l", "-v", self.target, password=password) + out, err = p.communicate() + return False if self.re_wrongpwd.search(err) else True - return True - def extract(self, progress, password=None): + def extract(self, password=None): command = "x" if self.fullpath else "e" - p = self.call_unrar(command, self.file, self.out, password=password) + p = self.call_cmd(command, self.target, self.out, password=password) + renice(p.pid, self.renice) - progress(0) progressstring = "" while True: c = p.stdout.read(1) @@ -138,66 +118,86 @@ class UnRar(AbtractExtractor): break # reading a percentage sign -> set progress and restart if c == '%': - progress(int(progressstring)) + self.notifyProgress(int(progressstring)) progressstring = "" # not reading a digit -> therefore restart elif c not in digits: progressstring = "" # add digit to progressstring else: - progressstring = progressstring + c - progress(100) + progressstring += c # retrieve stderr err = p.stderr.read() - if "CRC failed" in err and not password and not self.passwordProtected: + if self.re_wrongpwd.search(err): + raise PasswordError + + elif self.re_wrongcrc.search(err): raise CRCError - elif "CRC failed" in err: - raise WrongPassword - if err.strip(): #: raise error if anything is on stderr + + elif err.strip(): #: raise error if anything is on stderr raise ArchiveError(err.strip()) - if p.returncode: + + if p.returncode != 0: raise ArchiveError("Process terminated") if not self.files: - self.password = password - self.listContent() + self.files = self.list(password) + def getDeleteFiles(self): - if ".part" in self.file: - return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.IGNORECASE)) - # get files which matches .r* and filter unsuited files out - parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.IGNORECASE)) - return filter(lambda x: self.re_partfiles.match(x), parts) + files = [] + + for i in [1, 2]: + try: + dir, name = os.path.split(self.target) + part = self.getattr(self, "re_rarpart%d" % i).match(name).group(1) + filename = os.path.join(dir, name.replace(part, '*', 1)) + files.extend(glob(filename)) - def listContent(self): + except Exception: + continue + + if self.target not in files: + files.insert(0, self.target) + + return files + + + def list(self, password=None): command = "vb" if self.fullpath else "lb" - p = self.call_unrar(command, "-v", self.file, password=self.password) + + p = self.call_cmd(command, "-v", self.target, password=password) out, err = p.communicate() if "Cannot open" in err: raise ArchiveError("Cannot open file") if err.strip(): #: only log error at this point - self.m.logError(err.strip()) + self.manager.logError(err.strip()) result = set() - for f in decode(out).splitlines(): f = f.strip() result.add(save_join(self.out, f)) - self.files = result + return list(result) + - def call_unrar(self, command, *xargs, **kwargs): + def call_cmd(self, command, *xargs, **kwargs): args = [] + # overwrite flag - args.append("-o+") if self.overwrite else args.append("-o-") + if self.overwrite: + args.append("-o+") + else: + args.append("-o-") + if self.delete: + args.append("-or") - if self.excludefiles: - for word in self.excludefiles.split(';'): - args.append("-x%s" % word) + for word in self.excludefiles: + args.append("-x%s" % word.strip()) # assume yes on all queries args.append("-y") @@ -208,18 +208,13 @@ class UnRar(AbtractExtractor): else: args.append("-p-") + if self.keepbroken: + args.append("-kb") + # NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.m.logDebug(" ".join(call)) - p = Popen(call, stdout=PIPE, stderr=PIPE) + self.manager.logDebug(" ".join(call)) + p = Popen(call, stdout=PIPE, stderr=PIPE) return p - - -def renice(pid, value): - if os.name != "nt" and value: - try: - Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) - except: - print "Renice failed" diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index bace76789..026503be5 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -1,53 +1,52 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. +from __future__ import with_statement - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. -""" - -import zipfile +import os import sys +import zipfile -from module.plugins.internal.AbstractExtractor import AbtractExtractor +from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -class UnZip(AbtractExtractor): - __name__ = "UnZip" - __version__ = "0.1" +class UnZip(Extractor): + __name__ = "UnZip" + __version__ = "1.03" __description__ = """Zip extractor plugin""" - __author_name__ = "RaNaN" - __author_mail__ = "RaNaN@pyload.org" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] - @staticmethod - def checkDeps(): + EXTENSIONS = [".zip", ".zip64"] + + + @classmethod + def checkDeps(cls): return sys.version_info[:2] >= (2, 6) - @staticmethod - def getTargets(files_ids): - result = [] - for file, id in files_ids: - if file.endswith(".zip"): - result.append((file, id)) + @classmethod + def getTargets(cls, files_ids): + return [(filename, id) for filename, id in files_ids if cls.isArchive(filename)] + - return result + def extract(self, password=None): + try: + with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z: + z.setpassword(self.password) + if not z.testzip(): + z.extractall(self.out) + self.files = z.namelist() + else: + raise CRCError - def extract(self, progress, password=None): - z = zipfile.ZipFile(self.file) - self.files = z.namelist() - z.extractall(self.out) + except (BadZipfile, LargeZipFile), e: + raise ArchiveError(e) - def getDeleteFiles(self): - return [self.file] + except RuntimeError, e: + if "encrypted" in e: + raise PasswordError + else: + raise ArchiveError(e) diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py new file mode 100644 index 000000000..c3bd91e52 --- /dev/null +++ b/module/plugins/internal/XFSAccount.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- + +import re + +from time import gmtime, mktime, strptime +from urlparse import urljoin + +from module.plugins.Account import Account +from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies + + +class XFSAccount(Account): + __name__ = "XFSAccount" + __type__ = "account" + __version__ = "0.35" + + __description__ = """XFileSharing account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = None + HOSTER_URL = None + + COOKIES = True + + PREMIUM_PATTERN = r'\(Premium only\)' + + VALID_UNTIL_PATTERN = r'Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})' + + TRAFFIC_LEFT_PATTERN = r'Traffic available today:.*?<b>\s*(?P<S>[\d.,]+|[Uu]nlimited)\s*(?:(?P<U>[\w^_]+)\s*)?</b>' + TRAFFIC_LEFT_UNIT = "MB" #: used only if no group <U> was found + + LEECH_TRAFFIC_PATTERN = r'Leech Traffic left:<b>.*?(?P<S>[\d.,]+|[Uu]nlimited)\s*(?:(?P<U>[\w^_]+)\s*)?</b>' + LEECH_TRAFFIC_UNIT = "MB" #: used only if no group <U> was found + + LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<' + + + def __init__(self, manager, accounts): #@TODO: remove in 0.4.10 + self.init() + return super(XFSAccount, self).__init__(manager, accounts) + + + def init(self): + if not self.HOSTER_DOMAIN: + self.logError(_("Missing HOSTER_DOMAIN")) + self.COOKIES = False + + else: + if not self.HOSTER_URL: + self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN + + if isinstance(self.COOKIES, list): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + set_cookies(req.cj, self.COOKIES) + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = None + leechtraffic = None + premium = None + + if not self.HOSTER_URL: #@TODO: Remove in 0.4.10 + return {'validuntil' : validuntil, + 'trafficleft' : trafficleft, + 'leechtraffic': leechtraffic, + 'premium' : premium} + + html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) + + premium = True if re.search(self.PREMIUM_PATTERN, html) else False + + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: + expiredate = m.group(1).strip() + self.logDebug("Expire date: " + expiredate) + + try: + validuntil = mktime(strptime(expiredate, "%d %B %Y")) + + except Exception, e: + self.logError(e) + + else: + self.logDebug("Valid until: %s" % validuntil) + + if validuntil > mktime(gmtime()): + premium = True + trafficleft = -1 + else: + premium = False + validuntil = None #: registered account type (not premium) + else: + self.logDebug("VALID_UNTIL_PATTERN not found") + + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + try: + traffic = m.groupdict() + size = traffic['S'] + + if "nlimited" in size: + trafficleft = -1 + if validuntil is None: + validuntil = -1 + else: + if 'U' in traffic: + unit = traffic['U'] + elif isinstance(self.TRAFFIC_LEFT_UNIT, basestring): + unit = self.TRAFFIC_LEFT_UNIT + else: + unit = "" + + trafficleft = self.parseTraffic(size + unit) + + except Exception, e: + self.logError(e) + else: + self.logDebug("TRAFFIC_LEFT_PATTERN not found") + + leech = [m.groupdict() for m in re.finditer(self.LEECH_TRAFFIC_PATTERN, html)] + if leech: + leechtraffic = 0 + try: + for traffic in leech: + size = traffic['S'] + + if "nlimited" in size: + leechtraffic = -1 + if validuntil is None: + validuntil = -1 + break + else: + if 'U' in traffic: + unit = traffic['U'] + elif isinstance(self.LEECH_TRAFFIC_UNIT, basestring): + unit = self.LEECH_TRAFFIC_UNIT + else: + unit = "" + + leechtraffic += self.parseTraffic(size + unit) + + except Exception, e: + self.logError(e) + else: + self.logDebug("LEECH_TRAFFIC_PATTERN not found") + + return {'validuntil' : validuntil, + 'trafficleft' : trafficleft, + 'leechtraffic': leechtraffic, + 'premium' : premium} + + + def login(self, user, data, req): + if not self.HOSTER_URL: #@TODO: Remove in 0.4.10 + raise Exception(_("Missing HOSTER_DOMAIN")) + + url = urljoin(self.HOSTER_URL, "login.html") + html = req.load(url, decode=True) + + action, inputs = parseHtmlForm('name="FL"', html) + if not inputs: + inputs = {'op' : "login", + 'redirect': self.HOSTER_URL} + + inputs.update({'login' : user, + 'password': data['password']}) + + html = req.load(self.HOSTER_URL, post=inputs, decode=True) + + if re.search(self.LOGIN_FAIL_PATTERN, html): + self.wrongPassword() diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py new file mode 100644 index 000000000..665e13b18 --- /dev/null +++ b/module/plugins/internal/XFSCrypter.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo + + +class XFSCrypter(SimpleCrypter): + __name__ = "XFSCrypter" + __type__ = "crypter" + __version__ = "0.06" + + __pattern__ = r'^unmatchable$' + + __description__ = """XFileSharing decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = None + + URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')] + + LINK_PATTERN = r'<(?:td|TD).*?>\s*<a href="(.+?)".*?>.+?(?:</a>)?\s*</(?:td|TD)>' + NAME_PATTERN = r'<[tT]itle>.*?\: (?P<N>.+) folder</[tT]itle>' + + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' + + + def prepare(self): + if not self.HOSTER_DOMAIN: + if self.account: + account = self.account + else: + account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") + account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name) + + if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: + self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + else: + self.fail(_("Missing HOSTER_DOMAIN")) + + if isinstance(self.COOKIES, list): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + + return super(XFSCrypter, self).prepare() diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py new file mode 100644 index 000000000..2054b1824 --- /dev/null +++ b/module/plugins/internal/XFSHoster.py @@ -0,0 +1,350 @@ +# -*- coding: utf-8 -*- + +import re + +from random import random +from time import sleep +from urlparse import urljoin, urlparse + +from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME + +from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, secondsToMidnight +from module.utils import html_unescape + + +class XFSHoster(SimpleHoster): + __name__ = "XFSHoster" + __type__ = "hoster" + __version__ = "0.41" + + __pattern__ = r'^unmatchable$' + + __description__ = """XFileSharing hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = None + + TEXT_ENCODING = False + DIRECT_LINK = None + MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... + + NAME_PATTERN = r'(Filename[ ]*:[ ]*</b>(</td><td nowrap>)?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P<N>.+?)(\s*<|")' + SIZE_PATTERN = r'(Size[ ]*:[ ]*</b>(</td><td>)?|File:.*>|</font>\s*\(|<[\w^_]+ class="size">)\s*(?P<S>[\d.,]+)\s*(?P<U>[\w^_]+)' + + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' + + WAIT_PATTERN = r'<span id="countdown_str">.*?>(\d+)</span>|id="countdown" value=".*?(\d+).*?"' + PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' + ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter><b>|>Error</td>|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))' + + LINK_LEECH_PATTERN = r'<h2>Download Link</h2>\s*<textarea[^>]*>([^<]+)' + LINK_PATTERN = None #: final download url pattern + + CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' + CAPTCHA_BLOCK_PATTERN = r'>Enter code.*?<div.*?>(.+?)</div>' + RECAPTCHA_PATTERN = None + SOLVEMEDIA_PATTERN = None + + FORM_PATTERN = None + FORM_INPUTS_MAP = None #: dict passed as input_names to parseHtmlForm + + + def setup(self): + self.chunkLimit = -1 if self.premium else 1 + self.resumeDownload = self.multiDL = self.premium + + + def prepare(self): + """ Initialize important variables """ + if not self.HOSTER_DOMAIN: + if self.account: + account = self.account + else: + account = self.pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + + if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: + self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + else: + self.fail(_("Missing HOSTER_DOMAIN")) + + if isinstance(self.COOKIES, list): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + + if not self.LINK_PATTERN: + pattern = r'(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' + self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') + + self.captcha = None + self.errmsg = None + + super(XFSHoster, self).prepare() + + if self.DIRECT_LINK is None: + self.directDL = self.premium + + + def downloadLink(self, link, disposition=False): #@TODO: Set `disposition=True` in 0.4.10 + if link and isinstance(link, basestring): + self.correctCaptcha() + + if not urlparse(link).scheme: + url_p = urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + link = urljoin(baseurl, link) + + self.download(link, ref=False, disposition=disposition) + + elif self.errmsg: + if 'captcha' in self.errmsg: + self.fail(_("No valid captcha code entered")) + else: + self.fail(self.errmsg) + + else: + self.fail(_("Download link not found")) + + + def handleFree(self, pyfile): + for i in xrange(1, 6): + self.logDebug("Getting download link: #%d" % i) + + self.checkErrors() + + m = re.search(self.LINK_PATTERN, self.html, re.S) + if m: + break + + data = self.getPostParameters() + + self.req.http.c.setopt(FOLLOWLOCATION, 0) + + self.html = self.load(pyfile.url, post=data, ref=True, decode=True) + + self.req.http.c.setopt(FOLLOWLOCATION, 1) + + m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) + if m and not "op=" in m.group(1): + break + + m = re.search(self.LINK_PATTERN, self.html, re.S) + if m: + break + else: + self.logError(data['op'] if 'op' in data else _("UNKNOWN")) + return "" + + self.errmsg = None + + self.link = m.group(1).strip() #@TODO: Remove .strip() in 0.4.10 + + + def handlePremium(self, pyfile): + return self.handleFree(pyfile) + + + def handleMulti(self, pyfile): + if not self.account: + self.fail(_("Only registered or premium users can use url leech feature")) + + #only tested with easybytez.com + self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) + + action, inputs = self.parseHtmlForm() + + upload_id = "%012d" % int(random() * 10 ** 12) + action += upload_id + "&js_on=1&utype=prem&upload_type=url" + + inputs['tos'] = '1' + inputs['url_mass'] = pyfile.url + inputs['up1oad_type'] = 'url' + + self.logDebug(action, inputs) + + self.req.setOption("timeout", 600) #: wait for file to upload to easybytez.com + + self.html = self.load(action, post=inputs) + + self.checkErrors() + + action, inputs = self.parseHtmlForm('F1') + if not inputs: + if self.errmsg: + self.retry(reason=self.errmsg) + else: + self.error(_("TEXTAREA F1 not found")) + + self.logDebug(inputs) + + stmsg = inputs['st'] + + if stmsg == 'OK': + self.html = self.load(action, post=inputs) + + elif 'Can not leech file' in stmsg: + self.retry(20, 3 * 60, _("Can not leech file")) + + elif 'today' in stmsg: + self.retry(wait_time=secondsToMidnight(gmt=2), reason=_("You've used all Leech traffic today")) + + else: + self.fail(stmsg) + + #get easybytez.com link for uploaded file + m = re.search(self.LINK_LEECH_PATTERN, self.html) + if m is None: + self.error(_("LINK_LEECH_PATTERN not found")) + + header = self.load(m.group(1), just_header=True, decode=True) + + if 'location' in header: #: Direct download link + self.link = header['location'] + else: + self.fail(_("Download link not found")) + + + def checkErrors(self): + m = re.search(self.ERROR_PATTERN, self.html) + if m is None: + self.errmsg = None + else: + self.errmsg = m.group(1).strip() + + self.logWarning(re.sub(r"<.*?>", " ", self.errmsg)) + + if 'wait' in self.errmsg: + wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec)', self.errmsg, re.I)]) + self.wait(wait_time, True) + + elif 'country' in self.errmsg: + self.fail(_("Downloads are disabled for your country")) + + elif 'captcha' in self.errmsg: + self.invalidCaptcha() + + elif 'premium' in self.errmsg and 'require' in self.errmsg: + self.fail(_("File can be downloaded by premium users only")) + + elif 'limit' in self.errmsg: + if 'days' in self.errmsg: + delay = secondsToMidnight(gmt=2) + retries = 3 + else: + delay = 1 * 60 * 60 + retries = 24 + + self.wantReconnect = True + self.retry(retries, delay, _("Download limit exceeded")) + + elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: + self.retry(reason=_("Link expired")) + + elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: + self.tempOffline() + + elif 'up to' in self.errmsg: + self.fail(_("File too large for free download")) + + else: + self.wantReconnect = True + self.retry(wait_time=60, reason=self.errmsg) + + if self.errmsg: + self.info['error'] = self.errmsg + else: + self.info.pop('error', None) + + + def getPostParameters(self): + if self.FORM_PATTERN or self.FORM_INPUTS_MAP: + action, inputs = self.parseHtmlForm(self.FORM_PATTERN or "", self.FORM_INPUTS_MAP or {}) + else: + action, inputs = self.parseHtmlForm(input_names={'op': re.compile(r'^download')}) + + if not inputs: + action, inputs = self.parseHtmlForm('F1') + if not inputs: + if self.errmsg: + self.retry(reason=self.errmsg) + else: + self.error(_("TEXTAREA F1 not found")) + + self.logDebug(inputs) + + if 'op' in inputs: + if "password" in inputs: + password = self.getPassword() + if password: + inputs['password'] = password + else: + self.fail(_("Missing password")) + + if not self.premium: + m = re.search(self.WAIT_PATTERN, self.html) + if m: + wait_time = int(m.group(1)) + self.setWait(wait_time, False) + + self.captcha = self.handleCaptcha(inputs) + + self.wait() + else: + inputs['referer'] = self.pyfile.url + + if self.premium: + inputs['method_premium'] = "Premium Download" + inputs.pop('method_free', None) + else: + inputs['method_free'] = "Free Download" + inputs.pop('method_premium', None) + + return inputs + + + def handleCaptcha(self, inputs): + m = re.search(self.CAPTCHA_PATTERN, self.html) + if m: + captcha_url = m.group(1) + inputs['code'] = self.decryptCaptcha(captcha_url) + return 1 + + m = re.search(self.CAPTCHA_BLOCK_PATTERN, self.html, re.S) + if m: + captcha_div = m.group(1) + numerals = re.findall(r'<span.*?padding-left\s*:\s*(\d+).*?>(\d)</span>', html_unescape(captcha_div)) + self.logDebug(captcha_div) + inputs['code'] = "".join([a[1] for a in sorted(numerals, key=lambda num: int(num[0]))]) + self.logDebug("Captcha code: %s" % inputs['code'], numerals) + return 2 + + recaptcha = ReCaptcha(self) + try: + captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) + except Exception: + captcha_key = recaptcha.detect_key() + else: + self.logDebug("ReCaptcha key: %s" % captcha_key) + + if captcha_key: + inputs['recaptcha_response_field'], inputs['recaptcha_challenge_field'] = recaptcha.challenge(captcha_key) + return 3 + + solvemedia = SolveMedia(self) + try: + captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) + except Exception: + captcha_key = solvemedia.detect_key() + else: + self.logDebug("SolveMedia key: %s" % captcha_key) + + if captcha_key: + inputs['adcopy_response'], inputs['adcopy_challenge'] = solvemedia.challenge(captcha_key) + return 4 + + return 0 diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py deleted file mode 100644 index a1cba9af3..000000000 --- a/module/plugins/internal/XFSPAccount.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. -""" - -import re -from time import mktime, strptime -from module.plugins.Account import Account -from module.plugins.internal.SimpleHoster import parseHtmlForm -from module.utils import parseFileSize - - -class XFSPAccount(Account): - __name__ = "XFSPAccount" - __version__ = "0.06" - __type__ = "account" - - __description__ = """XFileSharingPro base account plugin""" - __author_name__ = "zoidberg" - __author_mail__ = "zoidberg@mujmail.cz" - - MAIN_PAGE = None - - VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:</TD><TD><b>([^<]+)</b>' - TRAFFIC_LEFT_PATTERN = r'>Traffic available today:</TD><TD><b>([^<]+)</b>' - LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|>Error<' - PREMIUM_PATTERN = r'>Renew premium<' - - - def loadAccountInfo(self, user, req): - html = req.load(self.MAIN_PAGE + "?op=my_account", decode=True) - - validuntil = trafficleft = None - premium = True if re.search(self.PREMIUM_PATTERN, html) else False - - m = re.search(self.VALID_UNTIL_PATTERN, html) - if m: - premium = True - trafficleft = -1 - try: - self.logDebug(m.group(1)) - validuntil = mktime(strptime(m.group(1), "%d %B %Y")) - except Exception, e: - self.logError(e) - else: - m = re.search(self.TRAFFIC_LEFT_PATTERN, html) - if m: - trafficleft = m.group(1) - if "Unlimited" in trafficleft: - premium = True - else: - trafficleft = parseFileSize(trafficleft) / 1024 - - return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} - - def login(self, user, data, req): - html = req.load('%slogin.html' % self.MAIN_PAGE, decode=True) - - action, inputs = parseHtmlForm('name="FL"', html) - if not inputs: - inputs = {"op": "login", - "redirect": self.MAIN_PAGE} - - inputs.update({"login": user, - "password": data['password']}) - - html = req.load(self.MAIN_PAGE, post=inputs, decode=True) - - if re.search(self.LOGIN_FAIL_PATTERN, html): - self.wrongPassword() |