From 8e7d14bae4d3c836f029a1235eb227380acc3f75 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 16 Feb 2015 21:59:10 +0100 Subject: Fix plugins to work on 0.4.10 --- module/plugins/internal/BasePlugin.py | 95 ---- module/plugins/internal/CaptchaService.py | 522 --------------------- module/plugins/internal/DeadCrypter.py | 31 -- module/plugins/internal/DeadHoster.py | 31 -- module/plugins/internal/Extractor.py | 139 ------ module/plugins/internal/MultiHook.py | 308 ------------- module/plugins/internal/MultiHoster.py | 85 ---- module/plugins/internal/SevenZip.py | 155 ------- module/plugins/internal/SimpleCrypter.py | 157 ------- module/plugins/internal/SimpleDereferer.py | 98 ---- module/plugins/internal/SimpleHoster.py | 701 ----------------------------- module/plugins/internal/UnRar.py | 248 ---------- module/plugins/internal/UnZip.py | 68 --- module/plugins/internal/UpdateManager.py | 300 ------------ module/plugins/internal/XFSAccount.py | 174 ------- module/plugins/internal/XFSCrypter.py | 45 -- module/plugins/internal/XFSHoster.py | 326 -------------- module/plugins/internal/__init__.py | 1 - 18 files changed, 3484 deletions(-) delete mode 100644 module/plugins/internal/BasePlugin.py delete mode 100644 module/plugins/internal/CaptchaService.py delete mode 100644 module/plugins/internal/DeadCrypter.py delete mode 100644 module/plugins/internal/DeadHoster.py delete mode 100644 module/plugins/internal/Extractor.py delete mode 100644 module/plugins/internal/MultiHook.py delete mode 100644 module/plugins/internal/MultiHoster.py delete mode 100644 module/plugins/internal/SevenZip.py delete mode 100644 module/plugins/internal/SimpleCrypter.py delete mode 100644 module/plugins/internal/SimpleDereferer.py delete mode 100644 module/plugins/internal/SimpleHoster.py delete mode 100644 module/plugins/internal/UnRar.py delete mode 100644 module/plugins/internal/UnZip.py delete mode 100644 module/plugins/internal/UpdateManager.py delete mode 100644 module/plugins/internal/XFSAccount.py delete mode 100644 module/plugins/internal/XFSCrypter.py delete mode 100644 module/plugins/internal/XFSHoster.py delete mode 100644 module/plugins/internal/__init__.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/BasePlugin.py b/module/plugins/internal/BasePlugin.py deleted file mode 100644 index 792497449..000000000 --- a/module/plugins/internal/BasePlugin.py +++ /dev/null @@ -1,95 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from urllib import unquote -from urlparse import urljoin, urlparse - -from module.network.HTTPRequest import BadHeader -from module.plugins.internal.SimpleHoster import create_getInfo, fileUrl -from module.plugins.Hoster import Hoster - - -class BasePlugin(Hoster): - __name__ = "BasePlugin" - __type__ = "hoster" - __version__ = "0.34" - - __pattern__ = r'^unmatchable$' - - __description__ = """Base plugin when any other didnt fit""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com")] - - - @classmethod - def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10 - url = unquote(url) - url_p = urlparse(url) - return {'name' : (url_p.path.split('/')[-1] - or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] - or url_p.netloc.split('.', 1)[0]), - 'size' : 0, - 'status': 3 if url else 8, - 'url' : url} - - - def setup(self): - self.chunkLimit = -1 - self.multiDL = True - self.resumeDownload = True - - - def process(self, pyfile): - """main function""" - - pyfile.name = self.getInfo(pyfile.url)['name'] - - if not pyfile.url.startswith("http"): - self.fail(_("No plugin matched")) - - for _i in xrange(5): - try: - link = fileUrl(self, unquote(pyfile.url)) - - if link: - self.download(link, ref=False, disposition=True) - else: - self.fail(_("File not found")) - - except BadHeader, e: - if e.code is 404: - self.offline() - - elif e.code in (401, 403): - self.logDebug("Auth required", "Received HTTP status code: %d" % e.code) - - account = self.core.accountManager.getAccountPlugin('Http') - servers = [x['login'] for x in account.getAllAccounts()] - server = urlparse(pyfile.url).netloc - - if server in servers: - self.logDebug("Logging on to %s" % server) - self.req.addAuth(account.getAccountData(server)['password']) - else: - pwd = self.getPassword() - if ':' in pwd: - self.req.addAuth(pwd) - else: - self.fail(_("Authorization required")) - else: - self.fail(e) - else: - break - else: - self.fail(_("No file downloaded")) #@TODO: Move to hoster class in 0.4.10 - - check = self.checkDownload({'empty file': re.compile(r'\A\Z'), - 'html file' : re.compile(r'\A\s*)?\d{3}(\Z|\s+)')}) - if check: - self.fail(check.capitalize()) - - -getInfo = create_getInfo(BasePlugin) diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py deleted file mode 100644 index 6f2c8e06d..000000000 --- a/module/plugins/internal/CaptchaService.py +++ /dev/null @@ -1,522 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -import time - -from base64 import b64encode -from random import random, randint -from urlparse import urljoin, urlparse - -from module.common.json_layer import json_loads -from module.plugins.Plugin import Base - - -#@TODO: Extend (new) Plugin class; remove all `html` args -class CaptchaService(Base): - __name__ = "CaptchaService" - __version__ = "0.25" - - __description__ = """Base captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] - - - key = None #: last key detected - - - def __init__(self, plugin): - self.plugin = plugin - super(CaptchaService, self).__init__(plugin.core) - - - def detect_key(self, html=None): - raise NotImplementedError - - - def challenge(self, key=None, html=None): - raise NotImplementedError - - - def result(self, server, challenge): - raise NotImplementedError - - -class ReCaptcha(CaptchaService): - __name__ = "ReCaptcha" - __version__ = "0.14" - - __description__ = """ReCaptcha captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com"), - ("zapp-brannigan", "fuerst.reinje@web.de")] - - - KEY_V2_PATTERN = r'(?:data-sitekey=["\']|["\']sitekey["\']:\s*["\'])([\w-]+)' - KEY_V1_PATTERN = r'(?:recaptcha(?:/api|\.net)/(?:challenge|noscript)\?k=|Recaptcha\.create\s*\(\s*["\'])([\w-]+)' - - - def detect_key(self, html=None): - if not html: - if hasattr(self.plugin, "html") and self.plugin.html: - html = self.plugin.html - else: - errmsg = _("ReCaptcha html not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - m = re.search(self.KEY_V2_PATTERN, html) or re.search(self.KEY_V1_PATTERN, html) - if m: - self.key = m.group(1).strip() - self.logDebug("Key: %s" % self.key) - return self.key - else: - self.logDebug("Key not found") - return None - - - def challenge(self, key=None, html=None, version=None): - if not key: - if self.detect_key(html): - key = self.key - else: - errmsg = _("ReCaptcha key not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - if version in (1, 2): - return getattr(self, "_challenge_v%s" % version)(key) - - elif not html and hasattr(self.plugin, "html") and self.plugin.html: - version = 2 if re.search(self.KEY_V2_PATTERN, self.plugin.html) else 1 - return self.challenge(key, self.plugin.html, version) - - else: - errmsg = _("ReCaptcha html not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - - def _challenge_v1(self, key): - html = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", - get={'k': key}) - try: - challenge = re.search("challenge : '(.+?)',", html).group(1) - server = re.search("server : '(.+?)',", html).group(1) - - except AttributeError: - errmsg = _("ReCaptcha challenge pattern not found") - self.plugin.fail(errmsg) - raise AttributeError(errmsg) - - self.logDebug("Challenge: %s" % challenge) - - return self.result(server, challenge), challenge - - - def result(self, server, challenge): - result = self.plugin.decryptCaptcha("%simage" % server, - get={'c': challenge}, - cookies=True, - forceUser=True, - imgtype="jpg") - - self.logDebug("Result: %s" % result) - - return result - - - def _collectApiInfo(self): - html = self.plugin.req.load("http://www.google.com/recaptcha/api.js") - a = re.search(r'po.src = \'(.*?)\';', html).group(1) - vers = a.split("/")[5] - - self.logDebug("API version: %s" %vers) - - language = a.split("__")[1].split(".")[0] - - self.logDebug("API language: %s" % language) - - html = self.plugin.req.load("https://apis.google.com/js/api.js") - b = re.search(r'"h":"(.*?)","', html).group(1) - jsh = b.decode('unicode-escape') - - self.logDebug("API jsh-string: %s" % jsh) - - return vers, language, jsh - - - def _prepareTimeAndRpc(self): - self.plugin.req.load("http://www.google.com/recaptcha/api2/demo") - - millis = int(round(time.time() * 1000)) - - self.logDebug("Time: %s" % millis) - - rand = randint(1, 99999999) - a = "0.%s" % str(rand * 2147483647) - rpc = int(100000000 * float(a)) - - self.logDebug("Rpc-token: %s" % rpc) - - return millis, rpc - - - def _challenge_v2(self, key, parent=None): - if parent is None: - try: - parent = urljoin("http://", urlparse(self.plugin.pyfile.url).netloc) - - except Exception: - parent = "" - - botguardstring = "!A" - vers, language, jsh = self._collectApiInfo() - millis, rpc = self._prepareTimeAndRpc() - - html = self.plugin.req.load("https://www.google.com/recaptcha/api2/anchor", - get={'k' : key, - 'hl' : language, - 'v' : vers, - 'usegapi' : "1", - 'jsh' : "%s#id=IO_%s" % (jsh, millis), - 'parent' : parent, - 'pfname' : "", - 'rpctoken': rpc}) - - token1 = re.search(r'id="recaptcha-token" value="(.*?)">', html) - self.logDebug("Token #1: %s" % token1.group(1)) - - html = self.plugin.req.load("https://www.google.com/recaptcha/api2/frame", - get={'c' : token1.group(1), - 'hl' : language, - 'v' : vers, - 'bg' : botguardstring, - 'k' : key, - 'usegapi': "1", - 'jsh' : jsh}).decode('unicode-escape') - - token2 = re.search(r'"finput","(.*?)",', html) - self.logDebug("Token #2: %s" % token2.group(1)) - - token3 = re.search(r'."asconf".\s,".*?".\s,"(.*?)".', html) - self.logDebug("Token #3: %s" % token3.group(1)) - - html = self.plugin.req.load("https://www.google.com/recaptcha/api2/reload", - post={'k' : key, - 'c' : token2.group(1), - 'reason': "fi", - 'fbg' : token3.group(1)}) - - token4 = re.search(r'"rresp","(.*?)",', html) - self.logDebug("Token #4: %s" % token4.group(1)) - - millis_captcha_loading = int(round(time.time() * 1000)) - captcha_response = self.plugin.decryptCaptcha("https://www.google.com/recaptcha/api2/payload", - get={'c':token4.group(1), 'k':key}, - cookies=True, - forceUser=True) - response = b64encode('{"response":"%s"}' % captcha_response) - - self.logDebug("Result: %s" % response) - - timeToSolve = int(round(time.time() * 1000)) - millis_captcha_loading - timeToSolveMore = timeToSolve + int(float("0." + str(randint(1, 99999999))) * 500) - - html = self.plugin.req.load("https://www.google.com/recaptcha/api2/userverify", - post={'k' : key, - 'c' : token4.group(1), - 'response': response, - 't' : timeToSolve, - 'ct' : timeToSolveMore, - 'bg' : botguardstring}) - - token5 = re.search(r'"uvresp","(.*?)",', html) - self.logDebug("Token #5: %s" % token5.group(1)) - - result = token5.group(1) - - return result, None - - - -class AdsCaptcha(CaptchaService): - __name__ = "AdsCaptcha" - __version__ = "0.08" - - __description__ = """AdsCaptcha captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] - - - CAPTCHAID_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(\d+)' - PUBLICKEY_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*PublicKey=([\w-]+)' - - - def detect_key(self, html=None): - if not html: - if hasattr(self.plugin, "html") and self.plugin.html: - html = self.plugin.html - else: - errmsg = _("AdsCaptcha html not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - m = re.search(self.PUBLICKEY_PATTERN, html) - n = re.search(self.CAPTCHAID_PATTERN, html) - if m and n: - self.key = (m.group(1).strip(), n.group(1).strip()) #: key is the tuple(PublicKey, CaptchaId) - self.logDebug("Key|id: %s | %s" % self.key) - return self.key - else: - self.logDebug("Key or id not found") - return None - - - def challenge(self, key=None, html=None): - if not key: - if self.detect_key(html): - key = self.key - else: - errmsg = _("AdsCaptcha key not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - PublicKey, CaptchaId = key - - html = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", - get={'CaptchaId': CaptchaId, - 'PublicKey': PublicKey}) - try: - challenge = re.search("challenge: '(.+?)',", html).group(1) - server = re.search("server: '(.+?)',", html).group(1) - - except AttributeError: - errmsg = _("AdsCaptcha challenge pattern not found") - self.plugin.fail(errmsg) - raise AttributeError(errmsg) - - self.logDebug("Challenge: %s" % challenge) - - return self.result(server, challenge), challenge - - - def result(self, server, challenge): - result = self.plugin.decryptCaptcha("%sChallenge.aspx" % server, - get={'cid': challenge, 'dummy': random()}, - cookies=True, - imgtype="jpg") - - self.logDebug("Result: %s" % result) - - return result - - -class SolveMedia(CaptchaService): - __name__ = "SolveMedia" - __version__ = "0.12" - - __description__ = """SolveMedia captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] - - - KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']' - - - def detect_key(self, html=None): - if not html: - if hasattr(self.plugin, "html") and self.plugin.html: - html = self.plugin.html - else: - errmsg = _("SolveMedia html not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - m = re.search(self.KEY_PATTERN, html) - if m: - self.key = m.group(1).strip() - self.logDebug("Key: %s" % self.key) - return self.key - else: - self.logDebug("Key not found") - return None - - - def challenge(self, key=None, html=None): - if not key: - if self.detect_key(html): - key = self.key - else: - errmsg = _("SolveMedia key not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", - get={'k': key}) - try: - challenge = re.search(r'', - html).group(1) - server = "http://api.solvemedia.com/papi/media" - - except AttributeError: - errmsg = _("SolveMedia challenge pattern not found") - self.plugin.fail(errmsg) - raise AttributeError(errmsg) - - self.logDebug("Challenge: %s" % challenge) - - result = self.result(server, challenge) - - try: - magic = re.search(r'name="magic" value="(.+?)"', html).group(1) - - except AttributeError: - self.logDebug("Magic code not found") - - else: - if not self._verify(key, magic, result, challenge): - self.logDebug("Captcha code was invalid") - - return result, challenge - - - def _verify(self, key, magic, result, challenge, ref=None): #@TODO: Clean up - if ref is None: - try: - ref = self.plugin.pyfile.url - - except Exception: - ref = "" - - html = self.plugin.req.load("http://api.solvemedia.com/papi/verify.noscript", - post={'adcopy_response' : result, - 'k' : key, - 'l' : "en", - 't' : "img", - 's' : "standard", - 'magic' : magic, - 'adcopy_challenge' : challenge, - 'ref' : ref}) - try: - html = self.plugin.req.load(re.search(r'URL=(.+?)">', html).group(1)) - gibberish = re.search(r'id=gibberish>(.+?)', html).group(1) - - except Exception: - return False - - else: - return True - - - def result(self, server, challenge): - result = self.plugin.decryptCaptcha(server, - get={'c': challenge}, - cookies=True, - imgtype="gif") - - self.logDebug("Result: %s" % result) - - return result - - -class AdYouLike(CaptchaService): - __name__ = "AdYouLike" - __version__ = "0.05" - - __description__ = """AdYouLike captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - AYL_PATTERN = r'Adyoulike\.create\s*\((.+?)\)' - CALLBACK_PATTERN = r'(Adyoulike\.g\._jsonp_\d+)' - - - def detect_key(self, html=None): - if not html: - if hasattr(self.plugin, "html") and self.plugin.html: - html = self.plugin.html - else: - errmsg = _("AdYouLike html not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - m = re.search(self.AYL_PATTERN, html) - n = re.search(self.CALLBACK_PATTERN, html) - if m and n: - self.key = (m.group(1).strip(), n.group(1).strip()) - self.logDebug("Ayl|callback: %s | %s" % self.key) - return self.key #: key is the tuple(ayl, callback) - else: - self.logDebug("Ayl or callback not found") - return None - - - def challenge(self, key=None, html=None): - if not key: - if self.detect_key(html): - key = self.key - else: - errmsg = _("AdYouLike key not found") - self.plugin.fail(errmsg) - raise TypeError(errmsg) - - ayl, callback = key - - # {"adyoulike":{"key":"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, - # "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} - ayl = json_loads(ayl) - - html = self.plugin.req.load("http://api-ayl.appspot.com/challenge", - get={'key' : ayl['adyoulike']['key'], - 'env' : ayl['all']['env'], - 'callback': callback}) - try: - challenge = json_loads(re.search(callback + r'\s*\((.+?)\)', html).group(1)) - - except AttributeError: - errmsg = _("AdYouLike challenge pattern not found") - self.plugin.fail(errmsg) - raise AttributeError(errmsg) - - self.logDebug("Challenge: %s" % challenge) - - return self.result(ayl, challenge), challenge - - - def result(self, server, challenge): - # Adyoulike.g._jsonp_5579316662423138 - # ({"translations":{"fr":{"instructions_visual":"Recopiez « Soonnight » ci-dessous :"}}, - # "site_under":true,"clickable":true,"pixels":{"VIDEO_050":[],"DISPLAY":[],"VIDEO_000":[],"VIDEO_100":[], - # "VIDEO_025":[],"VIDEO_075":[]},"medium_type":"image/adyoulike", - # "iframes":{"big":""},"shares":{},"id":256, - # "token":"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1","formats":{"small":{"y":300,"x":0,"w":300,"h":60}, - # "big":{"y":0,"x":0,"w":300,"h":250},"hover":{"y":440,"x":0,"w":300,"h":60}}, - # "tid":"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) - - if isinstance(server, basestring): - server = json_loads(server) - - if isinstance(challenge, basestring): - challenge = json_loads(challenge) - - try: - instructions_visual = challenge['translations'][server['all']['lang']]['instructions_visual'] - result = re.search(u'«(.+?)»', instructions_visual).group(1).strip() - - except AttributeError: - errmsg = _("AdYouLike result not found") - self.plugin.fail(errmsg) - raise AttributeError(errmsg) - - result = {'_ayl_captcha_engine' : "adyoulike", - '_ayl_env' : server['all']['env'], - '_ayl_tid' : challenge['tid'], - '_ayl_token_challenge': challenge['token'], - '_ayl_response' : response} - - self.logDebug("Result: %s" % result) - - return result diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py deleted file mode 100644 index ce56947fc..000000000 --- a/module/plugins/internal/DeadCrypter.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- - -from pyload.plugin.Crypter import Crypter as _Crypter -from pyload.plugin.internal.SimpleCrypter import create_getInfo - - -class DeadCrypter(_Crypter): - __name__ = "DeadCrypter" - __type__ = "crypter" - __version__ = "0.04" - - __pattern__ = r'^unmatchable$' - - __description__ = """Crypter is no longer available""" - __license__ = "GPLv3" - __authors__ = [("stickell", "l.stickell@yahoo.it")] - - - @classmethod - def apiInfo(cls, url="", get={}, post={}): - api = super(DeadCrypter, self).apiInfo(url, get, post) - api['status'] = 1 - return api - - - def setup(self): - self.pyfile.error = "Crypter is no longer available" - self.offline() #@TODO: self.offline("Crypter is no longer available") - - -getInfo = create_getInfo(DeadCrypter) diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py deleted file mode 100644 index 132e4741a..000000000 --- a/module/plugins/internal/DeadHoster.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- - -from pyload.plugin.Hoster import Hoster as _Hoster -from pyload.plugin.internal.SimpleHoster import create_getInfo - - -class DeadHoster(_Hoster): - __name__ = "DeadHoster" - __type__ = "hoster" - __version__ = "0.14" - - __pattern__ = r'^unmatchable$' - - __description__ = """Hoster is no longer available""" - __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] - - - @classmethod - def apiInfo(cls, url="", get={}, post={}): - api = super(DeadHoster, self).apiInfo(url, get, post) - api['status'] = 1 - return api - - - def setup(self): - self.pyfile.error = "Hoster is no longer available" - self.offline() #@TODO: self.offline("Hoster is no longer available") - - -getInfo = create_getInfo(DeadHoster) diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py deleted file mode 100644 index b445f1497..000000000 --- a/module/plugins/internal/Extractor.py +++ /dev/null @@ -1,139 +0,0 @@ -# -*- coding: utf-8 -*- - -import os - -from module.PyFile import PyFile - - -class ArchiveError(Exception): - pass - - -class CRCError(Exception): - pass - - -class PasswordError(Exception): - pass - - -class Extractor: - __name__ = "Extractor" - __version__ = "0.20" - - __description__ = """Base extractor plugin""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "ranan@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com"), - ("Immenz", "immenz@gmx.net")] - - - EXTENSIONS = [] - VERSION = "" - - - @classmethod - def isArchive(cls, filename): - name = os.path.basename(filename).lower() - return any(name.endswith(ext) for ext in cls.EXTENSIONS) and not cls.isMultipart(filename) - - - @classmethod - def isMultipart(cls,filename): - return False - - - @classmethod - def isUsable(cls): - """ Check if system statisfy dependencies - :return: boolean - """ - return None - - - @classmethod - def getTargets(cls, files_ids): - """ Filter suited targets from list of filename id tuple list - :param files_ids: List of filepathes - :return: List of targets, id tuple list - """ - return [(fname, id, fout) for fname, id, fout in files_ids if cls.isArchive(fname)] - - - def __init__(self, manager, filename, out, - fullpath=True, - overwrite=False, - excludefiles=[], - renice=0, - delete=False, - keepbroken=False, - fid=None): - """ Initialize extractor for specific file """ - self.manager = manager - self.filename = filename - self.out = out - self.fullpath = fullpath - self.overwrite = overwrite - self.excludefiles = excludefiles - self.renice = renice - self.delete = delete - self.keepbroken = keepbroken - self.files = [] #: Store extracted files here - - pyfile = self.manager.core.files.getFile(fid) if fid else None - self.notifyProgress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None - - - def init(self): - """ Initialize additional data structures """ - pass - - - def check(self): - """Check if password if needed. Raise ArchiveError if integrity is - questionable. - - :return: boolean - :raises ArchiveError - """ - raise PasswordError - - - def isPassword(self, password): - """ Check if the given password is/might be correct. - If it can not be decided at this point return true. - - :param password: - :return: boolean - """ - return None - - - def repair(self): - return None - - - def extract(self, password=None): - """Extract the archive. Raise specific errors in case of failure. - - :param progress: Progress function, call this to update status - :param password password to use - :raises PasswordError - :raises CRCError - :raises ArchiveError - :return: - """ - raise NotImplementedError - - - def getDeleteFiles(self): - """Return list of files to delete, do *not* delete them here. - - :return: List with paths of files to delete - """ - return [self.filename] - - - def list(self, password=None): - """Populate self.files at some point while extracting""" - return self.files diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py deleted file mode 100644 index 652443098..000000000 --- a/module/plugins/internal/MultiHook.py +++ /dev/null @@ -1,308 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from time import sleep - -from module.plugins.Hook import Hook -from module.utils import decode, remove_chars - - -class MultiHook(Hook): - __name__ = "MultiHook" - __type__ = "hook" - __version__ = "0.37" - - __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), - ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), - ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), - ("reload" , "bool" , "Reload plugin list" , True ), - ("reloadinterval", "int" , "Reload interval in hours" , 12 )] - - __description__ = """Hook plugin for multi hoster/crypter""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com")] - - - MIN_INTERVAL = 1 * 60 * 60 - - DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"), - (r'1fichier\.com' , "onefichier.com" ), - (r'2shared\.com' , "twoshared.com" ), - (r'4shared\.com' , "fourshared.com" ), - (r'bayfiles\.net' , "bayfiles.com" ), - (r'cloudnator\.com' , "shragle.com" ), - (r'dfiles\.eu' , "depositfiles.com" ), - (r'easy-share\.com' , "crocko.com" ), - (r'freakshare\.net' , "freakshare.com" ), - (r'hellshare\.com' , "hellshare.cz" ), - (r'ifile\.it' , "filecloud.io" ), - (r'nowdownload\.\w+', "nowdownload.sx" ), - (r'nowvideo\.\w+' , "nowvideo.sx" ), - (r'putlocker\.com' , "firedrive.com" ), - (r'share-?rapid\.cz', "multishare.cz" ), - (r'ul\.to' , "uploaded.to" ), - (r'uploaded\.net' , "uploaded.to" ), - (r'uploadhero\.co' , "uploadhero.com" ), - (r'zshares\.net' , "zshare.net" ), - (r'(\d+.+)' , "X\1" )] - - - def setup(self): - self.plugins = [] - self.supported = [] - self.new_supported = [] - - self.account = None - self.pluginclass = None - self.pluginmodule = None - self.pluginname = None - self.plugintype = None - - self._initPlugin() - - - def _initPlugin(self): - plugin, type = self.core.pluginManager.findPlugin(self.__name__) - - if not plugin: - self.logWarning("Hook plugin will be deactivated due missing plugin reference") - self.setConfig('activated', False) - else: - self.pluginname = self.__name__ - self.plugintype = type - self.pluginmodule = self.core.pluginManager.loadModule(type, self.__name__) - self.pluginclass = getattr(self.pluginmodule, self.__name__) - - - def _loadAccount(self): - self.account = self.core.accountManager.getAccountPlugin(self.pluginname) - - if self.account and not self.account.canUse(): - self.account = None - - if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: - self.logWarning("Hook plugin will be deactivated due missing account reference") - self.setConfig('activated', False) - - - def coreReady(self): - self._loadAccount() - - - def getURL(self, *args, **kwargs): #@TODO: Remove in 0.4.10 - """ see HTTPRequest for argument list """ - h = pyreq.getHTTPRequest(timeout=120) - try: - if not 'decode' in kwargs: - kwargs['decode'] = True - rep = h.load(*args, **kwargs) - finally: - h.close() - - return rep - - - def getConfig(self, option, default=''): - """getConfig with default value - sublass may not implements all config options""" - try: - return self.getConf(option) - - except KeyError: - return default - - - def pluginsCached(self): - if self.plugins: - return self.plugins - - for _i in xrange(3): - try: - pluginset = self._pluginSet(self.getHosters() if self.plugintype == "hoster" else self.getCrypters()) - - except Exception, e: - self.logError(e, "Waiting 1 minute and retry") - sleep(60) - - else: - break - else: - return list() - - try: - configmode = self.getConfig("pluginmode", 'all') - if configmode in ("listed", "unlisted"): - pluginlist = self.getConfig("pluginlist", '').replace('|', ',').replace(';', ',').split(',') - configset = self._pluginSet(pluginlist) - - if configmode == "listed": - pluginset &= configset - else: - pluginset -= configset - - except Exception, e: - self.logError(e) - - self.plugins = list(pluginset) - - return self.plugins - - - def _pluginSet(self, plugins): - plugins = set((decode(x).strip().lower() for x in plugins if '.' in x)) - - for rf, rt in self.DOMAIN_REPLACEMENTS: - regex = re.compile(rf) - for p in filter(lambda x: regex.match(x), plugins): - plugins.remove(p) - plugins.add(re.sub(rf, rt, p)) - - plugins.discard('') - - return plugins - - - def getHosters(self): - """Load list of supported hoster - - :return: List of domain names - """ - raise NotImplementedError - - - def getCrypters(self): - """Load list of supported crypters - - :return: List of domain names - """ - raise NotImplementedError - - - def periodical(self): - """reload plugin list periodically""" - self.logInfo(_("Reloading supported %s list") % self.plugintype) - - old_supported = self.supported - - self.supported = [] - self.new_supported = [] - self.plugins = [] - - self.overridePlugins() - - old_supported = [plugin for plugin in old_supported if plugin not in self.supported] - - if old_supported: - self.logDebug("Unload: %s" % ", ".join(old_supported)) - for plugin in old_supported: - self.unloadPlugin(plugin) - - if self.getConfig("reload", True): - self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_INTERVAL) - else: - self.core.scheduler.removeJob(self.cb) - self.cb = None - - - def overridePlugins(self): - excludedList = [] - - if self.plugintype == "hoster": - pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) - accountList = [account.type.lower() for account in self.core.api.getAccounts(False) if account.valid and account.premium] - else: - pluginMap = {} - accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.core.pluginManager.crypterPlugins.iterkeys()] - - for plugin in self.pluginsCached(): - name = remove_chars(plugin, "-.") - - if name in accountList: - excludedList.append(plugin) - else: - if name in pluginMap: - self.supported.append(pluginMap[name]) - else: - self.new_supported.append(plugin) - - if not self.supported and not self.new_supported: - self.logError(_("No %s loaded") % self.plugintype) - return - - # inject plugin plugin - self.logDebug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) - - for plugin in self.supported: - hdict = self.core.pluginManager.plugins[self.plugintype][plugin] - hdict['new_module'] = self.pluginmodule - hdict['new_name'] = self.pluginname - - if excludedList: - self.logInfo(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList)))) - - if self.new_supported: - plugins = sorted(self.new_supported) - - self.logDebug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) - - # create new regexp - regexp = r'.*(?P%s).*' % "|".join([x.replace(".", "\.") for x in plugins]) - if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and '://' in self.pluginclass.__pattern__: - regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) - - self.logDebug("Regexp: %s" % regexp) - - hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] - hdict['pattern'] = regexp - hdict['re'] = re.compile(regexp) - - - def unloadPlugin(self, plugin): - hdict = self.core.pluginManager.plugins[self.plugintype][plugin] - if "module" in hdict: - del hdict['module'] - - if "new_module" in hdict: - del hdict['new_module'] - del hdict['new_name'] - - - def unload(self): - """Remove override for all plugins. Scheduler job is removed by hookmanager""" - for plugin in self.supported: - self.unloadPlugin(plugin) - - # reset pattern - hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] - - hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') - hdict['re'] = re.compile(hdict['pattern']) - - - def downloadFailed(self, pyfile): - """remove plugin override if download fails but not if file is offline/temp.offline""" - if pyfile.status != 8 or not self.getConfig("revertfailed", True): - return - - hdict = self.core.pluginManager.plugins[self.plugintype][pyfile.pluginname] - if "new_name" in hdict and hdict['new_name'] == self.pluginname: - if pyfile.error == "MultiHook": - self.logDebug("Unload MultiHook", pyfile.pluginname, hdict) - self.unloadPlugin(pyfile.pluginname) - pyfile.setStatus("queued") - pyfile.sync() - else: - retries = max(self.getConfig("retry", 10), 0) - wait_time = max(self.getConfig("retryinterval", 1), 0) - - if 0 < retries > pyfile.plugin.retries: - self.logInfo(_("Retrying: %s") % pyfile.name) - pyfile.setCustomStatus("MultiHook", "queued") - pyfile.sync() - - pyfile.plugin.retries += 1 - pyfile.plugin.setWait(wait_time) - pyfile.plugin.wait() diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py deleted file mode 100644 index 63b7d76b1..000000000 --- a/module/plugins/internal/MultiHoster.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies - - -class MultiHoster(SimpleHoster): - __name__ = "MultiHoster" - __type__ = "hoster" - __version__ = "0.37" - - __pattern__ = r'^unmatchable$' - - __description__ = """Multi hoster plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - LOGIN_ACCOUNT = True - - - def setup(self): - self.chunkLimit = 1 - self.multiDL = bool(self.account) - self.resumeDownload = self.premium - - - def prepare(self): - self.info = {} - self.html = "" - self.link = "" #@TODO: Move to hoster class in 0.4.10 - self.directDL = False #@TODO: Move to hoster class in 0.4.10 - - if self.LOGIN_ACCOUNT and not self.account: - self.fail(_("Required account not found")) - - self.req.setOption("timeout", 120) - - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) - - if self.DIRECT_LINK is None: - self.directDL = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) - else: - self.directDL = self.DIRECT_LINK - - self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - - - def process(self, pyfile): - self.prepare() - - if self.directDL: - self.checkInfo() - self.logDebug("Looking for direct download link...") - self.handleDirect(pyfile) - - if not self.link and not self.lastDownload: - self.preload() - - self.checkErrors() - self.checkStatus(getinfo=False) - - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as premium download") - self.handlePremium(pyfile) - - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as free download") - self.handleFree(pyfile) - - self.downloadLink(self.link, True) - self.checkFile() - - - def handlePremium(self, pyfile): - return self.handleFree(pyfile) - - - def handleFree(self, pyfile): - if self.premium: - raise NotImplementedError - else: - self.fail(_("Required premium account not found")) diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py deleted file mode 100644 index 7ad6b0d7a..000000000 --- a/module/plugins/internal/SevenZip.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import re - -from subprocess import Popen, PIPE - -from module.plugins.internal.UnRar import ArchiveError, CRCError, PasswordError, UnRar, renice -from module.utils import fs_encode, save_join - - -class SevenZip(UnRar): - __name__ = "SevenZip" - __version__ = "0.08" - - __description__ = """7-Zip extractor plugin""" - __license__ = "GPLv3" - __authors__ = [("Michael Nowak", ""), - ("Walter Purcaro", "vuolter@gmail.com")] - - - CMD = "7z" - VERSION = "" - - EXTENSIONS = [".7z", ".xz", ".zip", ".gz", ".gzip", ".tgz", ".bz2", ".bzip2", - ".tbz2", ".tbz", ".tar", ".wim", ".swm", ".lzma", ".rar", ".cab", - ".arj", ".z", ".taz", ".cpio", ".rpm", ".deb", ".lzh", ".lha", - ".chm", ".chw", ".hxs", ".iso", ".msi", ".doc", ".xls", ".ppt", - ".dmg", ".xar", ".hfs", ".exe", ".ntfs", ".fat", ".vhd", ".mbr", - ".squashfs", ".cramfs", ".scap"] - - - #@NOTE: there are some more uncovered 7z formats - re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)') - re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password)', re.I) - re_wrongcrc = re.compile(r'Encrypted\s+\=\s+\+', re.I) - re_version = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I) - - - @classmethod - def isUsable(cls): - if os.name == "nt": - cls.CMD = os.path.join(pypath, "7z.exe") - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - out,err = p.communicate() - else: - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - out, err = p.communicate() - - cls.VERSION = cls.re_version.search(out).group(1) - - return True - - - def check(self): - file = fs_encode(self.filename) - - p = self.call_cmd("t", file) - out, err = p.communicate() - - if p.returncode > 1: - raise CRCError(err) - - p = self.call_cmd("l", "-slt", file) - out, err = p.communicate() - - if p.returncode > 1: - raise ArchiveError(_("Process return code: %d") % p.returncode) - - # check if output or error macthes the 'wrong password'-Regexp - if self.re_wrongpwd.search(out): - raise PasswordError - - # check if output matches 'Encrypted = +' - if self.re_wrongcrc.search(out): - raise CRCError(_("Header protected")) - - - def isPassword(self, password): - p = self.call_cmd("l", fs_encode(self.filename), password=password) - p.communicate() - return p.returncode == 0 - - - def repair(self): - return False - - - def extract(self, password=None): - command = "x" if self.fullpath else "e" - - p = self.call_cmd(command, '-o' + self.out, fs_encode(self.filename), password=password) - - renice(p.pid, self.renice) - - # communicate and retrieve stderr - self._progress(p) - err = p.stderr.read().strip() - - if err: - if self.re_wrongpwd.search(err): - raise PasswordError - - elif self.re_wrongcrc.search(err): - raise CRCError(err) - - else: #: raise error if anything is on stderr - raise ArchiveError(err) - - if p.returncode > 1: - raise ArchiveError(_("Process return code: %d") % p.returncode) - - self.files = self.list(password) - - - def list(self, password=None): - command = "l" if self.fullpath else "l" - - p = self.call_cmd(command, fs_encode(self.filename), password=password) - out, err = p.communicate() - - if "Can not open" in err: - raise ArchiveError(_("Cannot open file")) - - if p.returncode > 1: - raise ArchiveError(_("Process return code: %d") % p.returncode) - - result = set() - for groups in self.re_filelist.findall(out): - f = groups[-1].strip() - result.add(save_join(self.out, f)) - - return list(result) - - - def call_cmd(self, command, *xargs, **kwargs): - args = [] - - #overwrite flag - if self.overwrite: - args.append("-y") - - #set a password - if "password" in kwargs and kwargs["password"]: - args.append("-p'%s'" % kwargs["password"]) - else: - args.append("-p-") - - #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue - call = [self.CMD, command] + args + list(xargs) - - self.manager.logDebug(" ".join(call)) - - p = Popen(call, stdout=PIPE, stderr=PIPE) - return p diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py deleted file mode 100644 index dc34f864f..000000000 --- a/module/plugins/internal/SimpleCrypter.py +++ /dev/null @@ -1,157 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from urlparse import urljoin, urlparse - -from pyload.plugin.Crypter import Crypter -from pyload.plugin.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies -from pyload.utils import fixup - - -class SimpleCrypter(Crypter, SimpleHoster): - __name__ = "SimpleCrypter" - __type__ = "crypter" - __version__ = "0.43" - - __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] - - __description__ = """Simple decrypter plugin""" - __license__ = "GPLv3" - __authors__ = [("stickell", "l.stickell@yahoo.it"), - ("zoidberg", "zoidberg@mujmail.cz"), - ("Walter Purcaro", "vuolter@gmail.com")] - - - """ - Following patterns should be defined by each crypter: - - LINK_PATTERN: Download link or regex to catch links in group(1) - example: LINK_PATTERN = r'
Files of: (?P[^<]+) folder' - - OFFLINE_PATTERN: (optional) Checks if the page is unreachable - example: OFFLINE_PATTERN = r'File (deleted|not found)' - - TEMP_OFFLINE_PATTERN: (optional) Checks if the page is temporarily unreachable - example: TEMP_OFFLINE_PATTERN = r'Server maintainance' - - - You can override the getLinks method if you need a more sophisticated way to extract the links. - - - If the links are splitted on multiple pages you can define the PAGES_PATTERN regex: - - PAGES_PATTERN: (optional) group(1) should be the number of overall pages containing the links - example: PAGES_PATTERN = r'Pages: (\d+)' - - and its loadPage method: - - - def loadPage(self, page_n): - return the html of the page number page_n - """ - - LINK_PATTERN = None - - NAME_REPLACEMENTS = [("&#?\w+;", fixup)] - URL_REPLACEMENTS = [] - - TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] - - LOGIN_ACCOUNT = False - LOGIN_PREMIUM = False - - - def prepare(self): - self.pyfile.error = "" #@TODO: Remove in 0.4.10 - - self.info = {} - self.html = "" - self.links = [] #@TODO: Move to hoster class in 0.4.10 - - if self.LOGIN_PREMIUM and not self.premium: - self.fail(_("Required premium account not found")) - - if self.LOGIN_ACCOUNT and not self.account: - self.fail(_("Required account not found")) - - self.req.setOption("timeout", 120) - - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) - - self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - - - def decrypt(self, pyfile): - self.prepare() - - self.preload() - self.checkInfo() - - self.links = self.getLinks() - - if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): - self.handlePages(pyfile) - - self.logDebug("Package has %d links" % len(self.links)) - - if self.links: - self.packages = [(self.info['name'], self.links, self.info['folder'])] - - elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 - self.fail(_("No link grabbed")) - - - def checkNameSize(self, getinfo=True): - if not self.info or getinfo: - self.logDebug("File info (BEFORE): %s" % self.info) - self.info.update(self.getInfo(self.pyfile.url, self.html)) - self.logDebug("File info (AFTER): %s" % self.info) - - try: - url = self.info['url'].strip() - name = self.info['name'].strip() - if name and name != url: - self.pyfile.name = name - - except Exception: - pass - - try: - folder = self.info['folder'] = self.pyfile.name - - except Exception: - pass - - self.logDebug("File name: %s" % self.pyfile.name, - "File folder: %s" % self.pyfile.name) - - - def getLinks(self): - """ - Returns the links extracted from self.html - You should override this only if it's impossible to extract links using only the LINK_PATTERN. - """ - url_p = urlparse(self.pyfile.url) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - - return [urljoin(baseurl, link) if not urlparse(link).scheme else link \ - for link in re.findall(self.LINK_PATTERN, self.html)] - - - def handlePages(self, pyfile): - try: - pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) - except Exception: - pages = 1 - - for p in xrange(2, pages + 1): - self.html = self.loadPage(p) - self.links += self.getLinks() diff --git a/module/plugins/internal/SimpleDereferer.py b/module/plugins/internal/SimpleDereferer.py deleted file mode 100644 index bd00f5d25..000000000 --- a/module/plugins/internal/SimpleDereferer.py +++ /dev/null @@ -1,98 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from urllib import unquote - -from module.plugins.Crypter import Crypter -from module.plugins.internal.SimpleHoster import fileUrl, set_cookies - - -class SimpleDereferer(Crypter): - __name__ = "SimpleDereferer" - __type__ = "crypter" - __version__ = "0.07" - - __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] - - __description__ = """Simple dereferer plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - """ - Following patterns should be defined by each crypter: - - LINK_PATTERN: Regex to catch the redirect url in group(1) - example: LINK_PATTERN = r'
\s\"'][^>\s]*)\1" % attr_name, tag, re.I) - return m.group(2) if m else None - - -def parseHtmlForm(attr_str, html, input_names={}): - for form in re.finditer(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, - html, re.S | re.I): - inputs = {} - action = parseHtmlTagAttrValue("action", form.group('TAG')) - - for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=file_name) (?Pfile_size) (?Psize_unit)' - or - NAME_PATTERN: (optional) Name that will be set for the file - example: NAME_PATTERN = r'(?Pfile_name)' - SIZE_PATTERN: (optional) Size that will be checked for the file - example: SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' - - HASHSUM_PATTERN: (optional) Hash code and type of the file - example: HASHSUM_PATTERN = r'(?Phash_code) (?PMD5)' - - OFFLINE_PATTERN: (optional) Check if the page is unreachable - example: OFFLINE_PATTERN = r'File (deleted|not found)' - - TEMP_OFFLINE_PATTERN: (optional) Check if the page is temporarily unreachable - example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' - - - Error handling patterns are all optional: - - WAIT_PATTERN: (optional) Detect waiting time - example: WAIT_PATTERN = r'' - - PREMIUM_ONLY_PATTERN: (optional) Check if the file can be downloaded only with a premium account - example: PREMIUM_ONLY_PATTERN = r'Premium account required' - - ERROR_PATTERN: (optional) Detect any error preventing download - example: ERROR_PATTERN = r'' - - - Instead overriding handleFree and handlePremium methods you can define the following patterns for direct download: - - LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download - example: LINK_FREE_PATTERN = r'
)?\d{3}(\Z|\s+)')} - - if hasattr(self, 'ERROR_PATTERN'): - rules['error'] = re.compile(self.ERROR_PATTERN) - - check = self.checkDownload(rules) - if check: #@TODO: Move to hoster in 0.4.10 - errmsg = check.strip().capitalize() - if self.lastCheck: - errmsg += " | " + self.lastCheck.group(0).strip() - - self.lastDownload = "" - self.retry(10, 60, errmsg) - - - def checkErrors(self): - if not self.html: - self.logWarning(_("No html code to check")) - return - - if hasattr(self, 'PREMIUM_ONLY_PATTERN') and not self.premium and re.search(self.PREMIUM_ONLY_PATTERN, self.html): - self.fail(_("Link require a premium account to be handled")) - - elif hasattr(self, 'ERROR_PATTERN'): - m = re.search(self.ERROR_PATTERN, self.html) - if m: - errmsg = self.info['error'] = m.group(1) - self.error(errmsg) - - elif hasattr(self, 'WAIT_PATTERN'): - m = re.search(self.WAIT_PATTERN, self.html) - if m: - wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I)) - self.wait(wait_time, wait_time > 300) - return - - self.info.pop('error', None) - - - def checkStatus(self, getinfo=True): - if not self.info or getinfo: - self.logDebug("Update file info...") - self.logDebug("Previous file info: %s" % self.info) - self.info.update(self.getInfo(self.pyfile.url, self.html)) - self.logDebug("Current file info: %s" % self.info) - - try: - status = self.info['status'] - - if status is 1: - self.offline() - - elif status is 6: - self.tempOffline() - - elif status is 8: - self.fail() - - finally: - self.logDebug("File status: %s" % statusMap[status]) - - - def checkNameSize(self, getinfo=True): - if not self.info or getinfo: - self.logDebug("Update file info...") - self.logDebug("Previous file info: %s" % self.info) - self.info.update(self.getInfo(self.pyfile.url, self.html)) - self.logDebug("Current file info: %s" % self.info) - - try: - url = self.info['url'].strip() - name = self.info['name'].strip() - if name and name != url: - self.pyfile.name = name - - except Exception: - pass - - try: - size = self.info['size'] - if size > 0: - self.pyfile.size = size - - except Exception: - pass - - self.logDebug("File name: %s" % self.pyfile.name, - "File size: %s byte" % self.pyfile.size if self.pyfile.size > 0 else "File size: Unknown") - - - def checkInfo(self): - self.checkNameSize() - - if self.html: - self.checkErrors() - self.checkNameSize() - - self.checkStatus(getinfo=False) - - - #: Deprecated - def getFileInfo(self): - self.info = {} - self.checkInfo() - return self.info - - - def handleDirect(self, pyfile): - link = self.directLink(pyfile.url, self.resumeDownload) - - if link: - self.logInfo(_("Direct download link detected")) - - self.link = link - else: - self.logDebug("Direct download link not found") - - - def handleMulti(self, pyfile): #: Multi-hoster handler - pass - - - def handleFree(self, pyfile): - if not hasattr(self, 'LINK_FREE_PATTERN'): - self.logError(_("Free download not implemented")) - - m = re.search(self.LINK_FREE_PATTERN, self.html) - if m is None: - self.error(_("Free download link not found")) - else: - self.link = m.group(1) - - - def handlePremium(self, pyfile): - if not hasattr(self, 'LINK_PREMIUM_PATTERN'): - self.logError(_("Premium download not implemented")) - self.logDebug("Handled as free download") - self.handleFree(pyfile) - - m = re.search(self.LINK_PREMIUM_PATTERN, self.html) - if m is None: - self.error(_("Premium download link not found")) - else: - self.link = m.group(1) - - - def longWait(self, wait_time=None, max_tries=3): - if wait_time and isinstance(wait_time, (int, long, float)): - time_str = "%dh %dm" % divmod(wait_time / 60, 60) - else: - wait_time = 900 - time_str = _("(unknown time)") - max_tries = 100 - - self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str) - - self.wait(wait_time, True) - self.retry(max_tries=max_tries, reason=_("Download limit reached")) - - - def parseHtmlForm(self, attr_str="", input_names={}): - return parseHtmlForm(attr_str, self.html, input_names) - - - def checkTrafficLeft(self): - if not self.account: - return True - - traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] - - if traffic is None: - return False - elif traffic == -1: - return True - else: - size = self.pyfile.size / 1024 - self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) - return size <= traffic - - - #@TODO: Remove in 0.4.10 - def wait(self, seconds=0, reconnect=None): - return _wait(self, seconds, reconnect) - - - def error(self, reason="", type="parse"): - return _error(self, reason, type) diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py deleted file mode 100644 index 54d64c430..000000000 --- a/module/plugins/internal/UnRar.py +++ /dev/null @@ -1,248 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import re - -from glob import glob -from string import digits -from subprocess import Popen, PIPE - -from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import decode, fs_encode, save_join - - -def renice(pid, value): - if value and os.name != "nt": - try: - Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) - - except Exception: - pass - - -class UnRar(Extractor): - __name__ = "UnRar" - __version__ = "1.13" - - __description__ = """Rar extractor plugin""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com"), - ("Immenz", "immenz@gmx.net"),] - - - CMD = "unrar" - VERSION = "" - - EXTENSIONS = [".rar"] - - - re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?',re.I) - - re_filefixed = re.compile(r'Building (.+)') - re_filelist = re.compile(r'^(.)(\s*[\w\.\-]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I) - - re_wrongpwd = re.compile(r'password', re.I) - re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) - - re_version = re.compile(r'UNRAR\s(\d+\.\d+)', re.I) - - - @classmethod - def isUsable(cls): - if os.name == "nt": - cls.CMD = os.path.join(pypath, "UnRAR.exe") - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - out, err = p.communicate() - else: - try: - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - out, err = p.communicate() - - except OSError: #: fallback to rar - cls.CMD = "rar" - p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) - out, err = p.communicate() - - cls.VERSION = cls.re_version.search(out).group(1) - - return True - - - @classmethod - def isMultipart(cls,filename): - multipart = cls.re_multipart.search(filename) - if multipart: - # First Multipart file (part1.rar for *.part1-9.rar format or *.rar for .r1-9 format) handled as normal Archive - return False if (multipart.group(1) == "part" and int(multipart.group(2)) == 1) else True - - return False - - - def check(self): - p = self.call_cmd("l", "-v", fs_encode(self.filename)) - out, err = p.communicate() - - if self.re_wrongpwd.search(err): - raise PasswordError - - if self.re_wrongcrc.search(err): - raise CRCError(err) - - # output only used to check if passworded files are present - for attr in self.re_filelist.findall(out): - if attr[0].startswith("*"): - raise PasswordError - - - def isPassword(self, password): - # at this point we can only verify header protected files - p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password) - out, err = p.communicate() - return False if self.re_wrongpwd.search(err) else True - - - def repair(self): - p = self.call_cmd("rc", fs_encode(self.filename)) - - # communicate and retrieve stderr - self._progress(p) - err = p.stderr.read().strip() - - if err or p.returncode: - p = self.call_cmd("r", fs_encode(self.filename)) - - # communicate and retrieve stderr - self._progress(p) - err = p.stderr.read().strip() - - if err or p.returncode: - return False - else: - dir = os.path.dirname(filename) - name = re_filefixed.search(out).group(1) - - self.filename = os.path.join(dir, name) - - return True - - - def _progress(self, process): - s = "" - while True: - c = process.stdout.read(1) - # quit loop on eof - if not c: - break - # reading a percentage sign -> set progress and restart - if c == '%': - self.notifyProgress(int(s)) - s = "" - # not reading a digit -> therefore restart - elif c not in digits: - s = "" - # add digit to progressstring - else: - s += c - - - def extract(self, password=None): - command = "x" if self.fullpath else "e" - - p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password) - - renice(p.pid, self.renice) - - # communicate and retrieve stderr - self._progress(p) - err = p.stderr.read().strip() - - if err: - if self.re_wrongpwd.search(err): - raise PasswordError - - elif self.re_wrongcrc.search(err): - raise CRCError(err) - - else: #: raise error if anything is on stderr - raise ArchiveError(err) - - if p.returncode: - raise ArchiveError(_("Process return code: %d") % p.returncode) - - self.files = self.list(password) - - - def getDeleteFiles(self): - dir, name = os.path.split(self.filename) - - # actually extracted file - files = [self.filename] - - # eventually Multipart Files - files.extend(save_join(dir, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir)) - if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file)) - - return files - - - def list(self, password=None): - command = "vb" if self.fullpath else "lb" - - p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password) - out, err = p.communicate() - - if "Cannot open" in err: - raise ArchiveError(_("Cannot open file")) - - if err.strip(): #: only log error at this point - self.manager.logError(err.strip()) - - result = set() - if not self.fullpath and self.VERSION.startswith('5'): - # NOTE: Unrar 5 always list full path - for f in decode(out).splitlines(): - f = save_join(self.out, os.path.basename(f.strip())) - if os.path.isfile(f): - result.add(save_join(self.out, os.path.basename(f))) - else: - for f in decode(out).splitlines(): - f = f.strip() - result.add(save_join(self.out, f)) - - return list(result) - - - def call_cmd(self, command, *xargs, **kwargs): - args = [] - - # overwrite flag - if self.overwrite: - args.append("-o+") - else: - args.append("-o-") - if self.delete: - args.append("-or") - - for word in self.excludefiles: - args.append("-x'%s'" % word.strip()) - - # assume yes on all queries - args.append("-y") - - # set a password - if "password" in kwargs and kwargs['password']: - args.append("-p%s" % kwargs['password']) - else: - args.append("-p-") - - if self.keepbroken: - args.append("-kb") - - # NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue - call = [self.CMD, command] + args + list(xargs) - - self.manager.logDebug(" ".join(call)) - - p = Popen(call, stdout=PIPE, stderr=PIPE) - return p diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py deleted file mode 100644 index 704b49ca0..000000000 --- a/module/plugins/internal/UnZip.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import with_statement - -import os -import sys -import zipfile - -from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import fs_encode - - -class UnZip(Extractor): - __name__ = "UnZip" - __version__ = "1.10" - - __description__ = """Zip extractor plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - EXTENSIONS = [".zip", ".zip64"] - VERSION ="(python %s.%s.%s)" % (sys.version_info[0], sys.version_info[1], sys.version_info[2]) - - - @classmethod - def isUsable(cls): - return sys.version_info[:2] >= (2, 6) - - - def list(self, password=None): - with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: - z.setpassword(password) - return z.namelist() - - - def check(self): - with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: - badfile = z.testzip() - - if badfile: - raise CRCError(badfile) - else: - raise PasswordError - - - def extract(self, password=None): - try: - with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: - z.setpassword(password) - - badfile = z.testzip() - - if badfile: - raise CRCError(badfile) - else: - z.extractall(self.out) - - except (zipfile.BadZipfile, zipfile.LargeZipFile), e: - raise ArchiveError(e) - - except RuntimeError, e: - if "encrypted" in e: - raise PasswordError - else: - raise ArchiveError(e) - else: - self.files = z.namelist() diff --git a/module/plugins/internal/UpdateManager.py b/module/plugins/internal/UpdateManager.py deleted file mode 100644 index 7bbfc0c79..000000000 --- a/module/plugins/internal/UpdateManager.py +++ /dev/null @@ -1,300 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -import sys - -from operator import itemgetter -from os import path, remove, stat - -from pyload.network.RequestFactory import getURL -from pyload.plugin.Addon import Expose, Addon, threaded -from pyload.utils import safe_join - - -class UpdateManager(Addon): - __name__ = "UpdateManager" - __type__ = "addon" - __version__ = "0.40" - - __config__ = [("activated" , "bool" , "Activated" , True ), - ("mode" , "pyLoad + plugins;plugins only", "Check updates for" , "pyLoad + plugins"), - ("interval" , "int" , "Check interval in hours" , 8 ), - ("reloadplugins", "bool" , "Monitor plugins for code changes (debug mode only)", True ), - ("nodebugupdate", "bool" , "Don't check for updates in debug mode" , True )] - - __description__ = """Check for updates""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - # event_list = ["pluginConfigChanged"] - - SERVER_URL = "http://updatemanager.pyload.org" - MIN_INTERVAL = 6 * 60 * 60 #: 6h minimum check interval (value is in seconds) - - - def pluginConfigChanged(self, plugin, name, value): - if name == "interval": - interval = value * 60 * 60 - if self.MIN_INTERVAL <= interval != self.interval: - self.core.scheduler.removeJob(self.cb) - self.interval = interval - self.initPeriodical() - else: - self.logDebug("Invalid interval value, kept current") - - elif name == "reloadplugins": - if self.cb2: - self.core.scheduler.removeJob(self.cb2) - if value is True and self.core.debug: - self.periodical2() - - - def activate(self): - self.pluginConfigChanged(self.__name__, "interval", self.getConfig("interval")) - x = lambda: self.pluginConfigChanged(self.__name__, "reloadplugins", self.getConfig("reloadplugins")) - self.core.scheduler.addJob(10, x, threaded=False) - - - def deactivate(self): - self.pluginConfigChanged(self.__name__, "reloadplugins", False) - - - def setup(self): - self.cb2 = None - self.interval = self.MIN_INTERVAL - self.updating = False - self.info = {'pyload': False, 'version': None, 'plugins': False} - self.mtimes = {} #: store modification time for each plugin - - - def periodical2(self): - if not self.updating: - self.autoreloadPlugins() - - self.cb2 = self.core.scheduler.addJob(4, self.periodical2, threaded=False) - - - @Expose - def autoreloadPlugins(self): - """ reload and reindex all modified plugins """ - modules = filter( - lambda m: m and (m.__name__.startswith("pyload.plugin.") or - m.__name__.startswith("userplugins.")) and - m.__name__.count(".") >= 2, sys.modules.itervalues() - ) - - reloads = [] - - for m in modules: - root, type, name = m.__name__.rsplit(".", 2) - id = (type, name) - if type in self.core.pluginManager.plugins: - f = m.__file__.replace(".pyc", ".py") - if not path.isfile(f): - continue - - mtime = stat(f).st_mtime - - if id not in self.mtimes: - self.mtimes[id] = mtime - elif self.mtimes[id] < mtime: - reloads.append(id) - self.mtimes[id] = mtime - - return True if self.core.pluginManager.reloadPlugins(reloads) else False - - - def periodical(self): - if self.info['pyload'] or self.getConfig("nodebugupdate") and self.core.debug: - return - - self.updateThread() - - - def server_request(self): - try: - return getURL(self.SERVER_URL, get={'v': self.core.api.getServerVersion()}).splitlines() - except Exception: - self.logWarning(_("Unable to contact server to get updates")) - - - @threaded - def updateThread(self): - self.updating = True - - status = self.update(onlyplugin=self.getConfig("mode") == "plugins only") - - if status == 2: - self.core.api.restart() - else: - self.updating = False - - - @Expose - def updatePlugins(self): - """ simple wrapper for calling plugin update quickly """ - return self.update(onlyplugin=True) - - - @Expose - def update(self, onlyplugin=False): - """ check for updates """ - data = self.server_request() - - if not data: - exitcode = 0 - - elif data[0] == "None": - self.logInfo(_("No new pyLoad version available")) - updates = data[1:] - exitcode = self._updatePlugins(updates) - - elif onlyplugin: - exitcode = 0 - - else: - newversion = data[0] - self.logInfo(_("*** New pyLoad Version %s available ***") % newversion) - self.logInfo(_("*** Get it here: https://github.com/pyload/pyload/releases ***")) - exitcode = 3 - self.info['pyload'] = True - self.info['version'] = newversion - - return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required; 3 = No plugins updated, new pyLoad version available - - - def _updatePlugins(self, updates): - """ check for plugin updates """ - - if self.info['plugins']: - return False #: plugins were already updated - - exitcode = 0 - updated = [] - - vre = re.compile(r'__version__.*=.*("|\')([\d.]+)') - url = updates[0] - schema = updates[1].split('|') - - if "BLACKLIST" in updates: - blacklist = updates[updates.index('BLACKLIST') + 1:] - updates = updates[2:updates.index('BLACKLIST')] - else: - blacklist = None - updates = updates[2:] - - upgradable = sorted(map(lambda x: dict(zip(schema, x.split('|'))), updates), - key=itemgetter("type", "name")) - - for plugin in upgradable: - filename = plugin['name'] - type = plugin['type'] - version = plugin['version'] - - if filename.endswith(".pyc"): - name = filename[:filename.find("_")] - else: - name = filename.replace(".py", "") - - plugins = getattr(self.core.pluginManager, "%sPlugins" % type) - - oldver = float(plugins[name]['version']) if name in plugins else None - newver = float(version) - - if not oldver: - msg = "New plugin: [%(type)s] %(name)s (v%(newver).2f)" - elif newver > oldver: - msg = "New version of plugin: [%(type)s] %(name)s (v%(oldver).2f -> v%(newver).2f)" - else: - continue - - self.logInfo(_(msg) % {'type' : type, - 'name' : name, - 'oldver': oldver, - 'newver': newver}) - try: - content = getURL(url % plugin) - m = vre.search(content) - - if m and m.group(2) == version: - f = open(safe_join("userplugins", prefix, filename), "wb") - f.write(content) - f.close() - updated.append((prefix, name)) - else: - raise Exception, _("Version mismatch") - - except Exception, e: - self.logError(_("Error updating plugin %s") % filename, e) - - if blacklist: - blacklisted = map(lambda x: (x.split('|')[0], x.split('|')[1].rsplit('.', 1)[0]), blacklist) - - # Always protect internal plugins from removing - for i, n, t in blacklisted.enumerate(): - if t == "internal": - del blacklisted[i] - - blacklisted = sorted(blacklisted) - removed = self.removePlugins(blacklisted) - for t, n in removed: - self.logInfo(_("Removed blacklisted plugin [%(type)s] %(name)s") % { - 'type': t, - 'name': n, - }) - - if updated: - reloaded = self.core.pluginManager.reloadPlugins(updated) - if reloaded: - self.logInfo(_("Plugins updated and reloaded")) - exitcode = 1 - else: - self.logInfo(_("*** Plugins have been updated, but need a pyLoad restart to be reloaded ***")) - self.info['plugins'] = True - exitcode = 2 - else: - self.logInfo(_("No plugin updates available")) - - return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required - - - @Expose - def removePlugins(self, type_plugins): - """ delete plugins from disk """ - - if not type_plugins: - return - - self.logDebug("Requested deletion of plugins: %s" % type_plugins) - - removed = [] - - for type, name in type_plugins: - err = False - file = name + ".py" - - for root in ("userplugins", path.join(pypath, "pyload", "plugins")): - - filename = safe_join(root, type, file) - try: - remove(filename) - except Exception, e: - self.logDebug("Error deleting: %s" % path.basename(filename), e) - err = True - - filename += "c" - if path.isfile(filename): - try: - if type == "addon": - self.manager.deactivateAddon(name) - remove(filename) - except Exception, e: - self.logDebug("Error deleting: %s" % path.basename(filename), e) - err = True - - if not err: - id = (type, name) - removed.append(id) - - return removed #: return a list of the plugins successfully removed diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py deleted file mode 100644 index 2e6b7dc50..000000000 --- a/module/plugins/internal/XFSAccount.py +++ /dev/null @@ -1,174 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from time import gmtime, mktime, strptime -from urlparse import urljoin - -from pyload.plugin.Account import Account -from pyload.plugin.internal.SimpleHoster import parseHtmlForm, set_cookies - - -class XFSAccount(Account): - __name__ = "XFSAccount" - __type__ = "account" - __version__ = "0.36" - - __description__ = """XFileSharing account plugin""" - __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("Walter Purcaro", "vuolter@gmail.com")] - - - HOSTER_DOMAIN = None - HOSTER_URL = None - LOGIN_URL = None - - COOKIES = True - - PREMIUM_PATTERN = r'\(Premium only\)' - - VALID_UNTIL_PATTERN = r'Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})' - - TRAFFIC_LEFT_PATTERN = r'Traffic available today:.*?\s*(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' - TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found - - LEECH_TRAFFIC_PATTERN = r'Leech Traffic left:.*?(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' - LEECH_TRAFFIC_UNIT = "MB" #: used only if no group was found - - LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<' - - - def init(self): - if not self.HOSTER_DOMAIN: - self.logError(_("Missing HOSTER_DOMAIN")) - self.COOKIES = False - - else: - if not self.HOSTER_URL: - self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN - - if isinstance(self.COOKIES, list): - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) - set_cookies(req.cj, self.COOKIES) - - - def loadAccountInfo(self, user, req): - validuntil = None - trafficleft = None - leechtraffic = None - premium = None - - if not self.HOSTER_URL: #@TODO: Remove in 0.4.10 - return {'validuntil' : validuntil, - 'trafficleft' : trafficleft, - 'leechtraffic': leechtraffic, - 'premium' : premium} - - html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) - - premium = True if re.search(self.PREMIUM_PATTERN, html) else False - - m = re.search(self.VALID_UNTIL_PATTERN, html) - if m: - expiredate = m.group(1).strip() - self.logDebug("Expire date: " + expiredate) - - try: - validuntil = mktime(strptime(expiredate, "%d %B %Y")) - - except Exception, e: - self.logError(e) - - else: - self.logDebug("Valid until: %s" % validuntil) - - if validuntil > mktime(gmtime()): - premium = True - trafficleft = -1 - else: - premium = False - validuntil = None #: registered account type (not premium) - else: - self.logDebug("VALID_UNTIL_PATTERN not found") - - m = re.search(self.TRAFFIC_LEFT_PATTERN, html) - if m: - try: - traffic = m.groupdict() - size = traffic['S'] - - if "nlimited" in size: - trafficleft = -1 - if validuntil is None: - validuntil = -1 - else: - if 'U' in traffic: - unit = traffic['U'] - elif isinstance(self.TRAFFIC_LEFT_UNIT, basestring): - unit = self.TRAFFIC_LEFT_UNIT - else: - unit = "" - - trafficleft = self.parseTraffic(size + unit) - - except Exception, e: - self.logError(e) - else: - self.logDebug("TRAFFIC_LEFT_PATTERN not found") - - leech = [m.groupdict() for m in re.finditer(self.LEECH_TRAFFIC_PATTERN, html)] - if leech: - leechtraffic = 0 - try: - for traffic in leech: - size = traffic['S'] - - if "nlimited" in size: - leechtraffic = -1 - if validuntil is None: - validuntil = -1 - break - else: - if 'U' in traffic: - unit = traffic['U'] - elif isinstance(self.LEECH_TRAFFIC_UNIT, basestring): - unit = self.LEECH_TRAFFIC_UNIT - else: - unit = "" - - leechtraffic += self.parseTraffic(size + unit) - - except Exception, e: - self.logError(e) - else: - self.logDebug("LEECH_TRAFFIC_PATTERN not found") - - return {'validuntil' : validuntil, - 'trafficleft' : trafficleft, - 'leechtraffic': leechtraffic, - 'premium' : premium} - - - def login(self, user, data, req): - if not self.HOSTER_URL: #@TODO: Remove in 0.4.10 - raise Exception(_("Missing HOSTER_DOMAIN")) - - if not self.LOGIN_URL: - self.LOGIN_URL = urljoin(self.HOSTER_URL, "login.html") - html = req.load(self.LOGIN_URL, decode=True) - - action, inputs = parseHtmlForm('name="FL"', html) - if not inputs: - inputs = {'op' : "login", - 'redirect': self.HOSTER_URL} - - inputs.update({'login' : user, - 'password': data['password']}) - - if not action: - action = self.HOSTER_URL - html = req.load(action, post=inputs, decode=True) - - if re.search(self.LOGIN_FAIL_PATTERN, html): - self.wrongPassword() diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py deleted file mode 100644 index 665e13b18..000000000 --- a/module/plugins/internal/XFSCrypter.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo - - -class XFSCrypter(SimpleCrypter): - __name__ = "XFSCrypter" - __type__ = "crypter" - __version__ = "0.06" - - __pattern__ = r'^unmatchable$' - - __description__ = """XFileSharing decrypter plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - HOSTER_DOMAIN = None - - URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')] - - LINK_PATTERN = r'<(?:td|TD).*?>\s*.+?(?:)?\s*' - NAME_PATTERN = r'<[tT]itle>.*?\: (?P.+) folder' - - OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' - TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' - - - def prepare(self): - if not self.HOSTER_DOMAIN: - if self.account: - account = self.account - else: - account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") - account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name) - - if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: - self.HOSTER_DOMAIN = account.HOSTER_DOMAIN - else: - self.fail(_("Missing HOSTER_DOMAIN")) - - if isinstance(self.COOKIES, list): - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) - - return super(XFSCrypter, self).prepare() diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py deleted file mode 100644 index ecdd2f125..000000000 --- a/module/plugins/internal/XFSHoster.py +++ /dev/null @@ -1,326 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from random import random -from time import sleep -from urlparse import urljoin, urlparse - -from pyload.plugin.internal.captcha import ReCaptcha, SolveMedia -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, secondsToMidnight -from pyload.utils import html_unescape - - -class XFSHoster(SimpleHoster): - __name__ = "XFSHoster" - __type__ = "hoster" - __version__ = "0.44" - - __pattern__ = r'^unmatchable$' - - __description__ = """XFileSharing hoster plugin""" - __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("stickell", "l.stickell@yahoo.it"), - ("Walter Purcaro", "vuolter@gmail.com")] - - - HOSTER_DOMAIN = None - - TEXT_ENCODING = False - DIRECT_LINK = None - MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... - - NAME_PATTERN = r'(Filename[ ]*:[ ]*()?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P.+?)(\s*<|")' - SIZE_PATTERN = r'(Size[ ]*:[ ]*()?|File:.*>|\s*\(|<[\w^_]+ class="size">)\s*(?P[\d.,]+)\s*(?P[\w^_]+)' - - OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' - TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' - - WAIT_PATTERN = r'.*?>(\d+)|id="countdown" value=".*?(\d+).*?"' - PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' - ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter>|>Error|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))' - - LINK_LEECH_PATTERN = r'

Download Link

\s*]*>([^<]+)' - LINK_PATTERN = None #: final download url pattern - - CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' - CAPTCHA_BLOCK_PATTERN = r'>Enter code.*?(.+?)
' - RECAPTCHA_PATTERN = None - SOLVEMEDIA_PATTERN = None - - FORM_PATTERN = None - FORM_INPUTS_MAP = None #: dict passed as input_names to parseHtmlForm - - - def setup(self): - self.chunkLimit = -1 if self.premium else 1 - self.resumeDownload = self.multiDL = self.premium - - - def prepare(self): - """ Initialize important variables """ - if not self.HOSTER_DOMAIN: - if self.account: - account = self.account - else: - account = self.pyfile.m.core.accountManager.getAccountPlugin(self.__name__) - - if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: - self.HOSTER_DOMAIN = account.HOSTER_DOMAIN - else: - self.fail(_("Missing HOSTER_DOMAIN")) - - if isinstance(self.COOKIES, list): - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) - - if not self.LINK_PATTERN: - pattern = r'(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' - self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') - - self.captcha = None - self.errmsg = None - - super(XFSHoster, self).prepare() - - if self.DIRECT_LINK is None: - self.directDL = self.premium - - - def handleFree(self, pyfile): - for i in xrange(1, 6): - self.logDebug("Getting download link: #%d" % i) - - self.checkErrors() - - m = re.search(self.LINK_PATTERN, self.html, re.S) - if m: - break - - data = self.getPostParameters() - - self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True, follow_location=False) - - m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) - if m and not "op=" in m.group(1): - break - - m = re.search(self.LINK_PATTERN, self.html, re.S) - if m: - break - else: - self.logError(data['op'] if 'op' in data else _("UNKNOWN")) - return "" - - self.link = m.group(1).strip() #@TODO: Remove .strip() in 0.4.10 - - - def handlePremium(self, pyfile): - return self.handleFree(pyfile) - - - def handleMulti(self, pyfile): - if not self.account: - self.fail(_("Only registered or premium users can use url leech feature")) - - #only tested with easybytez.com - self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) - - action, inputs = self.parseHtmlForm() - - upload_id = "%012d" % int(random() * 10 ** 12) - action += upload_id + "&js_on=1&utype=prem&upload_type=url" - - inputs['tos'] = '1' - inputs['url_mass'] = pyfile.url - inputs['up1oad_type'] = 'url' - - self.logDebug(action, inputs) - - self.req.setOption("timeout", 600) #: wait for file to upload to easybytez.com - - self.html = self.load(action, post=inputs) - - self.checkErrors() - - action, inputs = self.parseHtmlForm('F1') - if not inputs: - if self.errmsg: - self.retry(reason=self.errmsg) - else: - self.error(_("TEXTAREA F1 not found")) - - self.logDebug(inputs) - - stmsg = inputs['st'] - - if stmsg == 'OK': - self.html = self.load(action, post=inputs) - - elif 'Can not leech file' in stmsg: - self.retry(20, 3 * 60, _("Can not leech file")) - - elif 'today' in stmsg: - self.retry(wait_time=secondsToMidnight(gmt=2), reason=_("You've used all Leech traffic today")) - - else: - self.fail(stmsg) - - #get easybytez.com link for uploaded file - m = re.search(self.LINK_LEECH_PATTERN, self.html) - if m is None: - self.error(_("LINK_LEECH_PATTERN not found")) - - header = self.load(m.group(1), just_header=True, decode=True) - - if 'location' in header: #: Direct download link - self.link = header['location'] - - - def checkErrors(self): - m = re.search(self.ERROR_PATTERN, self.html) - if m is None: - self.errmsg = None - else: - self.errmsg = m.group(1).strip() - - self.logWarning(re.sub(r"<.*?>", " ", self.errmsg)) - - if 'wait' in self.errmsg: - wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec)', self.errmsg, re.I)) - self.wait(wait_time, True) - - elif 'country' in self.errmsg: - self.fail(_("Downloads are disabled for your country")) - - elif 'captcha' in self.errmsg: - self.invalidCaptcha() - - elif 'premium' in self.errmsg and 'require' in self.errmsg: - self.fail(_("File can be downloaded by premium users only")) - - elif 'limit' in self.errmsg: - if 'days' in self.errmsg: - delay = secondsToMidnight(gmt=2) - retries = 3 - else: - delay = 1 * 60 * 60 - retries = 24 - - self.wantReconnect = True - self.retry(retries, delay, _("Download limit exceeded")) - - elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: - self.retry(reason=_("Link expired")) - - elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: - self.tempOffline() - - elif 'up to' in self.errmsg: - self.fail(_("File too large for free download")) - - else: - self.wantReconnect = True - self.retry(wait_time=60, reason=self.errmsg) - - if self.errmsg: - self.info['error'] = self.errmsg - else: - self.info.pop('error', None) - - - def getPostParameters(self): - if self.FORM_PATTERN or self.FORM_INPUTS_MAP: - action, inputs = self.parseHtmlForm(self.FORM_PATTERN or "", self.FORM_INPUTS_MAP or {}) - else: - action, inputs = self.parseHtmlForm(input_names={'op': re.compile(r'^download')}) - - if not inputs: - action, inputs = self.parseHtmlForm('F1') - if not inputs: - if self.errmsg: - self.retry(reason=self.errmsg) - else: - self.error(_("TEXTAREA F1 not found")) - - self.logDebug(inputs) - - if 'op' in inputs: - if "password" in inputs: - password = self.getPassword() - if password: - inputs['password'] = password - else: - self.fail(_("Missing password")) - - if not self.premium: - m = re.search(self.WAIT_PATTERN, self.html) - if m: - wait_time = int(m.group(1)) - self.setWait(wait_time, False) - - self.captcha = self.handleCaptcha(inputs) - - self.wait() - else: - inputs['referer'] = self.pyfile.url - - if self.premium: - inputs['method_premium'] = "Premium Download" - inputs.pop('method_free', None) - else: - inputs['method_free'] = "Free Download" - inputs.pop('method_premium', None) - - return inputs - - - def handleCaptcha(self, inputs): - m = re.search(self.CAPTCHA_PATTERN, self.html) - if m: - captcha_url = m.group(1) - inputs['code'] = self.decryptCaptcha(captcha_url) - return 1 - - m = re.search(self.CAPTCHA_BLOCK_PATTERN, self.html, re.S) - if m: - captcha_div = m.group(1) - numerals = re.findall(r'(\d)', html_unescape(captcha_div)) - - self.logDebug(captcha_div) - - inputs['code'] = "".join(a[1] for a in sorted(numerals, key=lambda num: int(num[0]))) - - self.logDebug("Captcha code: %s" % inputs['code'], numerals) - return 2 - - recaptcha = ReCaptcha(self) - try: - captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) - - except Exception: - captcha_key = recaptcha.detect_key() - - else: - self.logDebug("ReCaptcha key: %s" % captcha_key) - - if captcha_key: - inputs['recaptcha_response_field'], inputs['recaptcha_challenge_field'] = recaptcha.challenge(captcha_key) - return 3 - - solvemedia = SolveMedia(self) - try: - captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) - - except Exception: - captcha_key = solvemedia.detect_key() - - else: - self.logDebug("SolveMedia key: %s" % captcha_key) - - if captcha_key: - inputs['adcopy_response'], inputs['adcopy_challenge'] = solvemedia.challenge(captcha_key) - return 4 - - return 0 diff --git a/module/plugins/internal/__init__.py b/module/plugins/internal/__init__.py deleted file mode 100644 index 40a96afc6..000000000 --- a/module/plugins/internal/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- -- cgit v1.2.3