summaryrefslogtreecommitdiffstats
path: root/module/plugins/internal
diff options
context:
space:
mode:
Diffstat (limited to 'module/plugins/internal')
-rw-r--r--module/plugins/internal/BasePlugin.py95
-rw-r--r--module/plugins/internal/CaptchaService.py522
-rw-r--r--module/plugins/internal/DeadCrypter.py31
-rw-r--r--module/plugins/internal/DeadHoster.py31
-rw-r--r--module/plugins/internal/Extractor.py139
-rw-r--r--module/plugins/internal/MultiHook.py308
-rw-r--r--module/plugins/internal/MultiHoster.py85
-rw-r--r--module/plugins/internal/SevenZip.py155
-rw-r--r--module/plugins/internal/SimpleCrypter.py157
-rw-r--r--module/plugins/internal/SimpleDereferer.py98
-rw-r--r--module/plugins/internal/SimpleHoster.py701
-rw-r--r--module/plugins/internal/UnRar.py248
-rw-r--r--module/plugins/internal/UnZip.py68
-rw-r--r--module/plugins/internal/UpdateManager.py300
-rw-r--r--module/plugins/internal/XFSAccount.py174
-rw-r--r--module/plugins/internal/XFSCrypter.py45
-rw-r--r--module/plugins/internal/XFSHoster.py326
-rw-r--r--module/plugins/internal/__init__.py1
18 files changed, 0 insertions, 3484 deletions
diff --git a/module/plugins/internal/BasePlugin.py b/module/plugins/internal/BasePlugin.py
deleted file mode 100644
index 792497449..000000000
--- a/module/plugins/internal/BasePlugin.py
+++ /dev/null
@@ -1,95 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-
-from urllib import unquote
-from urlparse import urljoin, urlparse
-
-from module.network.HTTPRequest import BadHeader
-from module.plugins.internal.SimpleHoster import create_getInfo, fileUrl
-from module.plugins.Hoster import Hoster
-
-
-class BasePlugin(Hoster):
- __name__ = "BasePlugin"
- __type__ = "hoster"
- __version__ = "0.34"
-
- __pattern__ = r'^unmatchable$'
-
- __description__ = """Base plugin when any other didnt fit"""
- __license__ = "GPLv3"
- __authors__ = [("RaNaN", "RaNaN@pyload.org"),
- ("Walter Purcaro", "vuolter@gmail.com")]
-
-
- @classmethod
- def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10
- url = unquote(url)
- url_p = urlparse(url)
- return {'name' : (url_p.path.split('/')[-1]
- or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0]
- or url_p.netloc.split('.', 1)[0]),
- 'size' : 0,
- 'status': 3 if url else 8,
- 'url' : url}
-
-
- def setup(self):
- self.chunkLimit = -1
- self.multiDL = True
- self.resumeDownload = True
-
-
- def process(self, pyfile):
- """main function"""
-
- pyfile.name = self.getInfo(pyfile.url)['name']
-
- if not pyfile.url.startswith("http"):
- self.fail(_("No plugin matched"))
-
- for _i in xrange(5):
- try:
- link = fileUrl(self, unquote(pyfile.url))
-
- if link:
- self.download(link, ref=False, disposition=True)
- else:
- self.fail(_("File not found"))
-
- except BadHeader, e:
- if e.code is 404:
- self.offline()
-
- elif e.code in (401, 403):
- self.logDebug("Auth required", "Received HTTP status code: %d" % e.code)
-
- account = self.core.accountManager.getAccountPlugin('Http')
- servers = [x['login'] for x in account.getAllAccounts()]
- server = urlparse(pyfile.url).netloc
-
- if server in servers:
- self.logDebug("Logging on to %s" % server)
- self.req.addAuth(account.getAccountData(server)['password'])
- else:
- pwd = self.getPassword()
- if ':' in pwd:
- self.req.addAuth(pwd)
- else:
- self.fail(_("Authorization required"))
- else:
- self.fail(e)
- else:
- break
- else:
- self.fail(_("No file downloaded")) #@TODO: Move to hoster class in 0.4.10
-
- check = self.checkDownload({'empty file': re.compile(r'\A\Z'),
- 'html file' : re.compile(r'\A\s*<!DOCTYPE html'),
- 'html error': re.compile(r'\A\s*(<.+>)?\d{3}(\Z|\s+)')})
- if check:
- self.fail(check.capitalize())
-
-
-getInfo = create_getInfo(BasePlugin)
diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py
deleted file mode 100644
index 6f2c8e06d..000000000
--- a/module/plugins/internal/CaptchaService.py
+++ /dev/null
@@ -1,522 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-import time
-
-from base64 import b64encode
-from random import random, randint
-from urlparse import urljoin, urlparse
-
-from module.common.json_layer import json_loads
-from module.plugins.Plugin import Base
-
-
-#@TODO: Extend (new) Plugin class; remove all `html` args
-class CaptchaService(Base):
- __name__ = "CaptchaService"
- __version__ = "0.25"
-
- __description__ = """Base captcha service plugin"""
- __license__ = "GPLv3"
- __authors__ = [("pyLoad Team", "admin@pyload.org")]
-
-
- key = None #: last key detected
-
-
- def __init__(self, plugin):
- self.plugin = plugin
- super(CaptchaService, self).__init__(plugin.core)
-
-
- def detect_key(self, html=None):
- raise NotImplementedError
-
-
- def challenge(self, key=None, html=None):
- raise NotImplementedError
-
-
- def result(self, server, challenge):
- raise NotImplementedError
-
-
-class ReCaptcha(CaptchaService):
- __name__ = "ReCaptcha"
- __version__ = "0.14"
-
- __description__ = """ReCaptcha captcha service plugin"""
- __license__ = "GPLv3"
- __authors__ = [("pyLoad Team", "admin@pyload.org"),
- ("Walter Purcaro", "vuolter@gmail.com"),
- ("zapp-brannigan", "fuerst.reinje@web.de")]
-
-
- KEY_V2_PATTERN = r'(?:data-sitekey=["\']|["\']sitekey["\']:\s*["\'])([\w-]+)'
- KEY_V1_PATTERN = r'(?:recaptcha(?:/api|\.net)/(?:challenge|noscript)\?k=|Recaptcha\.create\s*\(\s*["\'])([\w-]+)'
-
-
- def detect_key(self, html=None):
- if not html:
- if hasattr(self.plugin, "html") and self.plugin.html:
- html = self.plugin.html
- else:
- errmsg = _("ReCaptcha html not found")
- self.plugin.fail(errmsg)
- raise TypeError(errmsg)
-
- m = re.search(self.KEY_V2_PATTERN, html) or re.search(self.KEY_V1_PATTERN, html)
- if m:
- self.key = m.group(1).strip()
- self.logDebug("Key: %s" % self.key)
- return self.key
- else:
- self.logDebug("Key not found")
- return None
-
-
- def challenge(self, key=None, html=None, version=None):
- if not key:
- if self.detect_key(html):
- key = self.key
- else:
- errmsg = _("ReCaptcha key not found")
- self.plugin.fail(errmsg)
- raise TypeError(errmsg)
-
- if version in (1, 2):
- return getattr(self, "_challenge_v%s" % version)(key)
-
- elif not html and hasattr(self.plugin, "html") and self.plugin.html:
- version = 2 if re.search(self.KEY_V2_PATTERN, self.plugin.html) else 1
- return self.challenge(key, self.plugin.html, version)
-
- else:
- errmsg = _("ReCaptcha html not found")
- self.plugin.fail(errmsg)
- raise TypeError(errmsg)
-
-
- def _challenge_v1(self, key):
- html = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge",
- get={'k': key})
- try:
- challenge = re.search("challenge : '(.+?)',", html).group(1)
- server = re.search("server : '(.+?)',", html).group(1)
-
- except AttributeError:
- errmsg = _("ReCaptcha challenge pattern not found")
- self.plugin.fail(errmsg)
- raise AttributeError(errmsg)
-
- self.logDebug("Challenge: %s" % challenge)
-
- return self.result(server, challenge), challenge
-
-
- def result(self, server, challenge):
- result = self.plugin.decryptCaptcha("%simage" % server,
- get={'c': challenge},
- cookies=True,
- forceUser=True,
- imgtype="jpg")
-
- self.logDebug("Result: %s" % result)
-
- return result
-
-
- def _collectApiInfo(self):
- html = self.plugin.req.load("http://www.google.com/recaptcha/api.js")
- a = re.search(r'po.src = \'(.*?)\';', html).group(1)
- vers = a.split("/")[5]
-
- self.logDebug("API version: %s" %vers)
-
- language = a.split("__")[1].split(".")[0]
-
- self.logDebug("API language: %s" % language)
-
- html = self.plugin.req.load("https://apis.google.com/js/api.js")
- b = re.search(r'"h":"(.*?)","', html).group(1)
- jsh = b.decode('unicode-escape')
-
- self.logDebug("API jsh-string: %s" % jsh)
-
- return vers, language, jsh
-
-
- def _prepareTimeAndRpc(self):
- self.plugin.req.load("http://www.google.com/recaptcha/api2/demo")
-
- millis = int(round(time.time() * 1000))
-
- self.logDebug("Time: %s" % millis)
-
- rand = randint(1, 99999999)
- a = "0.%s" % str(rand * 2147483647)
- rpc = int(100000000 * float(a))
-
- self.logDebug("Rpc-token: %s" % rpc)
-
- return millis, rpc
-
-
- def _challenge_v2(self, key, parent=None):
- if parent is None:
- try:
- parent = urljoin("http://", urlparse(self.plugin.pyfile.url).netloc)
-
- except Exception:
- parent = ""
-
- botguardstring = "!A"
- vers, language, jsh = self._collectApiInfo()
- millis, rpc = self._prepareTimeAndRpc()
-
- html = self.plugin.req.load("https://www.google.com/recaptcha/api2/anchor",
- get={'k' : key,
- 'hl' : language,
- 'v' : vers,
- 'usegapi' : "1",
- 'jsh' : "%s#id=IO_%s" % (jsh, millis),
- 'parent' : parent,
- 'pfname' : "",
- 'rpctoken': rpc})
-
- token1 = re.search(r'id="recaptcha-token" value="(.*?)">', html)
- self.logDebug("Token #1: %s" % token1.group(1))
-
- html = self.plugin.req.load("https://www.google.com/recaptcha/api2/frame",
- get={'c' : token1.group(1),
- 'hl' : language,
- 'v' : vers,
- 'bg' : botguardstring,
- 'k' : key,
- 'usegapi': "1",
- 'jsh' : jsh}).decode('unicode-escape')
-
- token2 = re.search(r'"finput","(.*?)",', html)
- self.logDebug("Token #2: %s" % token2.group(1))
-
- token3 = re.search(r'."asconf".\s,".*?".\s,"(.*?)".', html)
- self.logDebug("Token #3: %s" % token3.group(1))
-
- html = self.plugin.req.load("https://www.google.com/recaptcha/api2/reload",
- post={'k' : key,
- 'c' : token2.group(1),
- 'reason': "fi",
- 'fbg' : token3.group(1)})
-
- token4 = re.search(r'"rresp","(.*?)",', html)
- self.logDebug("Token #4: %s" % token4.group(1))
-
- millis_captcha_loading = int(round(time.time() * 1000))
- captcha_response = self.plugin.decryptCaptcha("https://www.google.com/recaptcha/api2/payload",
- get={'c':token4.group(1), 'k':key},
- cookies=True,
- forceUser=True)
- response = b64encode('{"response":"%s"}' % captcha_response)
-
- self.logDebug("Result: %s" % response)
-
- timeToSolve = int(round(time.time() * 1000)) - millis_captcha_loading
- timeToSolveMore = timeToSolve + int(float("0." + str(randint(1, 99999999))) * 500)
-
- html = self.plugin.req.load("https://www.google.com/recaptcha/api2/userverify",
- post={'k' : key,
- 'c' : token4.group(1),
- 'response': response,
- 't' : timeToSolve,
- 'ct' : timeToSolveMore,
- 'bg' : botguardstring})
-
- token5 = re.search(r'"uvresp","(.*?)",', html)
- self.logDebug("Token #5: %s" % token5.group(1))
-
- result = token5.group(1)
-
- return result, None
-
-
-
-class AdsCaptcha(CaptchaService):
- __name__ = "AdsCaptcha"
- __version__ = "0.08"
-
- __description__ = """AdsCaptcha captcha service plugin"""
- __license__ = "GPLv3"
- __authors__ = [("pyLoad Team", "admin@pyload.org")]
-
-
- CAPTCHAID_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(\d+)'
- PUBLICKEY_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*PublicKey=([\w-]+)'
-
-
- def detect_key(self, html=None):
- if not html:
- if hasattr(self.plugin, "html") and self.plugin.html:
- html = self.plugin.html
- else:
- errmsg = _("AdsCaptcha html not found")
- self.plugin.fail(errmsg)
- raise TypeError(errmsg)
-
- m = re.search(self.PUBLICKEY_PATTERN, html)
- n = re.search(self.CAPTCHAID_PATTERN, html)
- if m and n:
- self.key = (m.group(1).strip(), n.group(1).strip()) #: key is the tuple(PublicKey, CaptchaId)
- self.logDebug("Key|id: %s | %s" % self.key)
- return self.key
- else:
- self.logDebug("Key or id not found")
- return None
-
-
- def challenge(self, key=None, html=None):
- if not key:
- if self.detect_key(html):
- key = self.key
- else:
- errmsg = _("AdsCaptcha key not found")
- self.plugin.fail(errmsg)
- raise TypeError(errmsg)
-
- PublicKey, CaptchaId = key
-
- html = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx",
- get={'CaptchaId': CaptchaId,
- 'PublicKey': PublicKey})
- try:
- challenge = re.search("challenge: '(.+?)',", html).group(1)
- server = re.search("server: '(.+?)',", html).group(1)
-
- except AttributeError:
- errmsg = _("AdsCaptcha challenge pattern not found")
- self.plugin.fail(errmsg)
- raise AttributeError(errmsg)
-
- self.logDebug("Challenge: %s" % challenge)
-
- return self.result(server, challenge), challenge
-
-
- def result(self, server, challenge):
- result = self.plugin.decryptCaptcha("%sChallenge.aspx" % server,
- get={'cid': challenge, 'dummy': random()},
- cookies=True,
- imgtype="jpg")
-
- self.logDebug("Result: %s" % result)
-
- return result
-
-
-class SolveMedia(CaptchaService):
- __name__ = "SolveMedia"
- __version__ = "0.12"
-
- __description__ = """SolveMedia captcha service plugin"""
- __license__ = "GPLv3"
- __authors__ = [("pyLoad Team", "admin@pyload.org")]
-
-
- KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']'
-
-
- def detect_key(self, html=None):
- if not html:
- if hasattr(self.plugin, "html") and self.plugin.html:
- html = self.plugin.html
- else:
- errmsg = _("SolveMedia html not found")
- self.plugin.fail(errmsg)
- raise TypeError(errmsg)
-
- m = re.search(self.KEY_PATTERN, html)
- if m:
- self.key = m.group(1).strip()
- self.logDebug("Key: %s" % self.key)
- return self.key
- else:
- self.logDebug("Key not found")
- return None
-
-
- def challenge(self, key=None, html=None):
- if not key:
- if self.detect_key(html):
- key = self.key
- else:
- errmsg = _("SolveMedia key not found")
- self.plugin.fail(errmsg)
- raise TypeError(errmsg)
-
- html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript",
- get={'k': key})
- try:
- challenge = re.search(r'<input type=hidden name="adcopy_challenge" id="adcopy_challenge" value="([^"]+)">',
- html).group(1)
- server = "http://api.solvemedia.com/papi/media"
-
- except AttributeError:
- errmsg = _("SolveMedia challenge pattern not found")
- self.plugin.fail(errmsg)
- raise AttributeError(errmsg)
-
- self.logDebug("Challenge: %s" % challenge)
-
- result = self.result(server, challenge)
-
- try:
- magic = re.search(r'name="magic" value="(.+?)"', html).group(1)
-
- except AttributeError:
- self.logDebug("Magic code not found")
-
- else:
- if not self._verify(key, magic, result, challenge):
- self.logDebug("Captcha code was invalid")
-
- return result, challenge
-
-
- def _verify(self, key, magic, result, challenge, ref=None): #@TODO: Clean up
- if ref is None:
- try:
- ref = self.plugin.pyfile.url
-
- except Exception:
- ref = ""
-
- html = self.plugin.req.load("http://api.solvemedia.com/papi/verify.noscript",
- post={'adcopy_response' : result,
- 'k' : key,
- 'l' : "en",
- 't' : "img",
- 's' : "standard",
- 'magic' : magic,
- 'adcopy_challenge' : challenge,
- 'ref' : ref})
- try:
- html = self.plugin.req.load(re.search(r'URL=(.+?)">', html).group(1))
- gibberish = re.search(r'id=gibberish>(.+?)</textarea>', html).group(1)
-
- except Exception:
- return False
-
- else:
- return True
-
-
- def result(self, server, challenge):
- result = self.plugin.decryptCaptcha(server,
- get={'c': challenge},
- cookies=True,
- imgtype="gif")
-
- self.logDebug("Result: %s" % result)
-
- return result
-
-
-class AdYouLike(CaptchaService):
- __name__ = "AdYouLike"
- __version__ = "0.05"
-
- __description__ = """AdYouLike captcha service plugin"""
- __license__ = "GPLv3"
- __authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
-
-
- AYL_PATTERN = r'Adyoulike\.create\s*\((.+?)\)'
- CALLBACK_PATTERN = r'(Adyoulike\.g\._jsonp_\d+)'
-
-
- def detect_key(self, html=None):
- if not html:
- if hasattr(self.plugin, "html") and self.plugin.html:
- html = self.plugin.html
- else:
- errmsg = _("AdYouLike html not found")
- self.plugin.fail(errmsg)
- raise TypeError(errmsg)
-
- m = re.search(self.AYL_PATTERN, html)
- n = re.search(self.CALLBACK_PATTERN, html)
- if m and n:
- self.key = (m.group(1).strip(), n.group(1).strip())
- self.logDebug("Ayl|callback: %s | %s" % self.key)
- return self.key #: key is the tuple(ayl, callback)
- else:
- self.logDebug("Ayl or callback not found")
- return None
-
-
- def challenge(self, key=None, html=None):
- if not key:
- if self.detect_key(html):
- key = self.key
- else:
- errmsg = _("AdYouLike key not found")
- self.plugin.fail(errmsg)
- raise TypeError(errmsg)
-
- ayl, callback = key
-
- # {"adyoulike":{"key":"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"},
- # "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}}
- ayl = json_loads(ayl)
-
- html = self.plugin.req.load("http://api-ayl.appspot.com/challenge",
- get={'key' : ayl['adyoulike']['key'],
- 'env' : ayl['all']['env'],
- 'callback': callback})
- try:
- challenge = json_loads(re.search(callback + r'\s*\((.+?)\)', html).group(1))
-
- except AttributeError:
- errmsg = _("AdYouLike challenge pattern not found")
- self.plugin.fail(errmsg)
- raise AttributeError(errmsg)
-
- self.logDebug("Challenge: %s" % challenge)
-
- return self.result(ayl, challenge), challenge
-
-
- def result(self, server, challenge):
- # Adyoulike.g._jsonp_5579316662423138
- # ({"translations":{"fr":{"instructions_visual":"Recopiez « Soonnight » ci-dessous :"}},
- # "site_under":true,"clickable":true,"pixels":{"VIDEO_050":[],"DISPLAY":[],"VIDEO_000":[],"VIDEO_100":[],
- # "VIDEO_025":[],"VIDEO_075":[]},"medium_type":"image/adyoulike",
- # "iframes":{"big":"<iframe src=\"http://www.soonnight.com/campagn.html\" scrolling=\"no\"
- # height=\"250\" width=\"300\" frameborder=\"0\"></iframe>"},"shares":{},"id":256,
- # "token":"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1","formats":{"small":{"y":300,"x":0,"w":300,"h":60},
- # "big":{"y":0,"x":0,"w":300,"h":250},"hover":{"y":440,"x":0,"w":300,"h":60}},
- # "tid":"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"})
-
- if isinstance(server, basestring):
- server = json_loads(server)
-
- if isinstance(challenge, basestring):
- challenge = json_loads(challenge)
-
- try:
- instructions_visual = challenge['translations'][server['all']['lang']]['instructions_visual']
- result = re.search(u'«(.+?)»', instructions_visual).group(1).strip()
-
- except AttributeError:
- errmsg = _("AdYouLike result not found")
- self.plugin.fail(errmsg)
- raise AttributeError(errmsg)
-
- result = {'_ayl_captcha_engine' : "adyoulike",
- '_ayl_env' : server['all']['env'],
- '_ayl_tid' : challenge['tid'],
- '_ayl_token_challenge': challenge['token'],
- '_ayl_response' : response}
-
- self.logDebug("Result: %s" % result)
-
- return result
diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py
deleted file mode 100644
index ce56947fc..000000000
--- a/module/plugins/internal/DeadCrypter.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from pyload.plugin.Crypter import Crypter as _Crypter
-from pyload.plugin.internal.SimpleCrypter import create_getInfo
-
-
-class DeadCrypter(_Crypter):
- __name__ = "DeadCrypter"
- __type__ = "crypter"
- __version__ = "0.04"
-
- __pattern__ = r'^unmatchable$'
-
- __description__ = """Crypter is no longer available"""
- __license__ = "GPLv3"
- __authors__ = [("stickell", "l.stickell@yahoo.it")]
-
-
- @classmethod
- def apiInfo(cls, url="", get={}, post={}):
- api = super(DeadCrypter, self).apiInfo(url, get, post)
- api['status'] = 1
- return api
-
-
- def setup(self):
- self.pyfile.error = "Crypter is no longer available"
- self.offline() #@TODO: self.offline("Crypter is no longer available")
-
-
-getInfo = create_getInfo(DeadCrypter)
diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py
deleted file mode 100644
index 132e4741a..000000000
--- a/module/plugins/internal/DeadHoster.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from pyload.plugin.Hoster import Hoster as _Hoster
-from pyload.plugin.internal.SimpleHoster import create_getInfo
-
-
-class DeadHoster(_Hoster):
- __name__ = "DeadHoster"
- __type__ = "hoster"
- __version__ = "0.14"
-
- __pattern__ = r'^unmatchable$'
-
- __description__ = """Hoster is no longer available"""
- __license__ = "GPLv3"
- __authors__ = [("zoidberg", "zoidberg@mujmail.cz")]
-
-
- @classmethod
- def apiInfo(cls, url="", get={}, post={}):
- api = super(DeadHoster, self).apiInfo(url, get, post)
- api['status'] = 1
- return api
-
-
- def setup(self):
- self.pyfile.error = "Hoster is no longer available"
- self.offline() #@TODO: self.offline("Hoster is no longer available")
-
-
-getInfo = create_getInfo(DeadHoster)
diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py
deleted file mode 100644
index b445f1497..000000000
--- a/module/plugins/internal/Extractor.py
+++ /dev/null
@@ -1,139 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-
-from module.PyFile import PyFile
-
-
-class ArchiveError(Exception):
- pass
-
-
-class CRCError(Exception):
- pass
-
-
-class PasswordError(Exception):
- pass
-
-
-class Extractor:
- __name__ = "Extractor"
- __version__ = "0.20"
-
- __description__ = """Base extractor plugin"""
- __license__ = "GPLv3"
- __authors__ = [("RaNaN", "ranan@pyload.org"),
- ("Walter Purcaro", "vuolter@gmail.com"),
- ("Immenz", "immenz@gmx.net")]
-
-
- EXTENSIONS = []
- VERSION = ""
-
-
- @classmethod
- def isArchive(cls, filename):
- name = os.path.basename(filename).lower()
- return any(name.endswith(ext) for ext in cls.EXTENSIONS) and not cls.isMultipart(filename)
-
-
- @classmethod
- def isMultipart(cls,filename):
- return False
-
-
- @classmethod
- def isUsable(cls):
- """ Check if system statisfy dependencies
- :return: boolean
- """
- return None
-
-
- @classmethod
- def getTargets(cls, files_ids):
- """ Filter suited targets from list of filename id tuple list
- :param files_ids: List of filepathes
- :return: List of targets, id tuple list
- """
- return [(fname, id, fout) for fname, id, fout in files_ids if cls.isArchive(fname)]
-
-
- def __init__(self, manager, filename, out,
- fullpath=True,
- overwrite=False,
- excludefiles=[],
- renice=0,
- delete=False,
- keepbroken=False,
- fid=None):
- """ Initialize extractor for specific file """
- self.manager = manager
- self.filename = filename
- self.out = out
- self.fullpath = fullpath
- self.overwrite = overwrite
- self.excludefiles = excludefiles
- self.renice = renice
- self.delete = delete
- self.keepbroken = keepbroken
- self.files = [] #: Store extracted files here
-
- pyfile = self.manager.core.files.getFile(fid) if fid else None
- self.notifyProgress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None
-
-
- def init(self):
- """ Initialize additional data structures """
- pass
-
-
- def check(self):
- """Check if password if needed. Raise ArchiveError if integrity is
- questionable.
-
- :return: boolean
- :raises ArchiveError
- """
- raise PasswordError
-
-
- def isPassword(self, password):
- """ Check if the given password is/might be correct.
- If it can not be decided at this point return true.
-
- :param password:
- :return: boolean
- """
- return None
-
-
- def repair(self):
- return None
-
-
- def extract(self, password=None):
- """Extract the archive. Raise specific errors in case of failure.
-
- :param progress: Progress function, call this to update status
- :param password password to use
- :raises PasswordError
- :raises CRCError
- :raises ArchiveError
- :return:
- """
- raise NotImplementedError
-
-
- def getDeleteFiles(self):
- """Return list of files to delete, do *not* delete them here.
-
- :return: List with paths of files to delete
- """
- return [self.filename]
-
-
- def list(self, password=None):
- """Populate self.files at some point while extracting"""
- return self.files
diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py
deleted file mode 100644
index 652443098..000000000
--- a/module/plugins/internal/MultiHook.py
+++ /dev/null
@@ -1,308 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-
-from time import sleep
-
-from module.plugins.Hook import Hook
-from module.utils import decode, remove_chars
-
-
-class MultiHook(Hook):
- __name__ = "MultiHook"
- __type__ = "hook"
- __version__ = "0.37"
-
- __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"),
- ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ),
- ("revertfailed" , "bool" , "Revert to standard download if fails", True ),
- ("retry" , "int" , "Number of retries before revert" , 10 ),
- ("retryinterval" , "int" , "Retry interval in minutes" , 1 ),
- ("reload" , "bool" , "Reload plugin list" , True ),
- ("reloadinterval", "int" , "Reload interval in hours" , 12 )]
-
- __description__ = """Hook plugin for multi hoster/crypter"""
- __license__ = "GPLv3"
- __authors__ = [("pyLoad Team", "admin@pyload.org"),
- ("Walter Purcaro", "vuolter@gmail.com")]
-
-
- MIN_INTERVAL = 1 * 60 * 60
-
- DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"),
- (r'1fichier\.com' , "onefichier.com" ),
- (r'2shared\.com' , "twoshared.com" ),
- (r'4shared\.com' , "fourshared.com" ),
- (r'bayfiles\.net' , "bayfiles.com" ),
- (r'cloudnator\.com' , "shragle.com" ),
- (r'dfiles\.eu' , "depositfiles.com" ),
- (r'easy-share\.com' , "crocko.com" ),
- (r'freakshare\.net' , "freakshare.com" ),
- (r'hellshare\.com' , "hellshare.cz" ),
- (r'ifile\.it' , "filecloud.io" ),
- (r'nowdownload\.\w+', "nowdownload.sx" ),
- (r'nowvideo\.\w+' , "nowvideo.sx" ),
- (r'putlocker\.com' , "firedrive.com" ),
- (r'share-?rapid\.cz', "multishare.cz" ),
- (r'ul\.to' , "uploaded.to" ),
- (r'uploaded\.net' , "uploaded.to" ),
- (r'uploadhero\.co' , "uploadhero.com" ),
- (r'zshares\.net' , "zshare.net" ),
- (r'(\d+.+)' , "X\1" )]
-
-
- def setup(self):
- self.plugins = []
- self.supported = []
- self.new_supported = []
-
- self.account = None
- self.pluginclass = None
- self.pluginmodule = None
- self.pluginname = None
- self.plugintype = None
-
- self._initPlugin()
-
-
- def _initPlugin(self):
- plugin, type = self.core.pluginManager.findPlugin(self.__name__)
-
- if not plugin:
- self.logWarning("Hook plugin will be deactivated due missing plugin reference")
- self.setConfig('activated', False)
- else:
- self.pluginname = self.__name__
- self.plugintype = type
- self.pluginmodule = self.core.pluginManager.loadModule(type, self.__name__)
- self.pluginclass = getattr(self.pluginmodule, self.__name__)
-
-
- def _loadAccount(self):
- self.account = self.core.accountManager.getAccountPlugin(self.pluginname)
-
- if self.account and not self.account.canUse():
- self.account = None
-
- if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT:
- self.logWarning("Hook plugin will be deactivated due missing account reference")
- self.setConfig('activated', False)
-
-
- def coreReady(self):
- self._loadAccount()
-
-
- def getURL(self, *args, **kwargs): #@TODO: Remove in 0.4.10
- """ see HTTPRequest for argument list """
- h = pyreq.getHTTPRequest(timeout=120)
- try:
- if not 'decode' in kwargs:
- kwargs['decode'] = True
- rep = h.load(*args, **kwargs)
- finally:
- h.close()
-
- return rep
-
-
- def getConfig(self, option, default=''):
- """getConfig with default value - sublass may not implements all config options"""
- try:
- return self.getConf(option)
-
- except KeyError:
- return default
-
-
- def pluginsCached(self):
- if self.plugins:
- return self.plugins
-
- for _i in xrange(3):
- try:
- pluginset = self._pluginSet(self.getHosters() if self.plugintype == "hoster" else self.getCrypters())
-
- except Exception, e:
- self.logError(e, "Waiting 1 minute and retry")
- sleep(60)
-
- else:
- break
- else:
- return list()
-
- try:
- configmode = self.getConfig("pluginmode", 'all')
- if configmode in ("listed", "unlisted"):
- pluginlist = self.getConfig("pluginlist", '').replace('|', ',').replace(';', ',').split(',')
- configset = self._pluginSet(pluginlist)
-
- if configmode == "listed":
- pluginset &= configset
- else:
- pluginset -= configset
-
- except Exception, e:
- self.logError(e)
-
- self.plugins = list(pluginset)
-
- return self.plugins
-
-
- def _pluginSet(self, plugins):
- plugins = set((decode(x).strip().lower() for x in plugins if '.' in x))
-
- for rf, rt in self.DOMAIN_REPLACEMENTS:
- regex = re.compile(rf)
- for p in filter(lambda x: regex.match(x), plugins):
- plugins.remove(p)
- plugins.add(re.sub(rf, rt, p))
-
- plugins.discard('')
-
- return plugins
-
-
- def getHosters(self):
- """Load list of supported hoster
-
- :return: List of domain names
- """
- raise NotImplementedError
-
-
- def getCrypters(self):
- """Load list of supported crypters
-
- :return: List of domain names
- """
- raise NotImplementedError
-
-
- def periodical(self):
- """reload plugin list periodically"""
- self.logInfo(_("Reloading supported %s list") % self.plugintype)
-
- old_supported = self.supported
-
- self.supported = []
- self.new_supported = []
- self.plugins = []
-
- self.overridePlugins()
-
- old_supported = [plugin for plugin in old_supported if plugin not in self.supported]
-
- if old_supported:
- self.logDebug("Unload: %s" % ", ".join(old_supported))
- for plugin in old_supported:
- self.unloadPlugin(plugin)
-
- if self.getConfig("reload", True):
- self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_INTERVAL)
- else:
- self.core.scheduler.removeJob(self.cb)
- self.cb = None
-
-
- def overridePlugins(self):
- excludedList = []
-
- if self.plugintype == "hoster":
- pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys())
- accountList = [account.type.lower() for account in self.core.api.getAccounts(False) if account.valid and account.premium]
- else:
- pluginMap = {}
- accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.core.pluginManager.crypterPlugins.iterkeys()]
-
- for plugin in self.pluginsCached():
- name = remove_chars(plugin, "-.")
-
- if name in accountList:
- excludedList.append(plugin)
- else:
- if name in pluginMap:
- self.supported.append(pluginMap[name])
- else:
- self.new_supported.append(plugin)
-
- if not self.supported and not self.new_supported:
- self.logError(_("No %s loaded") % self.plugintype)
- return
-
- # inject plugin plugin
- self.logDebug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported))))
-
- for plugin in self.supported:
- hdict = self.core.pluginManager.plugins[self.plugintype][plugin]
- hdict['new_module'] = self.pluginmodule
- hdict['new_name'] = self.pluginname
-
- if excludedList:
- self.logInfo(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList))))
-
- if self.new_supported:
- plugins = sorted(self.new_supported)
-
- self.logDebug("New %ss: %s" % (self.plugintype, ", ".join(plugins)))
-
- # create new regexp
- regexp = r'.*(?P<DOMAIN>%s).*' % "|".join([x.replace(".", "\.") for x in plugins])
- if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and '://' in self.pluginclass.__pattern__:
- regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp)
-
- self.logDebug("Regexp: %s" % regexp)
-
- hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname]
- hdict['pattern'] = regexp
- hdict['re'] = re.compile(regexp)
-
-
- def unloadPlugin(self, plugin):
- hdict = self.core.pluginManager.plugins[self.plugintype][plugin]
- if "module" in hdict:
- del hdict['module']
-
- if "new_module" in hdict:
- del hdict['new_module']
- del hdict['new_name']
-
-
- def unload(self):
- """Remove override for all plugins. Scheduler job is removed by hookmanager"""
- for plugin in self.supported:
- self.unloadPlugin(plugin)
-
- # reset pattern
- hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname]
-
- hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$')
- hdict['re'] = re.compile(hdict['pattern'])
-
-
- def downloadFailed(self, pyfile):
- """remove plugin override if download fails but not if file is offline/temp.offline"""
- if pyfile.status != 8 or not self.getConfig("revertfailed", True):
- return
-
- hdict = self.core.pluginManager.plugins[self.plugintype][pyfile.pluginname]
- if "new_name" in hdict and hdict['new_name'] == self.pluginname:
- if pyfile.error == "MultiHook":
- self.logDebug("Unload MultiHook", pyfile.pluginname, hdict)
- self.unloadPlugin(pyfile.pluginname)
- pyfile.setStatus("queued")
- pyfile.sync()
- else:
- retries = max(self.getConfig("retry", 10), 0)
- wait_time = max(self.getConfig("retryinterval", 1), 0)
-
- if 0 < retries > pyfile.plugin.retries:
- self.logInfo(_("Retrying: %s") % pyfile.name)
- pyfile.setCustomStatus("MultiHook", "queued")
- pyfile.sync()
-
- pyfile.plugin.retries += 1
- pyfile.plugin.setWait(wait_time)
- pyfile.plugin.wait()
diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py
deleted file mode 100644
index 63b7d76b1..000000000
--- a/module/plugins/internal/MultiHoster.py
+++ /dev/null
@@ -1,85 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-
-from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies
-
-
-class MultiHoster(SimpleHoster):
- __name__ = "MultiHoster"
- __type__ = "hoster"
- __version__ = "0.37"
-
- __pattern__ = r'^unmatchable$'
-
- __description__ = """Multi hoster plugin"""
- __license__ = "GPLv3"
- __authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
-
-
- LOGIN_ACCOUNT = True
-
-
- def setup(self):
- self.chunkLimit = 1
- self.multiDL = bool(self.account)
- self.resumeDownload = self.premium
-
-
- def prepare(self):
- self.info = {}
- self.html = ""
- self.link = "" #@TODO: Move to hoster class in 0.4.10
- self.directDL = False #@TODO: Move to hoster class in 0.4.10
-
- if self.LOGIN_ACCOUNT and not self.account:
- self.fail(_("Required account not found"))
-
- self.req.setOption("timeout", 120)
-
- if isinstance(self.COOKIES, list):
- set_cookies(self.req.cj, self.COOKIES)
-
- if self.DIRECT_LINK is None:
- self.directDL = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url)
- else:
- self.directDL = self.DIRECT_LINK
-
- self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS)
-
-
- def process(self, pyfile):
- self.prepare()
-
- if self.directDL:
- self.checkInfo()
- self.logDebug("Looking for direct download link...")
- self.handleDirect(pyfile)
-
- if not self.link and not self.lastDownload:
- self.preload()
-
- self.checkErrors()
- self.checkStatus(getinfo=False)
-
- if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
- self.logDebug("Handled as premium download")
- self.handlePremium(pyfile)
-
- elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
- self.logDebug("Handled as free download")
- self.handleFree(pyfile)
-
- self.downloadLink(self.link, True)
- self.checkFile()
-
-
- def handlePremium(self, pyfile):
- return self.handleFree(pyfile)
-
-
- def handleFree(self, pyfile):
- if self.premium:
- raise NotImplementedError
- else:
- self.fail(_("Required premium account not found"))
diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py
deleted file mode 100644
index 7ad6b0d7a..000000000
--- a/module/plugins/internal/SevenZip.py
+++ /dev/null
@@ -1,155 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-import re
-
-from subprocess import Popen, PIPE
-
-from module.plugins.internal.UnRar import ArchiveError, CRCError, PasswordError, UnRar, renice
-from module.utils import fs_encode, save_join
-
-
-class SevenZip(UnRar):
- __name__ = "SevenZip"
- __version__ = "0.08"
-
- __description__ = """7-Zip extractor plugin"""
- __license__ = "GPLv3"
- __authors__ = [("Michael Nowak", ""),
- ("Walter Purcaro", "vuolter@gmail.com")]
-
-
- CMD = "7z"
- VERSION = ""
-
- EXTENSIONS = [".7z", ".xz", ".zip", ".gz", ".gzip", ".tgz", ".bz2", ".bzip2",
- ".tbz2", ".tbz", ".tar", ".wim", ".swm", ".lzma", ".rar", ".cab",
- ".arj", ".z", ".taz", ".cpio", ".rpm", ".deb", ".lzh", ".lha",
- ".chm", ".chw", ".hxs", ".iso", ".msi", ".doc", ".xls", ".ppt",
- ".dmg", ".xar", ".hfs", ".exe", ".ntfs", ".fat", ".vhd", ".mbr",
- ".squashfs", ".cramfs", ".scap"]
-
-
- #@NOTE: there are some more uncovered 7z formats
- re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)')
- re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password)', re.I)
- re_wrongcrc = re.compile(r'Encrypted\s+\=\s+\+', re.I)
- re_version = re.compile(r'7-Zip\s(?:\[64\]\s)?(\d+\.\d+)', re.I)
-
-
- @classmethod
- def isUsable(cls):
- if os.name == "nt":
- cls.CMD = os.path.join(pypath, "7z.exe")
- p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
- out,err = p.communicate()
- else:
- p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
- out, err = p.communicate()
-
- cls.VERSION = cls.re_version.search(out).group(1)
-
- return True
-
-
- def check(self):
- file = fs_encode(self.filename)
-
- p = self.call_cmd("t", file)
- out, err = p.communicate()
-
- if p.returncode > 1:
- raise CRCError(err)
-
- p = self.call_cmd("l", "-slt", file)
- out, err = p.communicate()
-
- if p.returncode > 1:
- raise ArchiveError(_("Process return code: %d") % p.returncode)
-
- # check if output or error macthes the 'wrong password'-Regexp
- if self.re_wrongpwd.search(out):
- raise PasswordError
-
- # check if output matches 'Encrypted = +'
- if self.re_wrongcrc.search(out):
- raise CRCError(_("Header protected"))
-
-
- def isPassword(self, password):
- p = self.call_cmd("l", fs_encode(self.filename), password=password)
- p.communicate()
- return p.returncode == 0
-
-
- def repair(self):
- return False
-
-
- def extract(self, password=None):
- command = "x" if self.fullpath else "e"
-
- p = self.call_cmd(command, '-o' + self.out, fs_encode(self.filename), password=password)
-
- renice(p.pid, self.renice)
-
- # communicate and retrieve stderr
- self._progress(p)
- err = p.stderr.read().strip()
-
- if err:
- if self.re_wrongpwd.search(err):
- raise PasswordError
-
- elif self.re_wrongcrc.search(err):
- raise CRCError(err)
-
- else: #: raise error if anything is on stderr
- raise ArchiveError(err)
-
- if p.returncode > 1:
- raise ArchiveError(_("Process return code: %d") % p.returncode)
-
- self.files = self.list(password)
-
-
- def list(self, password=None):
- command = "l" if self.fullpath else "l"
-
- p = self.call_cmd(command, fs_encode(self.filename), password=password)
- out, err = p.communicate()
-
- if "Can not open" in err:
- raise ArchiveError(_("Cannot open file"))
-
- if p.returncode > 1:
- raise ArchiveError(_("Process return code: %d") % p.returncode)
-
- result = set()
- for groups in self.re_filelist.findall(out):
- f = groups[-1].strip()
- result.add(save_join(self.out, f))
-
- return list(result)
-
-
- def call_cmd(self, command, *xargs, **kwargs):
- args = []
-
- #overwrite flag
- if self.overwrite:
- args.append("-y")
-
- #set a password
- if "password" in kwargs and kwargs["password"]:
- args.append("-p'%s'" % kwargs["password"])
- else:
- args.append("-p-")
-
- #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue
- call = [self.CMD, command] + args + list(xargs)
-
- self.manager.logDebug(" ".join(call))
-
- p = Popen(call, stdout=PIPE, stderr=PIPE)
- return p
diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py
deleted file mode 100644
index dc34f864f..000000000
--- a/module/plugins/internal/SimpleCrypter.py
+++ /dev/null
@@ -1,157 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-
-from urlparse import urljoin, urlparse
-
-from pyload.plugin.Crypter import Crypter
-from pyload.plugin.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies
-from pyload.utils import fixup
-
-
-class SimpleCrypter(Crypter, SimpleHoster):
- __name__ = "SimpleCrypter"
- __type__ = "crypter"
- __version__ = "0.43"
-
- __pattern__ = r'^unmatchable$'
- __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package']
- ("subfolder_per_package", "bool", "Create a subfolder for each package", True)]
-
- __description__ = """Simple decrypter plugin"""
- __license__ = "GPLv3"
- __authors__ = [("stickell", "l.stickell@yahoo.it"),
- ("zoidberg", "zoidberg@mujmail.cz"),
- ("Walter Purcaro", "vuolter@gmail.com")]
-
-
- """
- Following patterns should be defined by each crypter:
-
- LINK_PATTERN: Download link or regex to catch links in group(1)
- example: LINK_PATTERN = r'<div class="link"><a href="(.+?)"'
-
- NAME_PATTERN: (optional) folder name or page title
- example: NAME_PATTERN = r'<title>Files of: (?P<N>[^<]+) folder</title>'
-
- OFFLINE_PATTERN: (optional) Checks if the page is unreachable
- example: OFFLINE_PATTERN = r'File (deleted|not found)'
-
- TEMP_OFFLINE_PATTERN: (optional) Checks if the page is temporarily unreachable
- example: TEMP_OFFLINE_PATTERN = r'Server maintainance'
-
-
- You can override the getLinks method if you need a more sophisticated way to extract the links.
-
-
- If the links are splitted on multiple pages you can define the PAGES_PATTERN regex:
-
- PAGES_PATTERN: (optional) group(1) should be the number of overall pages containing the links
- example: PAGES_PATTERN = r'Pages: (\d+)'
-
- and its loadPage method:
-
-
- def loadPage(self, page_n):
- return the html of the page number page_n
- """
-
- LINK_PATTERN = None
-
- NAME_REPLACEMENTS = [("&#?\w+;", fixup)]
- URL_REPLACEMENTS = []
-
- TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct
- COOKIES = True #: or False or list of tuples [(domain, name, value)]
-
- LOGIN_ACCOUNT = False
- LOGIN_PREMIUM = False
-
-
- def prepare(self):
- self.pyfile.error = "" #@TODO: Remove in 0.4.10
-
- self.info = {}
- self.html = ""
- self.links = [] #@TODO: Move to hoster class in 0.4.10
-
- if self.LOGIN_PREMIUM and not self.premium:
- self.fail(_("Required premium account not found"))
-
- if self.LOGIN_ACCOUNT and not self.account:
- self.fail(_("Required account not found"))
-
- self.req.setOption("timeout", 120)
-
- if isinstance(self.COOKIES, list):
- set_cookies(self.req.cj, self.COOKIES)
-
- self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS)
-
-
- def decrypt(self, pyfile):
- self.prepare()
-
- self.preload()
- self.checkInfo()
-
- self.links = self.getLinks()
-
- if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'):
- self.handlePages(pyfile)
-
- self.logDebug("Package has %d links" % len(self.links))
-
- if self.links:
- self.packages = [(self.info['name'], self.links, self.info['folder'])]
-
- elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10
- self.fail(_("No link grabbed"))
-
-
- def checkNameSize(self, getinfo=True):
- if not self.info or getinfo:
- self.logDebug("File info (BEFORE): %s" % self.info)
- self.info.update(self.getInfo(self.pyfile.url, self.html))
- self.logDebug("File info (AFTER): %s" % self.info)
-
- try:
- url = self.info['url'].strip()
- name = self.info['name'].strip()
- if name and name != url:
- self.pyfile.name = name
-
- except Exception:
- pass
-
- try:
- folder = self.info['folder'] = self.pyfile.name
-
- except Exception:
- pass
-
- self.logDebug("File name: %s" % self.pyfile.name,
- "File folder: %s" % self.pyfile.name)
-
-
- def getLinks(self):
- """
- Returns the links extracted from self.html
- You should override this only if it's impossible to extract links using only the LINK_PATTERN.
- """
- url_p = urlparse(self.pyfile.url)
- baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
-
- return [urljoin(baseurl, link) if not urlparse(link).scheme else link \
- for link in re.findall(self.LINK_PATTERN, self.html)]
-
-
- def handlePages(self, pyfile):
- try:
- pages = int(re.search(self.PAGES_PATTERN, self.html).group(1))
- except Exception:
- pages = 1
-
- for p in xrange(2, pages + 1):
- self.html = self.loadPage(p)
- self.links += self.getLinks()
diff --git a/module/plugins/internal/SimpleDereferer.py b/module/plugins/internal/SimpleDereferer.py
deleted file mode 100644
index bd00f5d25..000000000
--- a/module/plugins/internal/SimpleDereferer.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-
-from urllib import unquote
-
-from module.plugins.Crypter import Crypter
-from module.plugins.internal.SimpleHoster import fileUrl, set_cookies
-
-
-class SimpleDereferer(Crypter):
- __name__ = "SimpleDereferer"
- __type__ = "crypter"
- __version__ = "0.07"
-
- __pattern__ = r'^unmatchable$'
- __config__ = [("use_subfolder", "bool", "Save package to subfolder", True),
- ("subfolder_per_package", "bool", "Create a subfolder for each package", True)]
-
- __description__ = """Simple dereferer plugin"""
- __license__ = "GPLv3"
- __authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
-
-
- """
- Following patterns should be defined by each crypter:
-
- LINK_PATTERN: Regex to catch the redirect url in group(1)
- example: LINK_PATTERN = r'<div class="link"><a href="(.+?)"'
-
- OFFLINE_PATTERN: (optional) Checks if the page is unreachable
- example: OFFLINE_PATTERN = r'File (deleted|not found)'
-
- TEMP_OFFLINE_PATTERN: (optional) Checks if the page is temporarily unreachable
- example: TEMP_OFFLINE_PATTERN = r'Server maintainance'
-
-
- You can override the getLinks method if you need a more sophisticated way to extract the redirect url.
- """
-
- LINK_PATTERN = None
-
- TEXT_ENCODING = False
- COOKIES = True
-
-
- def decrypt(self, pyfile):
- link = fileUrl(self, pyfile.url)
-
- if not link:
- try:
- link = unquote(re.match(self.__pattern__, pyfile.url).group('LINK'))
-
- except Exception:
- self.prepare()
- self.preload()
- self.checkStatus()
-
- link = self.getLink()
-
- if link.strip():
- self.urls = [link.strip()] #@TODO: Remove `.strip()` in 0.4.10
-
- elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10
- self.fail(_("No link grabbed"))
-
-
- def prepare(self):
- self.info = {}
- self.html = ""
-
- self.req.setOption("timeout", 120)
-
- if isinstance(self.COOKIES, list):
- set_cookies(self.req.cj, self.COOKIES)
-
-
- def preload(self):
- self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING)
-
- if isinstance(self.TEXT_ENCODING, basestring):
- self.html = unicode(self.html, self.TEXT_ENCODING)
-
-
- def checkStatus(self):
- if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, self.html):
- self.offline()
-
- elif hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, self.html):
- self.tempOffline()
-
-
- def getLink(self):
- try:
- return re.search(self.LINK_PATTERN, self.html).group(1)
-
- except Exception:
- pass
diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py
deleted file mode 100644
index e4ff1a2d8..000000000
--- a/module/plugins/internal/SimpleHoster.py
+++ /dev/null
@@ -1,701 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import mimetypes
-import os
-import re
-
-from datetime import datetime, timedelta
-from inspect import isclass
-from time import time
-from urllib import unquote
-from urlparse import urljoin, urlparse
-
-from module.PyFile import statusMap as _statusMap
-from module.network.CookieJar import CookieJar
-from module.network.HTTPRequest import BadHeader
-from module.network.RequestFactory import getURL
-from module.plugins.Hoster import Hoster
-from module.plugins.Plugin import Fail
-from module.utils import fixup, fs_encode, parseFileSize
-
-
-#@TODO: Adapt and move to PyFile in 0.4.10
-statusMap = dict((v, k) for k, v in _statusMap.iteritems())
-
-
-#@TODO: Remove in 0.4.10 and redirect to self.error instead
-def _error(self, reason, type):
- if not reason and not type:
- type = "unknown"
-
- msg = _("%s error") % type.strip().capitalize() if type else _("Error")
- msg += ": %s" % reason.strip() if reason else ""
- msg += _(" | Plugin may be out of date")
-
- raise Fail(msg)
-
-
-#@TODO: Remove in 0.4.10
-def _wait(self, seconds, reconnect):
- if seconds:
- self.setWait(int(seconds) + 1)
-
- if reconnect is not None:
- self.wantReconnect = reconnect
-
- super(SimpleHoster, self).wait()
-
-
-def replace_patterns(string, ruleslist):
- for r in ruleslist:
- rf, rt = r
- string = re.sub(rf, rt, string)
- return string
-
-
-def set_cookies(cj, cookies):
- for cookie in cookies:
- if isinstance(cookie, tuple) and len(cookie) == 3:
- domain, name, value = cookie
- cj.setCookie(domain, name, value)
-
-
-def parseHtmlTagAttrValue(attr_name, tag):
- m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^>\s\"'][^>\s]*)\1" % attr_name, tag, re.I)
- return m.group(2) if m else None
-
-
-def parseHtmlForm(attr_str, html, input_names={}):
- for form in re.finditer(r"(?P<TAG><form[^>]*%s[^>]*>)(?P<CONTENT>.*?)</?(form|body|html)[^>]*>" % attr_str,
- html, re.S | re.I):
- inputs = {}
- action = parseHtmlTagAttrValue("action", form.group('TAG'))
-
- for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.S | re.I):
- name = parseHtmlTagAttrValue("name", inputtag.group(1))
- if name:
- value = parseHtmlTagAttrValue("value", inputtag.group(1))
- if not value:
- inputs[name] = inputtag.group(3) or ''
- else:
- inputs[name] = value
-
- if input_names:
- # check input attributes
- for key, val in input_names.iteritems():
- if key in inputs:
- if isinstance(val, basestring) and inputs[key] == val:
- continue
- elif isinstance(val, tuple) and inputs[key] in val:
- continue
- elif hasattr(val, "search") and re.match(val, inputs[key]):
- continue
- break #: attibute value does not match
- else:
- break #: attibute name does not match
- else:
- return action, inputs #: passed attribute check
- else:
- # no attribute check
- return action, inputs
-
- return {}, None #: no matching form found
-
-
-#: Deprecated
-def parseFileInfo(plugin, url="", html=""):
- if hasattr(plugin, "getInfo"):
- info = plugin.getInfo(url, html)
- res = info['name'], info['size'], info['status'], info['url']
- else:
- url = unquote(url)
- url_p = urlparse(url)
- res = ((url_p.path.split('/')[-1]
- or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0]
- or url_p.netloc.split('.', 1)[0]),
- 0,
- 3 if url else 8,
- url)
-
- return res
-
-
-#@TODO: Remove in 0.4.10
-#@NOTE: Every plugin must have own parseInfos classmethod to work with 0.4.10
-def create_getInfo(plugin):
-
- def generator(list):
- for x in list:
- yield x
-
- if hasattr(plugin, "parseInfos"):
- fn = lambda urls: generator((info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfos(urls))
- else:
- fn = lambda urls: generator(parseFileInfo(url) for url in urls)
-
- return fn
-
-
-def timestamp():
- return int(time() * 1000)
-
-
-#@TODO: Move to hoster class in 0.4.10
-def fileUrl(self, url, follow_location=None):
- link = ""
- redirect = 1
-
- if type(follow_location) is int:
- redirect = max(follow_location, 1)
- else:
- redirect = 5
-
- for i in xrange(redirect):
- try:
- self.logDebug("Redirect #%d to: %s" % (i, url))
- header = self.load(url, ref=True, cookies=True, just_header=True, decode=True)
-
- except Exception: #: Bad bad bad...
- req = pyreq.getHTTPRequest()
- res = req.load(url, cookies=True, just_header=True, decode=True)
-
- req.close()
-
- header = {"code": req.code}
- for line in res.splitlines():
- line = line.strip()
- if not line or ":" not in line:
- continue
-
- key, none, value = line.partition(":")
- key = key.lower().strip()
- value = value.strip()
-
- if key in header:
- if type(header[key]) == list:
- header[key].append(value)
- else:
- header[key] = [header[key], value]
- else:
- header[key] = value
-
- if 'content-disposition' in header:
- link = url
-
- elif 'location' in header and header['location'].strip():
- location = header['location']
-
- if not urlparse(location).scheme:
- url_p = urlparse(url)
- baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
- location = urljoin(baseurl, location)
-
- if 'code' in header and header['code'] == 302:
- link = location
-
- if follow_location:
- url = location
- continue
-
- else:
- extension = os.path.splitext(urlparse(url).path.split('/')[-1])[-1]
-
- if 'content-type' in header and header['content-type'].strip():
- mimetype = header['content-type'].split(';')[0].strip()
-
- elif extension:
- mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream"
-
- else:
- mimetype = ""
-
- if mimetype and (link or 'html' not in mimetype):
- link = url
- else:
- link = ""
-
- break
-
- else:
- try:
- self.logError(_("Too many redirects"))
- except Exception:
- pass
-
- return link
-
-
-def secondsToMidnight(gmt=0):
- now = datetime.utcnow() + timedelta(hours=gmt)
-
- if now.hour is 0 and now.minute < 10:
- midnight = now
- else:
- midnight = now + timedelta(days=1)
-
- td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now
-
- if hasattr(td, 'total_seconds'):
- res = td.total_seconds()
- else: #@NOTE: work-around for python 2.5 and 2.6 missing timedelta.total_seconds
- res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
-
- return int(res)
-
-
-class SimpleHoster(Hoster):
- __name__ = "SimpleHoster"
- __type__ = "hoster"
- __version__ = "1.15"
-
- __pattern__ = r'^unmatchable$'
-
- __description__ = """Simple hoster plugin"""
- __license__ = "GPLv3"
- __authors__ = [("zoidberg", "zoidberg@mujmail.cz"),
- ("stickell", "l.stickell@yahoo.it"),
- ("Walter Purcaro", "vuolter@gmail.com")]
-
-
- """
- Info patterns should be defined by each hoster:
-
- INFO_PATTERN: (optional) Name and Size of the file
- example: INFO_PATTERN = r'(?P<N>file_name) (?P<S>file_size) (?P<U>size_unit)'
- or
- NAME_PATTERN: (optional) Name that will be set for the file
- example: NAME_PATTERN = r'(?P<N>file_name)'
- SIZE_PATTERN: (optional) Size that will be checked for the file
- example: SIZE_PATTERN = r'(?P<S>file_size) (?P<U>size_unit)'
-
- HASHSUM_PATTERN: (optional) Hash code and type of the file
- example: HASHSUM_PATTERN = r'(?P<H>hash_code) (?P<T>MD5)'
-
- OFFLINE_PATTERN: (optional) Check if the page is unreachable
- example: OFFLINE_PATTERN = r'File (deleted|not found)'
-
- TEMP_OFFLINE_PATTERN: (optional) Check if the page is temporarily unreachable
- example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)'
-
-
- Error handling patterns are all optional:
-
- WAIT_PATTERN: (optional) Detect waiting time
- example: WAIT_PATTERN = r''
-
- PREMIUM_ONLY_PATTERN: (optional) Check if the file can be downloaded only with a premium account
- example: PREMIUM_ONLY_PATTERN = r'Premium account required'
-
- ERROR_PATTERN: (optional) Detect any error preventing download
- example: ERROR_PATTERN = r''
-
-
- Instead overriding handleFree and handlePremium methods you can define the following patterns for direct download:
-
- LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download
- example: LINK_FREE_PATTERN = r'<div class="link"><a href="(.+?)"'
-
- LINK_PREMIUM_PATTERN: (optional) group(1) should be the direct link for premium download
- example: LINK_PREMIUM_PATTERN = r'<div class="link"><a href="(.+?)"'
- """
-
- NAME_REPLACEMENTS = [("&#?\w+;", fixup)]
- SIZE_REPLACEMENTS = []
- URL_REPLACEMENTS = []
-
- TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct
- COOKIES = True #: or False or list of tuples [(domain, name, value)]
- CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account
- DIRECT_LINK = None #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False
- MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method)
- LOGIN_ACCOUNT = False #: Set to True to require account login
- DISPOSITION = True #: Work-around to `filename*=UTF-8` bug; remove in 0.4.10
-
- directLink = fileUrl #@TODO: Remove in 0.4.10
-
-
- @classmethod
- def parseInfos(cls, urls): #@TODO: Built-in in 0.4.10 core, then remove from plugins
- for url in urls:
- url = replace_patterns(url, cls.URL_REPLACEMENTS)
- yield cls.getInfo(url)
-
-
- @classmethod
- def apiInfo(cls, url="", get={}, post={}):
- url = unquote(url)
- url_p = urlparse(url)
- return {'name' : (url_p.path.split('/')[-1]
- or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0]
- or url_p.netloc.split('.', 1)[0]),
- 'size' : 0,
- 'status': 3 if url else 8,
- 'url' : url}
-
-
- @classmethod
- def getInfo(cls, url="", html=""):
- info = cls.apiInfo(url)
- online = False if info['status'] != 2 else True
-
- try:
- info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here
-
- except Exception:
- info['pattern'] = {}
-
- if not html and not online:
- if not url:
- info['error'] = "missing url"
- info['status'] = 1
-
- elif info['status'] is 3 and not fileUrl(None, url):
- try:
- html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING)
-
- if isinstance(cls.TEXT_ENCODING, basestring):
- html = unicode(html, cls.TEXT_ENCODING)
-
- except BadHeader, e:
- info['error'] = "%d: %s" % (e.code, e.content)
-
- if e.code is 404:
- info['status'] = 1
-
- elif e.code is 503:
- info['status'] = 6
-
- if html:
- if hasattr(cls, "OFFLINE_PATTERN") and re.search(cls.OFFLINE_PATTERN, html):
- info['status'] = 1
-
- elif hasattr(cls, "TEMP_OFFLINE_PATTERN") and re.search(cls.TEMP_OFFLINE_PATTERN, html):
- info['status'] = 6
-
- else:
- for pattern in ("INFO_PATTERN", "NAME_PATTERN", "SIZE_PATTERN", "HASHSUM_PATTERN"):
- try:
- attr = getattr(cls, pattern)
- pdict = re.search(attr, html).groupdict()
-
- if all(True for k in pdict if k not in info['pattern']):
- info['pattern'].update(pdict)
-
- except AttributeError:
- continue
-
- else:
- online = True
-
- if online:
- info['status'] = 2
-
- if 'N' in info['pattern']:
- info['name'] = replace_patterns(unquote(info['pattern']['N'].strip()),
- cls.NAME_REPLACEMENTS)
-
- if 'S' in info['pattern']:
- size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info['pattern'] else info['pattern']['S'],
- cls.SIZE_REPLACEMENTS)
- info['size'] = parseFileSize(size)
-
- elif isinstance(info['size'], basestring):
- unit = info['units'] if 'units' in info else None
- info['size'] = parseFileSize(info['size'], unit)
-
- if 'H' in info['pattern']:
- hashtype = info['pattern']['T'] if 'T' in info['pattern'] else "hash"
- info[hashtype] = info['pattern']['H']
-
- if not info['pattern']:
- info.pop('pattern', None)
-
- return info
-
-
- def setup(self):
- self.resumeDownload = self.multiDL = self.premium
-
-
- def prepare(self):
- self.pyfile.error = "" #@TODO: Remove in 0.4.10
-
- self.info = {}
- self.html = ""
- self.link = "" #@TODO: Move to hoster class in 0.4.10
- self.directDL = False #@TODO: Move to hoster class in 0.4.10
- self.multihost = False #@TODO: Move to hoster class in 0.4.10
-
- if self.LOGIN_ACCOUNT and not self.account:
- self.fail(_("Required account not found"))
-
- self.req.setOption("timeout", 120)
-
- if isinstance(self.COOKIES, list):
- set_cookies(self.req.cj, self.COOKIES)
-
- if (self.MULTI_HOSTER
- and (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern']
- or re.match(self.__pattern__, self.pyfile.url) is None)):
- self.multihost = True
- return
-
- if self.DIRECT_LINK is None:
- self.directDL = bool(self.account)
- else:
- self.directDL = self.DIRECT_LINK
-
- self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS)
-
-
- def preload(self):
- self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING)
-
- if isinstance(self.TEXT_ENCODING, basestring):
- self.html = unicode(self.html, self.TEXT_ENCODING)
-
-
- def process(self, pyfile):
- self.prepare()
- self.checkInfo()
-
- if self.directDL:
- self.logDebug("Looking for direct download link...")
- self.handleDirect(pyfile)
-
- if self.multihost and not self.link and not self.lastDownload:
- self.logDebug("Looking for leeched download link...")
- self.handleMulti(pyfile)
-
- if not self.link and not self.lastDownload:
- self.MULTI_HOSTER = False
- self.retry(1, reason="Multi hoster fails")
-
- if not self.link and not self.lastDownload:
- self.preload()
- self.checkInfo()
-
- if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
- self.logDebug("Handled as premium download")
- self.handlePremium(pyfile)
-
- elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
- self.logDebug("Handled as free download")
- self.handleFree(pyfile)
-
- self.downloadLink(self.link, self.DISPOSITION) #: Remove `self.DISPOSITION` in 0.4.10
- self.checkFile()
-
-
- def downloadLink(self, link, disposition=True):
- if link and isinstance(link, basestring):
- self.correctCaptcha()
-
- if not urlparse(link).scheme:
- url_p = urlparse(self.pyfile.url)
- baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
- link = urljoin(baseurl, link)
-
- self.download(link, ref=False, disposition=disposition)
-
-
- def checkFile(self):
- if self.cTask and not self.lastDownload:
- self.invalidCaptcha()
- self.retry(10, reason=_("Wrong captcha"))
-
- elif not self.lastDownload or not os.path.exists(fs_encode(self.lastDownload)):
- self.lastDownload = ""
- self.error(self.pyfile.error or _("No file downloaded"))
-
- else:
- rules = {'empty file': re.compile(r'\A\Z'),
- 'html file' : re.compile(r'\A\s*<!DOCTYPE html'),
- 'html error': re.compile(r'\A\s*(<.+>)?\d{3}(\Z|\s+)')}
-
- if hasattr(self, 'ERROR_PATTERN'):
- rules['error'] = re.compile(self.ERROR_PATTERN)
-
- check = self.checkDownload(rules)
- if check: #@TODO: Move to hoster in 0.4.10
- errmsg = check.strip().capitalize()
- if self.lastCheck:
- errmsg += " | " + self.lastCheck.group(0).strip()
-
- self.lastDownload = ""
- self.retry(10, 60, errmsg)
-
-
- def checkErrors(self):
- if not self.html:
- self.logWarning(_("No html code to check"))
- return
-
- if hasattr(self, 'PREMIUM_ONLY_PATTERN') and not self.premium and re.search(self.PREMIUM_ONLY_PATTERN, self.html):
- self.fail(_("Link require a premium account to be handled"))
-
- elif hasattr(self, 'ERROR_PATTERN'):
- m = re.search(self.ERROR_PATTERN, self.html)
- if m:
- errmsg = self.info['error'] = m.group(1)
- self.error(errmsg)
-
- elif hasattr(self, 'WAIT_PATTERN'):
- m = re.search(self.WAIT_PATTERN, self.html)
- if m:
- wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in
- re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I))
- self.wait(wait_time, wait_time > 300)
- return
-
- self.info.pop('error', None)
-
-
- def checkStatus(self, getinfo=True):
- if not self.info or getinfo:
- self.logDebug("Update file info...")
- self.logDebug("Previous file info: %s" % self.info)
- self.info.update(self.getInfo(self.pyfile.url, self.html))
- self.logDebug("Current file info: %s" % self.info)
-
- try:
- status = self.info['status']
-
- if status is 1:
- self.offline()
-
- elif status is 6:
- self.tempOffline()
-
- elif status is 8:
- self.fail()
-
- finally:
- self.logDebug("File status: %s" % statusMap[status])
-
-
- def checkNameSize(self, getinfo=True):
- if not self.info or getinfo:
- self.logDebug("Update file info...")
- self.logDebug("Previous file info: %s" % self.info)
- self.info.update(self.getInfo(self.pyfile.url, self.html))
- self.logDebug("Current file info: %s" % self.info)
-
- try:
- url = self.info['url'].strip()
- name = self.info['name'].strip()
- if name and name != url:
- self.pyfile.name = name
-
- except Exception:
- pass
-
- try:
- size = self.info['size']
- if size > 0:
- self.pyfile.size = size
-
- except Exception:
- pass
-
- self.logDebug("File name: %s" % self.pyfile.name,
- "File size: %s byte" % self.pyfile.size if self.pyfile.size > 0 else "File size: Unknown")
-
-
- def checkInfo(self):
- self.checkNameSize()
-
- if self.html:
- self.checkErrors()
- self.checkNameSize()
-
- self.checkStatus(getinfo=False)
-
-
- #: Deprecated
- def getFileInfo(self):
- self.info = {}
- self.checkInfo()
- return self.info
-
-
- def handleDirect(self, pyfile):
- link = self.directLink(pyfile.url, self.resumeDownload)
-
- if link:
- self.logInfo(_("Direct download link detected"))
-
- self.link = link
- else:
- self.logDebug("Direct download link not found")
-
-
- def handleMulti(self, pyfile): #: Multi-hoster handler
- pass
-
-
- def handleFree(self, pyfile):
- if not hasattr(self, 'LINK_FREE_PATTERN'):
- self.logError(_("Free download not implemented"))
-
- m = re.search(self.LINK_FREE_PATTERN, self.html)
- if m is None:
- self.error(_("Free download link not found"))
- else:
- self.link = m.group(1)
-
-
- def handlePremium(self, pyfile):
- if not hasattr(self, 'LINK_PREMIUM_PATTERN'):
- self.logError(_("Premium download not implemented"))
- self.logDebug("Handled as free download")
- self.handleFree(pyfile)
-
- m = re.search(self.LINK_PREMIUM_PATTERN, self.html)
- if m is None:
- self.error(_("Premium download link not found"))
- else:
- self.link = m.group(1)
-
-
- def longWait(self, wait_time=None, max_tries=3):
- if wait_time and isinstance(wait_time, (int, long, float)):
- time_str = "%dh %dm" % divmod(wait_time / 60, 60)
- else:
- wait_time = 900
- time_str = _("(unknown time)")
- max_tries = 100
-
- self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str)
-
- self.wait(wait_time, True)
- self.retry(max_tries=max_tries, reason=_("Download limit reached"))
-
-
- def parseHtmlForm(self, attr_str="", input_names={}):
- return parseHtmlForm(attr_str, self.html, input_names)
-
-
- def checkTrafficLeft(self):
- if not self.account:
- return True
-
- traffic = self.account.getAccountInfo(self.user, True)['trafficleft']
-
- if traffic is None:
- return False
- elif traffic == -1:
- return True
- else:
- size = self.pyfile.size / 1024
- self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic))
- return size <= traffic
-
-
- #@TODO: Remove in 0.4.10
- def wait(self, seconds=0, reconnect=None):
- return _wait(self, seconds, reconnect)
-
-
- def error(self, reason="", type="parse"):
- return _error(self, reason, type)
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
deleted file mode 100644
index 54d64c430..000000000
--- a/module/plugins/internal/UnRar.py
+++ /dev/null
@@ -1,248 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-import re
-
-from glob import glob
-from string import digits
-from subprocess import Popen, PIPE
-
-from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError
-from module.utils import decode, fs_encode, save_join
-
-
-def renice(pid, value):
- if value and os.name != "nt":
- try:
- Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1)
-
- except Exception:
- pass
-
-
-class UnRar(Extractor):
- __name__ = "UnRar"
- __version__ = "1.13"
-
- __description__ = """Rar extractor plugin"""
- __license__ = "GPLv3"
- __authors__ = [("RaNaN", "RaNaN@pyload.org"),
- ("Walter Purcaro", "vuolter@gmail.com"),
- ("Immenz", "immenz@gmx.net"),]
-
-
- CMD = "unrar"
- VERSION = ""
-
- EXTENSIONS = [".rar"]
-
-
- re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?',re.I)
-
- re_filefixed = re.compile(r'Building (.+)')
- re_filelist = re.compile(r'^(.)(\s*[\w\.\-]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I)
-
- re_wrongpwd = re.compile(r'password', re.I)
- re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I)
-
- re_version = re.compile(r'UNRAR\s(\d+\.\d+)', re.I)
-
-
- @classmethod
- def isUsable(cls):
- if os.name == "nt":
- cls.CMD = os.path.join(pypath, "UnRAR.exe")
- p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
- out, err = p.communicate()
- else:
- try:
- p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
- out, err = p.communicate()
-
- except OSError: #: fallback to rar
- cls.CMD = "rar"
- p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
- out, err = p.communicate()
-
- cls.VERSION = cls.re_version.search(out).group(1)
-
- return True
-
-
- @classmethod
- def isMultipart(cls,filename):
- multipart = cls.re_multipart.search(filename)
- if multipart:
- # First Multipart file (part1.rar for *.part1-9.rar format or *.rar for .r1-9 format) handled as normal Archive
- return False if (multipart.group(1) == "part" and int(multipart.group(2)) == 1) else True
-
- return False
-
-
- def check(self):
- p = self.call_cmd("l", "-v", fs_encode(self.filename))
- out, err = p.communicate()
-
- if self.re_wrongpwd.search(err):
- raise PasswordError
-
- if self.re_wrongcrc.search(err):
- raise CRCError(err)
-
- # output only used to check if passworded files are present
- for attr in self.re_filelist.findall(out):
- if attr[0].startswith("*"):
- raise PasswordError
-
-
- def isPassword(self, password):
- # at this point we can only verify header protected files
- p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password)
- out, err = p.communicate()
- return False if self.re_wrongpwd.search(err) else True
-
-
- def repair(self):
- p = self.call_cmd("rc", fs_encode(self.filename))
-
- # communicate and retrieve stderr
- self._progress(p)
- err = p.stderr.read().strip()
-
- if err or p.returncode:
- p = self.call_cmd("r", fs_encode(self.filename))
-
- # communicate and retrieve stderr
- self._progress(p)
- err = p.stderr.read().strip()
-
- if err or p.returncode:
- return False
- else:
- dir = os.path.dirname(filename)
- name = re_filefixed.search(out).group(1)
-
- self.filename = os.path.join(dir, name)
-
- return True
-
-
- def _progress(self, process):
- s = ""
- while True:
- c = process.stdout.read(1)
- # quit loop on eof
- if not c:
- break
- # reading a percentage sign -> set progress and restart
- if c == '%':
- self.notifyProgress(int(s))
- s = ""
- # not reading a digit -> therefore restart
- elif c not in digits:
- s = ""
- # add digit to progressstring
- else:
- s += c
-
-
- def extract(self, password=None):
- command = "x" if self.fullpath else "e"
-
- p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password)
-
- renice(p.pid, self.renice)
-
- # communicate and retrieve stderr
- self._progress(p)
- err = p.stderr.read().strip()
-
- if err:
- if self.re_wrongpwd.search(err):
- raise PasswordError
-
- elif self.re_wrongcrc.search(err):
- raise CRCError(err)
-
- else: #: raise error if anything is on stderr
- raise ArchiveError(err)
-
- if p.returncode:
- raise ArchiveError(_("Process return code: %d") % p.returncode)
-
- self.files = self.list(password)
-
-
- def getDeleteFiles(self):
- dir, name = os.path.split(self.filename)
-
- # actually extracted file
- files = [self.filename]
-
- # eventually Multipart Files
- files.extend(save_join(dir, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir))
- if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file))
-
- return files
-
-
- def list(self, password=None):
- command = "vb" if self.fullpath else "lb"
-
- p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password)
- out, err = p.communicate()
-
- if "Cannot open" in err:
- raise ArchiveError(_("Cannot open file"))
-
- if err.strip(): #: only log error at this point
- self.manager.logError(err.strip())
-
- result = set()
- if not self.fullpath and self.VERSION.startswith('5'):
- # NOTE: Unrar 5 always list full path
- for f in decode(out).splitlines():
- f = save_join(self.out, os.path.basename(f.strip()))
- if os.path.isfile(f):
- result.add(save_join(self.out, os.path.basename(f)))
- else:
- for f in decode(out).splitlines():
- f = f.strip()
- result.add(save_join(self.out, f))
-
- return list(result)
-
-
- def call_cmd(self, command, *xargs, **kwargs):
- args = []
-
- # overwrite flag
- if self.overwrite:
- args.append("-o+")
- else:
- args.append("-o-")
- if self.delete:
- args.append("-or")
-
- for word in self.excludefiles:
- args.append("-x'%s'" % word.strip())
-
- # assume yes on all queries
- args.append("-y")
-
- # set a password
- if "password" in kwargs and kwargs['password']:
- args.append("-p%s" % kwargs['password'])
- else:
- args.append("-p-")
-
- if self.keepbroken:
- args.append("-kb")
-
- # NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue
- call = [self.CMD, command] + args + list(xargs)
-
- self.manager.logDebug(" ".join(call))
-
- p = Popen(call, stdout=PIPE, stderr=PIPE)
- return p
diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py
deleted file mode 100644
index 704b49ca0..000000000
--- a/module/plugins/internal/UnZip.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from __future__ import with_statement
-
-import os
-import sys
-import zipfile
-
-from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError
-from module.utils import fs_encode
-
-
-class UnZip(Extractor):
- __name__ = "UnZip"
- __version__ = "1.10"
-
- __description__ = """Zip extractor plugin"""
- __license__ = "GPLv3"
- __authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
-
-
- EXTENSIONS = [".zip", ".zip64"]
- VERSION ="(python %s.%s.%s)" % (sys.version_info[0], sys.version_info[1], sys.version_info[2])
-
-
- @classmethod
- def isUsable(cls):
- return sys.version_info[:2] >= (2, 6)
-
-
- def list(self, password=None):
- with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z:
- z.setpassword(password)
- return z.namelist()
-
-
- def check(self):
- with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z:
- badfile = z.testzip()
-
- if badfile:
- raise CRCError(badfile)
- else:
- raise PasswordError
-
-
- def extract(self, password=None):
- try:
- with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z:
- z.setpassword(password)
-
- badfile = z.testzip()
-
- if badfile:
- raise CRCError(badfile)
- else:
- z.extractall(self.out)
-
- except (zipfile.BadZipfile, zipfile.LargeZipFile), e:
- raise ArchiveError(e)
-
- except RuntimeError, e:
- if "encrypted" in e:
- raise PasswordError
- else:
- raise ArchiveError(e)
- else:
- self.files = z.namelist()
diff --git a/module/plugins/internal/UpdateManager.py b/module/plugins/internal/UpdateManager.py
deleted file mode 100644
index 7bbfc0c79..000000000
--- a/module/plugins/internal/UpdateManager.py
+++ /dev/null
@@ -1,300 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-import sys
-
-from operator import itemgetter
-from os import path, remove, stat
-
-from pyload.network.RequestFactory import getURL
-from pyload.plugin.Addon import Expose, Addon, threaded
-from pyload.utils import safe_join
-
-
-class UpdateManager(Addon):
- __name__ = "UpdateManager"
- __type__ = "addon"
- __version__ = "0.40"
-
- __config__ = [("activated" , "bool" , "Activated" , True ),
- ("mode" , "pyLoad + plugins;plugins only", "Check updates for" , "pyLoad + plugins"),
- ("interval" , "int" , "Check interval in hours" , 8 ),
- ("reloadplugins", "bool" , "Monitor plugins for code changes (debug mode only)", True ),
- ("nodebugupdate", "bool" , "Don't check for updates in debug mode" , True )]
-
- __description__ = """Check for updates"""
- __license__ = "GPLv3"
- __authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
-
-
- # event_list = ["pluginConfigChanged"]
-
- SERVER_URL = "http://updatemanager.pyload.org"
- MIN_INTERVAL = 6 * 60 * 60 #: 6h minimum check interval (value is in seconds)
-
-
- def pluginConfigChanged(self, plugin, name, value):
- if name == "interval":
- interval = value * 60 * 60
- if self.MIN_INTERVAL <= interval != self.interval:
- self.core.scheduler.removeJob(self.cb)
- self.interval = interval
- self.initPeriodical()
- else:
- self.logDebug("Invalid interval value, kept current")
-
- elif name == "reloadplugins":
- if self.cb2:
- self.core.scheduler.removeJob(self.cb2)
- if value is True and self.core.debug:
- self.periodical2()
-
-
- def activate(self):
- self.pluginConfigChanged(self.__name__, "interval", self.getConfig("interval"))
- x = lambda: self.pluginConfigChanged(self.__name__, "reloadplugins", self.getConfig("reloadplugins"))
- self.core.scheduler.addJob(10, x, threaded=False)
-
-
- def deactivate(self):
- self.pluginConfigChanged(self.__name__, "reloadplugins", False)
-
-
- def setup(self):
- self.cb2 = None
- self.interval = self.MIN_INTERVAL
- self.updating = False
- self.info = {'pyload': False, 'version': None, 'plugins': False}
- self.mtimes = {} #: store modification time for each plugin
-
-
- def periodical2(self):
- if not self.updating:
- self.autoreloadPlugins()
-
- self.cb2 = self.core.scheduler.addJob(4, self.periodical2, threaded=False)
-
-
- @Expose
- def autoreloadPlugins(self):
- """ reload and reindex all modified plugins """
- modules = filter(
- lambda m: m and (m.__name__.startswith("pyload.plugin.") or
- m.__name__.startswith("userplugins.")) and
- m.__name__.count(".") >= 2, sys.modules.itervalues()
- )
-
- reloads = []
-
- for m in modules:
- root, type, name = m.__name__.rsplit(".", 2)
- id = (type, name)
- if type in self.core.pluginManager.plugins:
- f = m.__file__.replace(".pyc", ".py")
- if not path.isfile(f):
- continue
-
- mtime = stat(f).st_mtime
-
- if id not in self.mtimes:
- self.mtimes[id] = mtime
- elif self.mtimes[id] < mtime:
- reloads.append(id)
- self.mtimes[id] = mtime
-
- return True if self.core.pluginManager.reloadPlugins(reloads) else False
-
-
- def periodical(self):
- if self.info['pyload'] or self.getConfig("nodebugupdate") and self.core.debug:
- return
-
- self.updateThread()
-
-
- def server_request(self):
- try:
- return getURL(self.SERVER_URL, get={'v': self.core.api.getServerVersion()}).splitlines()
- except Exception:
- self.logWarning(_("Unable to contact server to get updates"))
-
-
- @threaded
- def updateThread(self):
- self.updating = True
-
- status = self.update(onlyplugin=self.getConfig("mode") == "plugins only")
-
- if status == 2:
- self.core.api.restart()
- else:
- self.updating = False
-
-
- @Expose
- def updatePlugins(self):
- """ simple wrapper for calling plugin update quickly """
- return self.update(onlyplugin=True)
-
-
- @Expose
- def update(self, onlyplugin=False):
- """ check for updates """
- data = self.server_request()
-
- if not data:
- exitcode = 0
-
- elif data[0] == "None":
- self.logInfo(_("No new pyLoad version available"))
- updates = data[1:]
- exitcode = self._updatePlugins(updates)
-
- elif onlyplugin:
- exitcode = 0
-
- else:
- newversion = data[0]
- self.logInfo(_("*** New pyLoad Version %s available ***") % newversion)
- self.logInfo(_("*** Get it here: https://github.com/pyload/pyload/releases ***"))
- exitcode = 3
- self.info['pyload'] = True
- self.info['version'] = newversion
-
- return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required; 3 = No plugins updated, new pyLoad version available
-
-
- def _updatePlugins(self, updates):
- """ check for plugin updates """
-
- if self.info['plugins']:
- return False #: plugins were already updated
-
- exitcode = 0
- updated = []
-
- vre = re.compile(r'__version__.*=.*("|\')([\d.]+)')
- url = updates[0]
- schema = updates[1].split('|')
-
- if "BLACKLIST" in updates:
- blacklist = updates[updates.index('BLACKLIST') + 1:]
- updates = updates[2:updates.index('BLACKLIST')]
- else:
- blacklist = None
- updates = updates[2:]
-
- upgradable = sorted(map(lambda x: dict(zip(schema, x.split('|'))), updates),
- key=itemgetter("type", "name"))
-
- for plugin in upgradable:
- filename = plugin['name']
- type = plugin['type']
- version = plugin['version']
-
- if filename.endswith(".pyc"):
- name = filename[:filename.find("_")]
- else:
- name = filename.replace(".py", "")
-
- plugins = getattr(self.core.pluginManager, "%sPlugins" % type)
-
- oldver = float(plugins[name]['version']) if name in plugins else None
- newver = float(version)
-
- if not oldver:
- msg = "New plugin: [%(type)s] %(name)s (v%(newver).2f)"
- elif newver > oldver:
- msg = "New version of plugin: [%(type)s] %(name)s (v%(oldver).2f -> v%(newver).2f)"
- else:
- continue
-
- self.logInfo(_(msg) % {'type' : type,
- 'name' : name,
- 'oldver': oldver,
- 'newver': newver})
- try:
- content = getURL(url % plugin)
- m = vre.search(content)
-
- if m and m.group(2) == version:
- f = open(safe_join("userplugins", prefix, filename), "wb")
- f.write(content)
- f.close()
- updated.append((prefix, name))
- else:
- raise Exception, _("Version mismatch")
-
- except Exception, e:
- self.logError(_("Error updating plugin %s") % filename, e)
-
- if blacklist:
- blacklisted = map(lambda x: (x.split('|')[0], x.split('|')[1].rsplit('.', 1)[0]), blacklist)
-
- # Always protect internal plugins from removing
- for i, n, t in blacklisted.enumerate():
- if t == "internal":
- del blacklisted[i]
-
- blacklisted = sorted(blacklisted)
- removed = self.removePlugins(blacklisted)
- for t, n in removed:
- self.logInfo(_("Removed blacklisted plugin [%(type)s] %(name)s") % {
- 'type': t,
- 'name': n,
- })
-
- if updated:
- reloaded = self.core.pluginManager.reloadPlugins(updated)
- if reloaded:
- self.logInfo(_("Plugins updated and reloaded"))
- exitcode = 1
- else:
- self.logInfo(_("*** Plugins have been updated, but need a pyLoad restart to be reloaded ***"))
- self.info['plugins'] = True
- exitcode = 2
- else:
- self.logInfo(_("No plugin updates available"))
-
- return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required
-
-
- @Expose
- def removePlugins(self, type_plugins):
- """ delete plugins from disk """
-
- if not type_plugins:
- return
-
- self.logDebug("Requested deletion of plugins: %s" % type_plugins)
-
- removed = []
-
- for type, name in type_plugins:
- err = False
- file = name + ".py"
-
- for root in ("userplugins", path.join(pypath, "pyload", "plugins")):
-
- filename = safe_join(root, type, file)
- try:
- remove(filename)
- except Exception, e:
- self.logDebug("Error deleting: %s" % path.basename(filename), e)
- err = True
-
- filename += "c"
- if path.isfile(filename):
- try:
- if type == "addon":
- self.manager.deactivateAddon(name)
- remove(filename)
- except Exception, e:
- self.logDebug("Error deleting: %s" % path.basename(filename), e)
- err = True
-
- if not err:
- id = (type, name)
- removed.append(id)
-
- return removed #: return a list of the plugins successfully removed
diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py
deleted file mode 100644
index 2e6b7dc50..000000000
--- a/module/plugins/internal/XFSAccount.py
+++ /dev/null
@@ -1,174 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-
-from time import gmtime, mktime, strptime
-from urlparse import urljoin
-
-from pyload.plugin.Account import Account
-from pyload.plugin.internal.SimpleHoster import parseHtmlForm, set_cookies
-
-
-class XFSAccount(Account):
- __name__ = "XFSAccount"
- __type__ = "account"
- __version__ = "0.36"
-
- __description__ = """XFileSharing account plugin"""
- __license__ = "GPLv3"
- __authors__ = [("zoidberg", "zoidberg@mujmail.cz"),
- ("Walter Purcaro", "vuolter@gmail.com")]
-
-
- HOSTER_DOMAIN = None
- HOSTER_URL = None
- LOGIN_URL = None
-
- COOKIES = True
-
- PREMIUM_PATTERN = r'\(Premium only\)'
-
- VALID_UNTIL_PATTERN = r'Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})'
-
- TRAFFIC_LEFT_PATTERN = r'Traffic available today:.*?<b>\s*(?P<S>[\d.,]+|[Uu]nlimited)\s*(?:(?P<U>[\w^_]+)\s*)?</b>'
- TRAFFIC_LEFT_UNIT = "MB" #: used only if no group <U> was found
-
- LEECH_TRAFFIC_PATTERN = r'Leech Traffic left:<b>.*?(?P<S>[\d.,]+|[Uu]nlimited)\s*(?:(?P<U>[\w^_]+)\s*)?</b>'
- LEECH_TRAFFIC_UNIT = "MB" #: used only if no group <U> was found
-
- LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<'
-
-
- def init(self):
- if not self.HOSTER_DOMAIN:
- self.logError(_("Missing HOSTER_DOMAIN"))
- self.COOKIES = False
-
- else:
- if not self.HOSTER_URL:
- self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN
-
- if isinstance(self.COOKIES, list):
- self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english"))
- set_cookies(req.cj, self.COOKIES)
-
-
- def loadAccountInfo(self, user, req):
- validuntil = None
- trafficleft = None
- leechtraffic = None
- premium = None
-
- if not self.HOSTER_URL: #@TODO: Remove in 0.4.10
- return {'validuntil' : validuntil,
- 'trafficleft' : trafficleft,
- 'leechtraffic': leechtraffic,
- 'premium' : premium}
-
- html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True)
-
- premium = True if re.search(self.PREMIUM_PATTERN, html) else False
-
- m = re.search(self.VALID_UNTIL_PATTERN, html)
- if m:
- expiredate = m.group(1).strip()
- self.logDebug("Expire date: " + expiredate)
-
- try:
- validuntil = mktime(strptime(expiredate, "%d %B %Y"))
-
- except Exception, e:
- self.logError(e)
-
- else:
- self.logDebug("Valid until: %s" % validuntil)
-
- if validuntil > mktime(gmtime()):
- premium = True
- trafficleft = -1
- else:
- premium = False
- validuntil = None #: registered account type (not premium)
- else:
- self.logDebug("VALID_UNTIL_PATTERN not found")
-
- m = re.search(self.TRAFFIC_LEFT_PATTERN, html)
- if m:
- try:
- traffic = m.groupdict()
- size = traffic['S']
-
- if "nlimited" in size:
- trafficleft = -1
- if validuntil is None:
- validuntil = -1
- else:
- if 'U' in traffic:
- unit = traffic['U']
- elif isinstance(self.TRAFFIC_LEFT_UNIT, basestring):
- unit = self.TRAFFIC_LEFT_UNIT
- else:
- unit = ""
-
- trafficleft = self.parseTraffic(size + unit)
-
- except Exception, e:
- self.logError(e)
- else:
- self.logDebug("TRAFFIC_LEFT_PATTERN not found")
-
- leech = [m.groupdict() for m in re.finditer(self.LEECH_TRAFFIC_PATTERN, html)]
- if leech:
- leechtraffic = 0
- try:
- for traffic in leech:
- size = traffic['S']
-
- if "nlimited" in size:
- leechtraffic = -1
- if validuntil is None:
- validuntil = -1
- break
- else:
- if 'U' in traffic:
- unit = traffic['U']
- elif isinstance(self.LEECH_TRAFFIC_UNIT, basestring):
- unit = self.LEECH_TRAFFIC_UNIT
- else:
- unit = ""
-
- leechtraffic += self.parseTraffic(size + unit)
-
- except Exception, e:
- self.logError(e)
- else:
- self.logDebug("LEECH_TRAFFIC_PATTERN not found")
-
- return {'validuntil' : validuntil,
- 'trafficleft' : trafficleft,
- 'leechtraffic': leechtraffic,
- 'premium' : premium}
-
-
- def login(self, user, data, req):
- if not self.HOSTER_URL: #@TODO: Remove in 0.4.10
- raise Exception(_("Missing HOSTER_DOMAIN"))
-
- if not self.LOGIN_URL:
- self.LOGIN_URL = urljoin(self.HOSTER_URL, "login.html")
- html = req.load(self.LOGIN_URL, decode=True)
-
- action, inputs = parseHtmlForm('name="FL"', html)
- if not inputs:
- inputs = {'op' : "login",
- 'redirect': self.HOSTER_URL}
-
- inputs.update({'login' : user,
- 'password': data['password']})
-
- if not action:
- action = self.HOSTER_URL
- html = req.load(action, post=inputs, decode=True)
-
- if re.search(self.LOGIN_FAIL_PATTERN, html):
- self.wrongPassword()
diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py
deleted file mode 100644
index 665e13b18..000000000
--- a/module/plugins/internal/XFSCrypter.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo
-
-
-class XFSCrypter(SimpleCrypter):
- __name__ = "XFSCrypter"
- __type__ = "crypter"
- __version__ = "0.06"
-
- __pattern__ = r'^unmatchable$'
-
- __description__ = """XFileSharing decrypter plugin"""
- __license__ = "GPLv3"
- __authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
-
-
- HOSTER_DOMAIN = None
-
- URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')]
-
- LINK_PATTERN = r'<(?:td|TD).*?>\s*<a href="(.+?)".*?>.+?(?:</a>)?\s*</(?:td|TD)>'
- NAME_PATTERN = r'<[tT]itle>.*?\: (?P<N>.+) folder</[tT]itle>'
-
- OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)'
- TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)'
-
-
- def prepare(self):
- if not self.HOSTER_DOMAIN:
- if self.account:
- account = self.account
- else:
- account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "")
- account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name)
-
- if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN:
- self.HOSTER_DOMAIN = account.HOSTER_DOMAIN
- else:
- self.fail(_("Missing HOSTER_DOMAIN"))
-
- if isinstance(self.COOKIES, list):
- self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english"))
-
- return super(XFSCrypter, self).prepare()
diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py
deleted file mode 100644
index ecdd2f125..000000000
--- a/module/plugins/internal/XFSHoster.py
+++ /dev/null
@@ -1,326 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-
-from random import random
-from time import sleep
-from urlparse import urljoin, urlparse
-
-from pyload.plugin.internal.captcha import ReCaptcha, SolveMedia
-from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, secondsToMidnight
-from pyload.utils import html_unescape
-
-
-class XFSHoster(SimpleHoster):
- __name__ = "XFSHoster"
- __type__ = "hoster"
- __version__ = "0.44"
-
- __pattern__ = r'^unmatchable$'
-
- __description__ = """XFileSharing hoster plugin"""
- __license__ = "GPLv3"
- __authors__ = [("zoidberg", "zoidberg@mujmail.cz"),
- ("stickell", "l.stickell@yahoo.it"),
- ("Walter Purcaro", "vuolter@gmail.com")]
-
-
- HOSTER_DOMAIN = None
-
- TEXT_ENCODING = False
- DIRECT_LINK = None
- MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy...
-
- NAME_PATTERN = r'(Filename[ ]*:[ ]*</b>(</td><td nowrap>)?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P<N>.+?)(\s*<|")'
- SIZE_PATTERN = r'(Size[ ]*:[ ]*</b>(</td><td>)?|File:.*>|</font>\s*\(|<[\w^_]+ class="size">)\s*(?P<S>[\d.,]+)\s*(?P<U>[\w^_]+)'
-
- OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)'
- TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)'
-
- WAIT_PATTERN = r'<span id="countdown_str">.*?>(\d+)</span>|id="countdown" value=".*?(\d+).*?"'
- PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only'
- ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter><b>|>Error</td>|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))'
-
- LINK_LEECH_PATTERN = r'<h2>Download Link</h2>\s*<textarea[^>]*>([^<]+)'
- LINK_PATTERN = None #: final download url pattern
-
- CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)'
- CAPTCHA_BLOCK_PATTERN = r'>Enter code.*?<div.*?>(.+?)</div>'
- RECAPTCHA_PATTERN = None
- SOLVEMEDIA_PATTERN = None
-
- FORM_PATTERN = None
- FORM_INPUTS_MAP = None #: dict passed as input_names to parseHtmlForm
-
-
- def setup(self):
- self.chunkLimit = -1 if self.premium else 1
- self.resumeDownload = self.multiDL = self.premium
-
-
- def prepare(self):
- """ Initialize important variables """
- if not self.HOSTER_DOMAIN:
- if self.account:
- account = self.account
- else:
- account = self.pyfile.m.core.accountManager.getAccountPlugin(self.__name__)
-
- if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN:
- self.HOSTER_DOMAIN = account.HOSTER_DOMAIN
- else:
- self.fail(_("Missing HOSTER_DOMAIN"))
-
- if isinstance(self.COOKIES, list):
- self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english"))
-
- if not self.LINK_PATTERN:
- pattern = r'(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]'
- self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.')
-
- self.captcha = None
- self.errmsg = None
-
- super(XFSHoster, self).prepare()
-
- if self.DIRECT_LINK is None:
- self.directDL = self.premium
-
-
- def handleFree(self, pyfile):
- for i in xrange(1, 6):
- self.logDebug("Getting download link: #%d" % i)
-
- self.checkErrors()
-
- m = re.search(self.LINK_PATTERN, self.html, re.S)
- if m:
- break
-
- data = self.getPostParameters()
-
- self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True, follow_location=False)
-
- m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I)
- if m and not "op=" in m.group(1):
- break
-
- m = re.search(self.LINK_PATTERN, self.html, re.S)
- if m:
- break
- else:
- self.logError(data['op'] if 'op' in data else _("UNKNOWN"))
- return ""
-
- self.link = m.group(1).strip() #@TODO: Remove .strip() in 0.4.10
-
-
- def handlePremium(self, pyfile):
- return self.handleFree(pyfile)
-
-
- def handleMulti(self, pyfile):
- if not self.account:
- self.fail(_("Only registered or premium users can use url leech feature"))
-
- #only tested with easybytez.com
- self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN)
-
- action, inputs = self.parseHtmlForm()
-
- upload_id = "%012d" % int(random() * 10 ** 12)
- action += upload_id + "&js_on=1&utype=prem&upload_type=url"
-
- inputs['tos'] = '1'
- inputs['url_mass'] = pyfile.url
- inputs['up1oad_type'] = 'url'
-
- self.logDebug(action, inputs)
-
- self.req.setOption("timeout", 600) #: wait for file to upload to easybytez.com
-
- self.html = self.load(action, post=inputs)
-
- self.checkErrors()
-
- action, inputs = self.parseHtmlForm('F1')
- if not inputs:
- if self.errmsg:
- self.retry(reason=self.errmsg)
- else:
- self.error(_("TEXTAREA F1 not found"))
-
- self.logDebug(inputs)
-
- stmsg = inputs['st']
-
- if stmsg == 'OK':
- self.html = self.load(action, post=inputs)
-
- elif 'Can not leech file' in stmsg:
- self.retry(20, 3 * 60, _("Can not leech file"))
-
- elif 'today' in stmsg:
- self.retry(wait_time=secondsToMidnight(gmt=2), reason=_("You've used all Leech traffic today"))
-
- else:
- self.fail(stmsg)
-
- #get easybytez.com link for uploaded file
- m = re.search(self.LINK_LEECH_PATTERN, self.html)
- if m is None:
- self.error(_("LINK_LEECH_PATTERN not found"))
-
- header = self.load(m.group(1), just_header=True, decode=True)
-
- if 'location' in header: #: Direct download link
- self.link = header['location']
-
-
- def checkErrors(self):
- m = re.search(self.ERROR_PATTERN, self.html)
- if m is None:
- self.errmsg = None
- else:
- self.errmsg = m.group(1).strip()
-
- self.logWarning(re.sub(r"<.*?>", " ", self.errmsg))
-
- if 'wait' in self.errmsg:
- wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in
- re.findall(r'(\d+)\s*(hr|hour|min|sec)', self.errmsg, re.I))
- self.wait(wait_time, True)
-
- elif 'country' in self.errmsg:
- self.fail(_("Downloads are disabled for your country"))
-
- elif 'captcha' in self.errmsg:
- self.invalidCaptcha()
-
- elif 'premium' in self.errmsg and 'require' in self.errmsg:
- self.fail(_("File can be downloaded by premium users only"))
-
- elif 'limit' in self.errmsg:
- if 'days' in self.errmsg:
- delay = secondsToMidnight(gmt=2)
- retries = 3
- else:
- delay = 1 * 60 * 60
- retries = 24
-
- self.wantReconnect = True
- self.retry(retries, delay, _("Download limit exceeded"))
-
- elif 'countdown' in self.errmsg or 'Expired' in self.errmsg:
- self.retry(reason=_("Link expired"))
-
- elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg:
- self.tempOffline()
-
- elif 'up to' in self.errmsg:
- self.fail(_("File too large for free download"))
-
- else:
- self.wantReconnect = True
- self.retry(wait_time=60, reason=self.errmsg)
-
- if self.errmsg:
- self.info['error'] = self.errmsg
- else:
- self.info.pop('error', None)
-
-
- def getPostParameters(self):
- if self.FORM_PATTERN or self.FORM_INPUTS_MAP:
- action, inputs = self.parseHtmlForm(self.FORM_PATTERN or "", self.FORM_INPUTS_MAP or {})
- else:
- action, inputs = self.parseHtmlForm(input_names={'op': re.compile(r'^download')})
-
- if not inputs:
- action, inputs = self.parseHtmlForm('F1')
- if not inputs:
- if self.errmsg:
- self.retry(reason=self.errmsg)
- else:
- self.error(_("TEXTAREA F1 not found"))
-
- self.logDebug(inputs)
-
- if 'op' in inputs:
- if "password" in inputs:
- password = self.getPassword()
- if password:
- inputs['password'] = password
- else:
- self.fail(_("Missing password"))
-
- if not self.premium:
- m = re.search(self.WAIT_PATTERN, self.html)
- if m:
- wait_time = int(m.group(1))
- self.setWait(wait_time, False)
-
- self.captcha = self.handleCaptcha(inputs)
-
- self.wait()
- else:
- inputs['referer'] = self.pyfile.url
-
- if self.premium:
- inputs['method_premium'] = "Premium Download"
- inputs.pop('method_free', None)
- else:
- inputs['method_free'] = "Free Download"
- inputs.pop('method_premium', None)
-
- return inputs
-
-
- def handleCaptcha(self, inputs):
- m = re.search(self.CAPTCHA_PATTERN, self.html)
- if m:
- captcha_url = m.group(1)
- inputs['code'] = self.decryptCaptcha(captcha_url)
- return 1
-
- m = re.search(self.CAPTCHA_BLOCK_PATTERN, self.html, re.S)
- if m:
- captcha_div = m.group(1)
- numerals = re.findall(r'<span.*?padding-left\s*:\s*(\d+).*?>(\d)</span>', html_unescape(captcha_div))
-
- self.logDebug(captcha_div)
-
- inputs['code'] = "".join(a[1] for a in sorted(numerals, key=lambda num: int(num[0])))
-
- self.logDebug("Captcha code: %s" % inputs['code'], numerals)
- return 2
-
- recaptcha = ReCaptcha(self)
- try:
- captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1)
-
- except Exception:
- captcha_key = recaptcha.detect_key()
-
- else:
- self.logDebug("ReCaptcha key: %s" % captcha_key)
-
- if captcha_key:
- inputs['recaptcha_response_field'], inputs['recaptcha_challenge_field'] = recaptcha.challenge(captcha_key)
- return 3
-
- solvemedia = SolveMedia(self)
- try:
- captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1)
-
- except Exception:
- captcha_key = solvemedia.detect_key()
-
- else:
- self.logDebug("SolveMedia key: %s" % captcha_key)
-
- if captcha_key:
- inputs['adcopy_response'], inputs['adcopy_challenge'] = solvemedia.challenge(captcha_key)
- return 4
-
- return 0
diff --git a/module/plugins/internal/__init__.py b/module/plugins/internal/__init__.py
deleted file mode 100644
index 40a96afc6..000000000
--- a/module/plugins/internal/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-# -*- coding: utf-8 -*-
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-24 01:52:18 +0100