summaryrefslogtreecommitdiffstats
path: root/module/plugins/internal
diff options
context:
space:
mode:
Diffstat (limited to 'module/plugins/internal')
-rw-r--r--module/plugins/internal/CaptchaService.py233
-rw-r--r--module/plugins/internal/DeadCrypter.py3
-rw-r--r--module/plugins/internal/DeadHoster.py3
-rw-r--r--module/plugins/internal/Extractor.py99
-rw-r--r--module/plugins/internal/MultiHook.py4
-rw-r--r--module/plugins/internal/MultiHoster.py24
-rw-r--r--module/plugins/internal/SevenZip.py163
-rw-r--r--module/plugins/internal/SimpleCrypter.py21
-rw-r--r--module/plugins/internal/SimpleDereferer.py15
-rw-r--r--module/plugins/internal/SimpleHoster.py139
-rw-r--r--module/plugins/internal/UnRar.py175
-rw-r--r--module/plugins/internal/UnZip.py64
-rw-r--r--module/plugins/internal/XFSAccount.py31
-rw-r--r--module/plugins/internal/XFSCrypter.py23
-rw-r--r--module/plugins/internal/XFSHoster.py43
15 files changed, 666 insertions, 374 deletions
diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py
index b429fd6e0..c6e83fe7d 100644
--- a/module/plugins/internal/CaptchaService.py
+++ b/module/plugins/internal/CaptchaService.py
@@ -1,24 +1,24 @@
# -*- coding: utf-8 -*-
import re
+import time
-from base64 import urlsafe_b64encode
-from random import random
+from base64 import b64encode
+from random import random, randint
+from urlparse import urljoin, urlparse
from module.common.json_layer import json_loads
class CaptchaService:
__name__ = "CaptchaService"
- __version__ = "0.16"
+ __version__ = "0.20"
__description__ = """Base captcha service plugin"""
__license__ = "GPLv3"
__authors__ = [("pyLoad Team", "admin@pyload.org")]
- KEY_PATTERN = None
-
key = None #: last key detected
@@ -27,25 +27,10 @@ class CaptchaService:
def detect_key(self, html=None):
- if not html:
- if hasattr(self.plugin, "html") and self.plugin.html:
- html = self.plugin.html
- else:
- errmsg = _("%s html not found") % self.__name__
- self.plugin.fail(errmsg) #@TODO: replace all plugin.fail(errmsg) with plugin.error(errmsg) in 0.4.10
- raise TypeError(errmsg)
-
- m = re.search(self.KEY_PATTERN, html)
- if m:
- self.key = m.group(1).strip()
- self.plugin.logDebug("%s key: %s" % (self.__name__, self.key))
- return self.key
- else:
- self.plugin.logDebug("%s key not found" % self.__name__)
- return None
+ raise NotImplementedError
- def challenge(self, key=None):
+ def challenge(self, key=None, html=None):
raise NotImplementedError
@@ -55,15 +40,17 @@ class CaptchaService:
class ReCaptcha(CaptchaService):
__name__ = "ReCaptcha"
- __version__ = "0.09"
+ __version__ = "0.11"
__description__ = """ReCaptcha captcha service plugin"""
__license__ = "GPLv3"
- __authors__ = [("pyLoad Team", "admin@pyload.org")]
+ __authors__ = [("pyLoad Team", "admin@pyload.org"),
+ ("Walter Purcaro", "vuolter@gmail.com"),
+ ("zapp-brannigan", "fuerst.reinje@web.de")]
- KEY_PATTERN = r'(?:class="g-recaptcha"\s+data-sitekey="|recaptcha(?:/api|\.net)/(?:challenge|noscript)\?k=)([\w-]+)'
- KEY_AJAX_PATTERN = r'Recaptcha\.create\s*\(\s*["\']([\w-]+)'
+ KEY_V2_PATTERN = r'(?:data-sitekey=["\']|["\']sitekey["\']:\s*["\'])([\w-]+)'
+ KEY_V1_PATTERN = r'(?:recaptcha(?:/api|\.net)/(?:challenge|noscript)\?k=|Recaptcha\.create\s*\(\s*["\'])([\w-]+)'
def detect_key(self, html=None):
@@ -75,7 +62,7 @@ class ReCaptcha(CaptchaService):
self.plugin.fail(errmsg)
raise TypeError(errmsg)
- m = re.search(self.KEY_PATTERN, html) or re.search(self.KEY_AJAX_PATTERN, html)
+ m = re.search(self.KEY_V2_PATTERN, html) or re.search(self.KEY_V1_PATTERN, html)
if m:
self.key = m.group(1).strip()
self.plugin.logDebug("ReCaptcha key: %s" % self.key)
@@ -85,9 +72,24 @@ class ReCaptcha(CaptchaService):
return None
- def challenge(self, key=None, userverify=False):
+ def challenge(self, key=None, html=None, version=None):
+ if not html:
+ if hasattr(self.plugin, "html") and self.plugin.html:
+ html = self.plugin.html
+ else:
+ errmsg = _("ReCaptcha html not found")
+ self.plugin.fail(errmsg)
+ raise TypeError(errmsg)
+
+ challenge = "challenge_v%s" % (version if version in (1, 2) else
+ 2 if re.search(self.KEY_V2_PATTERN, html) else 1)
+
+ return getattr(self, challenge)(key, html)
+
+
+ def challenge_v1(self, key=None, html=None):
if not key:
- if self.detect_key():
+ if self.detect_key(html):
key = self.key
else:
errmsg = _("ReCaptcha key not found")
@@ -106,22 +108,7 @@ class ReCaptcha(CaptchaService):
self.plugin.logDebug("ReCaptcha challenge: %s" % challenge)
- response = challenge, self.result(server, challenge)
-
- return self.userverify(*response) if userverify else response
-
-
- def userverify(self, challenge, result):
- response = self.plugin.req.load("https://www.google.com/recaptcha/api2/userverify",
- post={'c' : challenge,
- 'response': urlsafe_b64encode('{"response":"%s"}' % result)})
- try:
- return re.search(r'"uvresp","(.+?)"', response).group(1)
-
- except AttributeError:
- errmsg = _("ReCaptcha userverify response not found")
- self.plugin.fail(errmsg)
- raise AttributeError(errmsg)
+ return self.result(server, challenge), challenge
def result(self, server, challenge):
@@ -136,9 +123,122 @@ class ReCaptcha(CaptchaService):
return result
+ def _collectApiInfo(self):
+ html = self.plugin.req.load("http://www.google.com/recaptcha/api.js")
+ a = re.search(r'po.src = \'(.*?)\';', html).group(1)
+ vers = a.split("/")[5]
+
+ self.plugin.logDebug("ReCaptcha API version: %s" %vers)
+
+ language = a.split("__")[1].split(".")[0]
+
+ self.plugin.logDebug("ReCaptcha API language: %s" % language)
+
+ html = self.plugin.req.load("https://apis.google.com/js/api.js")
+ b = re.search(r'"h":"(.*?)","', html).group(1)
+ jsh = b.decode('unicode-escape')
+
+ self.plugin.logDebug("ReCaptcha API jsh-string: %s" % jsh)
+
+ return vers, language, jsh
+
+
+ def _prepareTimeAndRpc(self):
+ self.plugin.req.load("http://www.google.com/recaptcha/api2/demo")
+
+ millis = int(round(time.time() * 1000))
+
+ self.plugin.logDebug("ReCaptcha time: %s" % millis)
+
+ rand = randint(1, 99999999)
+ a = "0.%s" % str(rand * 2147483647)
+ rpc = int(100000000 * float(a))
+
+ self.plugin.logDebug("ReCaptcha rpc-token: %s" % rpc)
+
+ return millis, rpc
+
+
+ def challenge_v2(self, key=None, html=None):
+ if not key:
+ if self.detect_key(html):
+ key = self.key
+ else:
+ errmsg = _("ReCaptcha key not found")
+ self.plugin.fail(errmsg)
+ raise TypeError(errmsg)
+
+ try:
+ parent = urljoin("http://", urlparse(self.plugin.pyfile.url).netloc)
+
+ except Exception:
+ parent = ""
+
+ botguardstring = "!A"
+ vers, language, jsh = self._collectApiInfo()
+ millis, rpc = self._prepareTimeAndRpc()
+
+ html = self.plugin.req.load("https://www.google.com/recaptcha/api2/anchor",
+ get={'k' : key,
+ 'hl' : language,
+ 'v' : vers,
+ 'usegapi' : "1",
+ 'jsh' : "%s#id=IO_%s" % (jsh, millis),
+ 'parent' : parent,
+ 'pfname' : "",
+ 'rpctoken': rpc})
+
+ token1 = re.search(r'id="recaptcha-token" value="(.*?)">', html)
+ self.plugin.logDebug("ReCaptcha token #1: %s" % token1.group(1))
+
+ html = self.plugin.req.load("https://www.google.com/recaptcha/api2/frame",
+ get={'c' : token1.group(1),
+ 'hl' : language,
+ 'v' : vers,
+ 'bg' : botguardstring,
+ 'usegapi': "1",
+ 'jsh' : jsh}).decode('unicode-escape')
+
+ token2 = re.search(r'"finput","(.*?)",', html)
+ self.plugin.logDebug("ReCaptcha token #2: %s" % token2.group(1))
+
+ token3 = re.search(r'."asconf".\s.\s,"(.*?)".', html)
+ self.plugin.logDebug("ReCaptcha token #3: %s" % token3.group(1))
+
+ html = self.plugin.req.load("https://www.google.com/recaptcha/api2/reload",
+ post={'c' : token2.group(1),
+ 'reason': "fi",
+ 'fbg' : token3.group(1)})
+
+ token4 = re.search(r'"rresp","(.*?)",', html)
+ self.plugin.logDebug("ReCaptcha token #4: %s" % token4.group(1))
+
+ millis_captcha_loading = int(round(time.time() * 1000))
+ captcha_response = self.plugin.decryptCaptcha("https://www.google.com/recaptcha/api2/payload",
+ get={'c':token4.group(1)}, forceUser=True)
+ response = b64encode('{"response":"%s"}' % captcha_response)
+
+ self.plugin.logDebug("ReCaptcha result: %s" % response)
+
+ timeToSolve = int(round(time.time() * 1000)) - millis_captcha_loading
+ timeToSolveMore = timeToSolve + int(float("0." + str(randint(1, 99999999))) * 500)
+
+ html = self.plugin.req.load("https://www.google.com/recaptcha/api2/userverify",
+ post={'c' : token4.group(1),
+ 'response': response,
+ 't' : timeToSolve,
+ 'ct' : timeToSolveMore,
+ 'bg' : botguardstring})
+
+ token5 = re.search(r'"uvresp","(.*?)",', html)
+ self.plugin.logDebug("ReCaptcha token #5: %s" % token5.group(1))
+
+ return token5.group(1), None
+
+
class AdsCaptcha(CaptchaService):
__name__ = "AdsCaptcha"
- __version__ = "0.06"
+ __version__ = "0.07"
__description__ = """AdsCaptcha captcha service plugin"""
__license__ = "GPLv3"
@@ -169,9 +269,9 @@ class AdsCaptcha(CaptchaService):
return None
- def challenge(self, key=None):
+ def challenge(self, key=None, html=None):
if not key:
- if self.detect_key():
+ if self.detect_key(html):
key = self.key
else:
errmsg = _("AdsCaptcha key not found")
@@ -192,7 +292,7 @@ class AdsCaptcha(CaptchaService):
self.plugin.logDebug("AdsCaptcha challenge: %s" % challenge)
- return challenge, self.result(server, challenge)
+ return self.result(server, challenge), challenge
def result(self, server, challenge):
@@ -208,7 +308,7 @@ class AdsCaptcha(CaptchaService):
class SolveMedia(CaptchaService):
__name__ = "SolveMedia"
- __version__ = "0.06"
+ __version__ = "0.08"
__description__ = """SolveMedia captcha service plugin"""
__license__ = "GPLv3"
@@ -218,9 +318,28 @@ class SolveMedia(CaptchaService):
KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']'
- def challenge(self, key=None):
+ def detect_key(self, html=None):
+ if not html:
+ if hasattr(self.plugin, "html") and self.plugin.html:
+ html = self.plugin.html
+ else:
+ errmsg = _("SolveMedia html not found")
+ self.plugin.fail(errmsg)
+ raise TypeError(errmsg)
+
+ m = re.search(self.KEY_PATTERN, html)
+ if m:
+ self.key = m.group(1).strip()
+ self.plugin.logDebug("SolveMedia key: %s" % self.key)
+ return self.key
+ else:
+ self.plugin.logDebug("SolveMedia key not found")
+ return None
+
+
+ def challenge(self, key=None, html=None):
if not key:
- if self.detect_key():
+ if self.detect_key(html):
key = self.key
else:
errmsg = _("SolveMedia key not found")
@@ -240,7 +359,7 @@ class SolveMedia(CaptchaService):
self.plugin.logDebug("SolveMedia challenge: %s" % challenge)
- return challenge, self.result(server, challenge)
+ return self.result(server, challenge), challenge
def result(self, server, challenge):
@@ -253,7 +372,7 @@ class SolveMedia(CaptchaService):
class AdYouLike(CaptchaService):
__name__ = "AdYouLike"
- __version__ = "0.02"
+ __version__ = "0.04"
__description__ = """AdYouLike captcha service plugin"""
__license__ = "GPLv3"
@@ -284,9 +403,9 @@ class AdYouLike(CaptchaService):
return None
- def challenge(self, key=None):
+ def challenge(self, key=None, html=None):
if not key:
- if self.detect_key():
+ if self.detect_key(html):
key = self.key
else:
errmsg = _("AdYouLike key not found")
@@ -313,7 +432,7 @@ class AdYouLike(CaptchaService):
self.plugin.logDebug("AdYouLike challenge: %s" % challenge)
- return self.result(ayl, challenge)
+ return self.result(ayl, challenge), challenge
def result(self, server, challenge):
diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py
index 0fa23eef3..866d177cf 100644
--- a/module/plugins/internal/DeadCrypter.py
+++ b/module/plugins/internal/DeadCrypter.py
@@ -1,8 +1,5 @@
# -*- coding: utf-8 -*-
-from urllib import unquote
-from urlparse import urlparse
-
from module.plugins.internal.SimpleCrypter import create_getInfo
from module.plugins.Crypter import Crypter as _Crypter
diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py
index cc7adf4df..a6ad92607 100644
--- a/module/plugins/internal/DeadHoster.py
+++ b/module/plugins/internal/DeadHoster.py
@@ -1,8 +1,5 @@
# -*- coding: utf-8 -*-
-from urllib import unquote
-from urlparse import urlparse
-
from module.plugins.internal.SimpleHoster import create_getInfo
from module.plugins.Hoster import Hoster as _Hoster
diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py
index 0b2462dac..ddf0f8a85 100644
--- a/module/plugins/internal/Extractor.py
+++ b/module/plugins/internal/Extractor.py
@@ -1,5 +1,11 @@
# -*- coding: utf-8 -*-
+import os
+
+from module.PyFile import PyFile
+from module.utils import fs_encode
+
+
class ArchiveError(Exception):
pass
@@ -14,7 +20,7 @@ class PasswordError(Exception):
class Extractor:
__name__ = "Extractor"
- __version__ = "0.13"
+ __version__ = "0.15"
__description__ = """Base extractor plugin"""
__license__ = "GPLv3"
@@ -26,6 +32,12 @@ class Extractor:
@classmethod
+ def isArchive(cls, filename):
+ name = os.path.basename(filename).lower()
+ return any(name.endswith(ext) for ext in cls.EXTENSIONS)
+
+
+ @classmethod
def checkDeps(cls):
""" Check if system statisfy dependencies
:return: boolean
@@ -34,46 +46,36 @@ class Extractor:
@classmethod
- def isArchive(cls, file):
- raise NotImplementedError
-
-
- @classmethod
def getTargets(cls, files_ids):
""" Filter suited targets from list of filename id tuple list
:param files_ids: List of filepathes
:return: List of targets, id tuple list
"""
- targets = []
-
- for file, id in files_ids:
- if cls.isArchive(file):
- targets.append((file, id))
-
- return targets
-
+ raise NotImplementedError
- def __init__(self, m, file, out, password, fullpath, overwrite, excludefiles, renice, delete, keepbroken):
- """Initialize extractor for specific file
- :param m: ExtractArchive Hook plugin
- :param file: Absolute filepath
- :param out: Absolute path to destination directory
- :param fullpath: extract to fullpath
- :param overwrite: Overwrite existing archives
- :param renice: Renice value
- """
- self.m = m
- self.file = file
- self.out = out
- self.password = password
- self.fullpath = fullpath
- self.overwrite = overwrite
- self.excludefiles = excludefiles
- self.renice = renice
- self.delete = delete
- self.keepbroken = keepbroken
- self.files = [] #: Store extracted files here
+ def __init__(self, manager, filename, out,
+ fullpath=True,
+ overwrite=False,
+ excludefiles=[],
+ renice=0,
+ delete=False,
+ keepbroken=False,
+ fid=None):
+ """ Initialize extractor for specific file """
+ self.manager = manager
+ self.target = fs_encode(filename)
+ self.out = out
+ self.fullpath = fullpath
+ self.overwrite = overwrite
+ self.excludefiles = excludefiles
+ self.renice = renice
+ self.delete = delete
+ self.keepbroken = keepbroken
+ self.files = [] #: Store extracted files here
+
+ pyfile = self.manager.core.files.getFile(fid) if fid else None
+ self.notifyProgress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None
def init(self):
@@ -81,44 +83,31 @@ class Extractor:
pass
- def verify(self):
+ def checkArchive(self):
"""Check if password if needed. Raise ArchiveError if integrity is
questionable.
+ :return: boolean
:raises ArchiveError
"""
- pass
+ return False
- def isPassword(self, password):
+ def checkPassword(self, password):
""" Check if the given password is/might be correct.
If it can not be decided at this point return true.
:param password:
:return: boolean
"""
- if isinstance(password, basestring):
- return True
- else:
- return False
-
-
- def setPassword(self, password):
- if self.isPassword(password):
- self.password = password
- return True
- else:
- return False
-
-
- def repair(self):
- return False
+ return True
- def extract(self, progress=lambda x: None):
+ def extract(self, password=None):
"""Extract the archive. Raise specific errors in case of failure.
:param progress: Progress function, call this to update status
+ :param password password to use
:raises PasswordError
:raises CRCError
:raises ArchiveError
@@ -132,7 +121,7 @@ class Extractor:
:return: List with paths of files to delete
"""
- raise NotImplementedError
+ return [self.target]
def getExtractedFiles(self):
diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py
index 795565d73..652443098 100644
--- a/module/plugins/internal/MultiHook.py
+++ b/module/plugins/internal/MultiHook.py
@@ -11,7 +11,7 @@ from module.utils import decode, remove_chars
class MultiHook(Hook):
__name__ = "MultiHook"
__type__ = "hook"
- __version__ = "0.36"
+ __version__ = "0.37"
__config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"),
("pluginlist" , "str" , "Plugin list (comma separated)" , "" ),
@@ -201,7 +201,7 @@ class MultiHook(Hook):
self.unloadPlugin(plugin)
if self.getConfig("reload", True):
- self.interval = max(self.getConfig("reloadinterval", 12), self.MIN_INTERVAL)
+ self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_INTERVAL)
else:
self.core.scheduler.removeJob(self.cb)
self.cb = None
diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py
index ba613c997..ad5b6283e 100644
--- a/module/plugins/internal/MultiHoster.py
+++ b/module/plugins/internal/MultiHoster.py
@@ -8,7 +8,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r
class MultiHoster(SimpleHoster):
__name__ = "MultiHoster"
__type__ = "hoster"
- __version__ = "0.33"
+ __version__ = "0.36"
__pattern__ = r'^unmatchable$'
@@ -27,9 +27,10 @@ class MultiHoster(SimpleHoster):
def prepare(self):
- self.info = {}
- self.link = "" #@TODO: Move to hoster class in 0.4.10
- self.directDL = False #@TODO: Move to hoster class in 0.4.10
+ self.info = {}
+ self.html = ""
+ self.link = "" #@TODO: Move to hoster class in 0.4.10
+ self.directDL = False #@TODO: Move to hoster class in 0.4.10
if self.LOGIN_ACCOUNT and not self.account:
self.fail(_("Required account not found"))
@@ -40,7 +41,7 @@ class MultiHoster(SimpleHoster):
set_cookies(self.req.cj, self.COOKIES)
if self.DIRECT_LINK is None:
- self.directDL = self.__pattern__ != r'^unmatchable$'
+ self.directDL = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url)
else:
self.directDL = self.DIRECT_LINK
@@ -51,10 +52,8 @@ class MultiHoster(SimpleHoster):
def process(self, pyfile):
self.prepare()
- if self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, pyfile.url):
- self.checkInfo()
-
if self.directDL:
+ self.checkInfo()
self.logDebug("Looking for direct download link...")
self.handleDirect(pyfile)
@@ -72,17 +71,10 @@ class MultiHoster(SimpleHoster):
self.logDebug("Handled as free download")
self.handleFree(pyfile)
- self.downloadLink(self.link)
+ self.downloadLink(self.link, True)
self.checkFile()
- #@TODO: Remove in 0.4.10
- def downloadLink(self, link):
- if link and isinstance(link, basestring):
- self.correctCaptcha()
- self.download(link, disposition=True)
-
-
def handlePremium(self, pyfile):
return self.handleFree(pyfile)
diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py
new file mode 100644
index 000000000..508cf9c8d
--- /dev/null
+++ b/module/plugins/internal/SevenZip.py
@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+
+import os
+import re
+
+from subprocess import Popen, PIPE
+
+from module.plugins.internal.UnRar import UnRar, renice
+from module.utils import save_join
+
+
+class SevenZip(UnRar):
+ __name__ = "SevenZip"
+ __version__ = "0.02"
+
+ __description__ = """7-Zip extractor plugin"""
+ __license__ = "GPLv3"
+ __authors__ = [("Michael Nowak", ""),
+ ("Walter Purcaro", "vuolter@gmail.com")]
+
+
+ CMD = "7z"
+
+ EXTENSIONS = [".7z", ".xz", ".zip", ".gz", ".gzip", ".tgz", ".bz2", ".bzip2",
+ ".tbz2", ".tbz", ".tar", ".wim", ".swm", ".lzma", ".rar", ".cab",
+ ".arj", ".z", ".taz", ".cpio", ".rpm", ".deb", ".lzh", ".lha",
+ ".chm", ".chw", ".hxs", ".iso", ".msi", ".doc", ".xls", ".ppt",
+ ".dmg", ".xar", ".hfs", ".exe", ".ntfs", ".fat", ".vhd", ".mbr",
+ ".squashfs", ".cramfs", ".scap"]
+
+
+ #@NOTE: there are some more uncovered 7z formats
+ re_filelist = re.compile(r'([\d\:]+)\s+([\d\:]+)\s+([\w\.]+)\s+(\d+)\s+(\d+)\s+(.+)')
+ re_wrongpwd = re.compile(r'(Can not open encrypted archive|Wrong password)', re.I)
+ re_wrongcrc = re.compile(r'Encrypted\s+\=\s+\+', re.I)
+
+
+ @classmethod
+ def checkDeps(cls):
+ if os.name == "nt":
+ cls.CMD = os.path.join(pypath, "7z.exe")
+ p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
+ p.communicate()
+ else:
+ p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
+ p.communicate()
+
+ return True
+
+
+ def checkArchive(self):
+ p = self.call_cmd("l", "-slt", self.target)
+ out, err = p.communicate()
+
+ if p.returncode > 1:
+ raise ArchiveError("Process terminated")
+
+ # check if output or error macthes the 'wrong password'-Regexp
+ if self.re_wrongpwd.search(out):
+ return True
+
+ # check if output matches 'Encrypted = +'
+ if self.re_wrongcrc.search(out):
+ return True
+
+ # check if archive is empty
+ self.files = self.list()
+ if not self.files:
+ raise ArchiveError("Empty Archive")
+
+ return False
+
+
+ def checkPassword(self, password):
+ p = self.call_cmd("l", self.target, password=password)
+ p.communicate()
+ return p.returncode == 0
+
+
+ def extract(self, password=None):
+ command = "x" if self.fullpath else "e"
+
+ p = self.call_cmd(command, '-o' + self.out, self.target, password=password)
+
+ renice(p.pid, self.renice)
+
+ progressstring = ""
+ while True:
+ c = p.stdout.read(1)
+ # quit loop on eof
+ if not c:
+ break
+ # reading a percentage sign -> set progress and restart
+ if c == '%':
+ self.notifyProgress(int(progressstring))
+ progressstring = ""
+ # not reading a digit -> therefore restart
+ elif c not in digits:
+ progressstring = ""
+ # add digit to progressstring
+ else:
+ progressstring += c
+
+ # retrieve stderr
+ err = p.stderr.read()
+
+ if self.re_wrongpwd.search(err):
+ raise PasswordError
+
+ elif self.re_wrongcrc.search(err):
+ raise CRCError
+
+ elif err.strip(): #: raise error if anything is on stderr
+ raise ArchiveError(err.strip())
+
+ if p.returncode > 1:
+ raise ArchiveError("Process terminated")
+
+ if not self.files:
+ self.files = self.list(password)
+
+
+ def list(self, password=None):
+ command = "l" if self.fullpath else "l"
+
+ p = self.call_cmd(command, self.target, password=password)
+ out, err = p.communicate()
+ code = p.returncode
+
+ if "Can not open" in err:
+ raise ArchiveError("Cannot open file")
+
+ if code != 0:
+ raise ArchiveError("Process terminated unsuccessful")
+
+ result = set()
+ for groups in self.re_filelist.findall(out):
+ f = groups[-1].strip()
+ result.add(save_join(self.out, f))
+
+ return list(result)
+
+
+ def call_cmd(self, command, *xargs, **kwargs):
+ args = []
+
+ #overwrite flag
+ if self.overwrite:
+ args.append("-y")
+
+ #set a password
+ if "password" in kwargs and kwargs["password"]:
+ args.append("-p%s" % kwargs["password"])
+ else:
+ args.append("-p-")
+
+ #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue
+ call = [self.cmd, command] + args + list(xargs)
+
+ self.manager.logDebug(" ".join([decode(arg) for arg in call]))
+
+ p = Popen(call, stdout=PIPE, stderr=PIPE)
+ return p
diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py
index d33ad77e9..b2622b5e0 100644
--- a/module/plugins/internal/SimpleCrypter.py
+++ b/module/plugins/internal/SimpleCrypter.py
@@ -12,7 +12,7 @@ from module.utils import fixup
class SimpleCrypter(Crypter, SimpleHoster):
__name__ = "SimpleCrypter"
__type__ = "crypter"
- __version__ = "0.39"
+ __version__ = "0.42"
__pattern__ = r'^unmatchable$'
__config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package']
@@ -71,12 +71,12 @@ class SimpleCrypter(Crypter, SimpleHoster):
#@TODO: Remove in 0.4.10
def init(self):
account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "")
- account = self.core.accountManager.getAccountPlugin(account_name)
+ account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name)
if account and account.canUse():
self.user, data = account.selectAccount()
- self.req = account.getAccountRequest(self.user)
- self.premium = account.isPremium(self.user)
+ self.req = account.getAccountRequest(self.user)
+ self.premium = account.isPremium(self.user)
self.account = account
@@ -85,6 +85,7 @@ class SimpleCrypter(Crypter, SimpleHoster):
self.pyfile.error = "" #@TODO: Remove in 0.4.10
self.info = {}
+ self.html = ""
self.links = [] #@TODO: Move to hoster class in 0.4.10
if self.LOGIN_PREMIUM and not self.premium:
@@ -118,7 +119,7 @@ class SimpleCrypter(Crypter, SimpleHoster):
self.packages = [(self.info['name'], self.links, self.info['folder'])]
elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10
- self.fail("No link grabbed")
+ self.fail(_("No link grabbed"))
def checkNameSize(self, getinfo=True):
@@ -151,17 +152,17 @@ class SimpleCrypter(Crypter, SimpleHoster):
Returns the links extracted from self.html
You should override this only if it's impossible to extract links using only the LINK_PATTERN.
"""
- parsed = urlparse(self.pyfile.url)
- base = "%s://%s" % (parsed.scheme, parsed.netloc)
+ url_p = urlparse(self.pyfile.url)
+ baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
- return [urljoin(base, link) if not urlparse(link).scheme else link \
- for link in re.finditer(self.LINK_PATTERN, self.html)]
+ return [urljoin(baseurl, link) if not urlparse(link).scheme else link \
+ for link in re.findall(self.LINK_PATTERN, self.html)]
def handlePages(self, pyfile):
try:
pages = int(re.search(self.PAGES_PATTERN, self.html).group(1))
- except:
+ except Exception:
pages = 1
for p in xrange(2, pages + 1):
diff --git a/module/plugins/internal/SimpleDereferer.py b/module/plugins/internal/SimpleDereferer.py
index 53b80f827..bd00f5d25 100644
--- a/module/plugins/internal/SimpleDereferer.py
+++ b/module/plugins/internal/SimpleDereferer.py
@@ -5,13 +5,13 @@ import re
from urllib import unquote
from module.plugins.Crypter import Crypter
-from module.plugins.internal.SimpleHoster import directLink, set_cookies
+from module.plugins.internal.SimpleHoster import fileUrl, set_cookies
class SimpleDereferer(Crypter):
__name__ = "SimpleDereferer"
__type__ = "crypter"
- __version__ = "0.03"
+ __version__ = "0.07"
__pattern__ = r'^unmatchable$'
__config__ = [("use_subfolder", "bool", "Save package to subfolder", True),
@@ -45,7 +45,7 @@ class SimpleDereferer(Crypter):
def decrypt(self, pyfile):
- link = directLink(self, pyfile.url)
+ link = fileUrl(self, pyfile.url)
if not link:
try:
@@ -54,10 +54,6 @@ class SimpleDereferer(Crypter):
except Exception:
self.prepare()
self.preload()
-
- if self.html is None:
- self.fail(_("No html retrieved"))
-
self.checkStatus()
link = self.getLink()
@@ -66,10 +62,13 @@ class SimpleDereferer(Crypter):
self.urls = [link.strip()] #@TODO: Remove `.strip()` in 0.4.10
elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10
- self.fail("No link grabbed")
+ self.fail(_("No link grabbed"))
def prepare(self):
+ self.info = {}
+ self.html = ""
+
self.req.setOption("timeout", 120)
if isinstance(self.COOKIES, list):
diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py
index 3f14ca711..c74e33d59 100644
--- a/module/plugins/internal/SimpleHoster.py
+++ b/module/plugins/internal/SimpleHoster.py
@@ -1,10 +1,11 @@
# -*- coding: utf-8 -*-
+import mimetypes
+import os
import re
from datetime import datetime, timedelta
from inspect import isclass
-from os.path import exists
from time import time
from urllib import unquote
from urlparse import urljoin, urlparse
@@ -107,7 +108,13 @@ def parseFileInfo(plugin, url="", html=""):
info = plugin.getInfo(url, html)
res = info['name'], info['size'], info['status'], info['url']
else:
- res = urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 0, 3, url
+ url = unquote(url)
+ res = ((urlparse(url).path.split('/')[-1]
+ or urlparse(url).query.split('=', 1)[::-1][0].split('&', 1)[0]
+ or _("Unknown")),
+ 0,
+ 3 if url else 8,
+ url)
return res
@@ -133,37 +140,86 @@ def timestamp():
#@TODO: Move to hoster class in 0.4.10
-def directLink(self, url, resumable=False):
- link = ""
+def fileUrl(self, url, follow_location=None):
+ link = ""
+ redirect = 1
- for i in xrange(5 if resumable else 1):
- header = self.load(url, ref=True, cookies=True, just_header=True, decode=True)
+ if type(follow_location) is int:
+ redirect = max(follow_location, 1)
+ else:
+ redirect = 5
+
+ for i in xrange(redirect):
+ try:
+ self.logDebug("Redirect #%d to: %s" % (i, url))
+ header = self.load(url, ref=True, cookies=True, just_header=True, decode=True)
+
+ except Exception: #: Bad bad bad...
+ req = pyreq.getHTTPRequest()
+ res = req.load(url, cookies=True, just_header=True, decode=True)
+
+ req.close()
+
+ header = {"code": req.code}
+ for line in res.splitlines():
+ line = line.strip()
+ if not line or ":" not in line:
+ continue
+
+ key, none, value = line.partition(":")
+ key = key.lower().strip()
+ value = value.strip()
+
+ if key in header:
+ if type(header[key]) == list:
+ header[key].append(value)
+ else:
+ header[key] = [header[key], value]
+ else:
+ header[key] = value
if 'content-disposition' in header:
link = url
- elif 'location' in header and header['location']:
+ elif 'location' in header and header['location'].strip():
location = header['location']
if not urlparse(location).scheme:
- parsed = urlparse(url)
- base = "%s://%s" % (parsed.scheme, parsed.netloc)
- location = urljoin(base, location)
+ url_p = urlparse(url)
+ baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
+ location = urljoin(baseurl, location)
+
+ if 'code' in header and header['code'] == 302:
+ link = location
- if resumable:
+ if follow_location:
url = location
- self.logDebug("Redirect #%d to: %s" % (++i, location))
continue
- elif 'code' in header and header['code'] == 302:
- link = location
+ else:
+ extension = os.path.splitext(urlparse(url).path.split('/')[-1])[-1]
- elif 'content-type' in header and header['content-type'] and "html" not in header['content-type']:
- link = url
+ if 'content-type' in header and header['content-type'].strip():
+ mimetype = header['content-type'].split(';')[0].strip()
+
+ elif extension:
+ mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream"
+
+ else:
+ mimetype = ""
+
+ if mimetype and (link or 'html' not in mimetype):
+ link = url
+ else:
+ link = ""
break
+
else:
- self.logError(_("Too many redirects"))
+ try:
+ self.logError(_("Too many redirects"))
+ except Exception:
+ pass
return link
@@ -189,7 +245,7 @@ def secondsToMidnight(gmt=0):
class SimpleHoster(Hoster):
__name__ = "SimpleHoster"
__type__ = "hoster"
- __version__ = "0.98"
+ __version__ = "1.12"
__pattern__ = r'^unmatchable$'
@@ -253,7 +309,7 @@ class SimpleHoster(Hoster):
MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method)
LOGIN_ACCOUNT = False #: Set to True to require account login
- directLink = directLink #@TODO: Remove in 0.4.10
+ directLink = fileUrl #@TODO: Remove in 0.4.10
@classmethod
@@ -277,7 +333,7 @@ class SimpleHoster(Hoster):
@classmethod
def getInfo(cls, url="", html=""):
info = cls.apiInfo(url)
- online = False
+ online = False if info['status'] != 2 else True
try:
info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here
@@ -285,12 +341,12 @@ class SimpleHoster(Hoster):
except Exception:
info['pattern'] = {}
- if not html:
+ if not html and not online:
if not url:
info['error'] = "missing url"
info['status'] = 1
- elif info['status'] is 3:
+ elif info['status'] is 3 and not fileUrl(None, url):
try:
html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING)
@@ -368,6 +424,7 @@ class SimpleHoster(Hoster):
self.pyfile.error = "" #@TODO: Remove in 0.4.10
self.info = {}
+ self.html = ""
self.link = "" #@TODO: Move to hoster class in 0.4.10
self.directDL = False #@TODO: Move to hoster class in 0.4.10
self.multihost = False #@TODO: Move to hoster class in 0.4.10
@@ -434,10 +491,16 @@ class SimpleHoster(Hoster):
self.checkFile()
- def downloadLink(self, link):
+ def downloadLink(self, link, disposition=False): #@TODO: Set `disposition=True` in 0.4.10
if link and isinstance(link, basestring):
self.correctCaptcha()
- self.download(link, disposition=False) #@TODO: Set `disposition=True` in 0.4.10
+
+ if not urlparse(link).scheme:
+ url_p = urlparse(self.pyfile.url)
+ baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
+ link = urljoin(baseurl, link)
+
+ self.download(link, ref=False, disposition=disposition)
def checkFile(self):
@@ -445,7 +508,7 @@ class SimpleHoster(Hoster):
self.invalidCaptcha()
self.retry(10, reason=_("Wrong captcha"))
- elif not self.lastDownload or not exists(fs_encode(self.lastDownload)):
+ elif not self.lastDownload or not os.path.exists(fs_encode(self.lastDownload)):
self.lastDownload = ""
self.error(self.pyfile.error or _("No file downloaded"))
@@ -472,7 +535,7 @@ class SimpleHoster(Hoster):
self.logWarning(_("No html code to check"))
return
- if hasattr(self, 'PREMIUM_ONLY_PATTERN') and self.premium and re.search(self.PREMIUM_ONLY_PATTERN, self.html):
+ if hasattr(self, 'PREMIUM_ONLY_PATTERN') and not self.premium and re.search(self.PREMIUM_ONLY_PATTERN, self.html):
self.fail(_("Link require a premium account to be handled"))
elif hasattr(self, 'ERROR_PATTERN'):
@@ -577,16 +640,12 @@ class SimpleHoster(Hoster):
if not hasattr(self, 'LINK_FREE_PATTERN'):
self.logError(_("Free download not implemented"))
- try:
- m = re.search(self.LINK_FREE_PATTERN, self.html)
- if m is None:
- self.error(_("Free download link not found"))
-
+ m = re.search(self.LINK_FREE_PATTERN, self.html)
+ if m is None:
+ self.error(_("Free download link not found"))
+ else:
self.link = m.group(1)
- except Exception, e:
- self.fail(e)
-
def handlePremium(self, pyfile):
if not hasattr(self, 'LINK_PREMIUM_PATTERN'):
@@ -594,16 +653,12 @@ class SimpleHoster(Hoster):
self.logDebug("Handled as free download")
self.handleFree(pyfile)
- try:
- m = re.search(self.LINK_PREMIUM_PATTERN, self.html)
- if m is None:
- self.error(_("Premium download link not found"))
-
+ m = re.search(self.LINK_PREMIUM_PATTERN, self.html)
+ if m is None:
+ self.error(_("Premium download link not found"))
+ else:
self.link = m.group(1)
- except Exception, e:
- self.fail(e)
-
def longWait(self, wait_time=None, max_tries=3):
if wait_time and isinstance(wait_time, (int, long, float)):
diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py
index 572fe95b9..7f1b08caf 100644
--- a/module/plugins/internal/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -4,7 +4,6 @@ import os
import re
from glob import glob
-from os.path import basename, dirname, join
from string import digits
from subprocess import Popen, PIPE
@@ -13,30 +12,31 @@ from module.utils import save_join, decode
def renice(pid, value):
- if os.name != "nt" and value:
+ if value and os.name != "nt":
try:
Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1)
- except:
- print "Renice failed"
+ except Exception:
+ pass
class UnRar(Extractor):
__name__ = "UnRar"
- __version__ = "1.01"
+ __version__ = "1.04"
__description__ = """Rar extractor plugin"""
__license__ = "GPLv3"
- __authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
+ __authors__ = [("RaNaN", "RaNaN@pyload.org"),
+ ("Walter Purcaro", "vuolter@gmail.com")]
CMD = "unrar"
- EXTENSIONS = ["rar", "zip", "cab", "arj", "lzh", "tar", "gz", "bz2", "ace", "uue", "jar", "iso", "7z", "xz", "z"]
-
+ EXTENSIONS = [".rar", ".zip", ".cab", ".arj", ".lzh", ".tar", ".gz", ".bz2",
+ ".ace", ".uue", ".jar", ".iso", ".7z", ".xz", ".z"]
#@NOTE: there are some more uncovered rar formats
- re_rarpart = re.compile(r'(.*)\.part(\d+)\.rar$', re.I)
- re_rarfile = re.compile(r'.*\.(rar|r\d+)$', re.I)
+ re_rarpart1 = re.compile(r'\.part(\d+)\.rar$', re.I)
+ re_rarpart2 = re.compile(r'\.r(\d+)$', re.I)
re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)')
re_wrongpwd = re.compile(r'password', re.I)
@@ -46,7 +46,7 @@ class UnRar(Extractor):
@classmethod
def checkDeps(cls):
if os.name == "nt":
- cls.CMD = join(pypath, "UnRAR.exe")
+ cls.CMD = os.path.join(pypath, "UnRAR.exe")
p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
p.communicate()
else:
@@ -54,8 +54,7 @@ class UnRar(Extractor):
p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
p.communicate()
- except OSError:
- # fallback to rar
+ except OSError: #: fallback to rar
cls.CMD = "rar"
p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE)
p.communicate()
@@ -64,97 +63,50 @@ class UnRar(Extractor):
@classmethod
- def isArchive(cls, file):
- f = basename(file).lower()
- return any(f.endswith('.%s' % ext) for ext in cls.EXTENSIONS)
-
-
- @classmethod
def getTargets(cls, files_ids):
targets = []
- for file, id in files_ids:
- if not cls.isArchive(file):
+ for filename, id in files_ids:
+ if not cls.isArchive(filename):
continue
- m = cls.re_rarpart.findall(file)
- if m:
- # only add first parts
- if int(m[0][1]) == 1:
- targets.append((file, id))
- else:
- targets.append((file, id))
+ m = cls.re_rarpart1.match(filename)
+ if not m or int(m.group(1)) is 1: #@NOTE: only add first part file
+ targets.append((filename, id))
return targets
- def check(self, out="", err=""):
- if not out or not err:
- return
-
- if err.strip():
- if self.re_wrongpwd.search(err):
- raise PasswordError
-
- elif self.re_wrongcrc.search(err):
- raise CRCError
+ def checkArchive(self):
+ p = self.call_cmd("l", "-v", self.target)
+ out, err = p.communicate()
- else: #: raise error if anything is on stderr
- raise ArchiveError(err.strip())
+ if self.re_wrongpwd.search(err):
+ return True
# output only used to check if passworded files are present
for attr in self.re_filelist.findall(out):
if attr[0].startswith("*"):
- raise PasswordError
-
-
- def verify(self):
- p = self.call_cmd("l", "-v", self.file, password=self.password)
-
- self.check(*p.communicate())
-
- if p and p.returncode:
- raise ArchiveError("Process terminated")
-
- if not self.list():
- raise ArchiveError("Empty archive")
-
-
- def isPassword(self, password):
- if isinstance(password, basestring):
- p = self.call_cmd("l", "-v", self.file, password=password)
- out, err = p.communicate()
-
- if not self.re_wrongpwd.search(err):
return True
+ self.files = self.list()
+ if not self.files:
+ raise ArchiveError("Empty Archive")
+
return False
- def repair(self):
- p = self.call_cmd("rc", self.file)
+ def checkPassword(self, password):
+ # at this point we can only verify header protected files
+ p = self.call_cmd("l", "-v", self.target, password=password)
out, err = p.communicate()
+ return False if self.re_wrongpwd.search(err) else True
- if p.returncode or err.strip():
- p = self.call_cmd("r", self.file)
- out, err = p.communicate()
-
- if p.returncode or err.strip():
- return False
- else:
- self.file = join(dirname(self.file), re.search(r'(fixed|rebuild)\.%s' % basename(self.file), out).group(0))
-
- return True
-
-
- def extract(self, progress=lambda x: None):
- self.verify()
-
- progress(0)
+ def extract(self, password=None):
command = "x" if self.fullpath else "e"
- p = self.call_cmd(command, self.file, self.out, password=self.password)
+ p = self.call_cmd(command, self.target, self.out, password=password)
renice(p.pid, self.renice)
@@ -165,8 +117,8 @@ class UnRar(Extractor):
if not c:
break
# reading a percentage sign -> set progress and restart
- if c is '%':
- progress(int(progressstring))
+ if c == '%':
+ self.notifyProgress(int(progressstring))
progressstring = ""
# not reading a digit -> therefore restart
elif c not in digits:
@@ -175,44 +127,57 @@ class UnRar(Extractor):
else:
progressstring += c
- progress(100)
+ # retrieve stderr
+ err = p.stderr.read()
- self.files = self.list()
+ if self.re_wrongpwd.search(err):
+ raise PasswordError
- # retrieve stderr
- self.check(err=p.stderr.read())
+ elif self.re_wrongcrc.search(err):
+ raise CRCError
+
+ elif err.strip(): #: raise error if anything is on stderr
+ raise ArchiveError(err.strip())
- if p.returncode:
+ if p.returncode != 0:
raise ArchiveError("Process terminated")
+ if not self.files:
+ self.files = self.list(password)
+
def getDeleteFiles(self):
- if ".part" in basename(self.file):
- return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.I))
+ files = []
- # get files which matches .r* and filter unsuited files out
- parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.I))
+ for i in [1, 2]:
+ try:
+ dir, name = os.path.split(self.target)
+ part = self.getattr(self, "re_rarpart%d" % i).match(name).group(1)
+ filename = os.path.join(dir, name.replace(part, '*', 1))
+ files.extend(glob(filename))
- return filter(lambda x: self.re_rarfile.match(x), parts)
+ except Exception:
+ continue
+ if self.target not in files:
+ files.insert(0, self.target)
- def list(self):
+ return files
+
+
+ def list(self, password=None):
command = "vb" if self.fullpath else "lb"
- p = self.call_cmd(command, "-v", self.file, password=self.password)
+ p = self.call_cmd(command, "-v", self.target, password=password)
out, err = p.communicate()
- if err.strip():
- self.m.logError(err)
- if "Cannot open" in err:
- return list()
+ if "Cannot open" in err:
+ raise ArchiveError("Cannot open file")
- if p.returncode:
- self.m.logError("Process terminated")
- return list()
+ if err.strip(): #: only log error at this point
+ self.manager.logError(err.strip())
result = set()
-
for f in decode(out).splitlines():
f = f.strip()
result.add(save_join(self.out, f))
@@ -248,6 +213,8 @@ class UnRar(Extractor):
# NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue
call = [self.CMD, command] + args + list(xargs)
- self.m.logDebug(" ".join(call))
- return Popen(call, stdout=PIPE, stderr=PIPE)
+ self.manager.logDebug(" ".join(call))
+
+ p = Popen(call, stdout=PIPE, stderr=PIPE)
+ return p
diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py
index 5ec56cbdf..026503be5 100644
--- a/module/plugins/internal/UnZip.py
+++ b/module/plugins/internal/UnZip.py
@@ -2,6 +2,7 @@
from __future__ import with_statement
+import os
import sys
import zipfile
@@ -10,14 +11,15 @@ from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError,
class UnZip(Extractor):
__name__ = "UnZip"
- __version__ = "1.01"
+ __version__ = "1.03"
__description__ = """Zip extractor plugin"""
__license__ = "GPLv3"
- __authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
+ __authors__ = [("RaNaN", "RaNaN@pyload.org"),
+ ("Walter Purcaro", "vuolter@gmail.com")]
- EXTENSIONS = ["zip", "zip64"]
+ EXTENSIONS = [".zip", ".zip64"]
@classmethod
@@ -26,61 +28,25 @@ class UnZip(Extractor):
@classmethod
- def isArchive(cls, file):
- return zipfile.is_zipfile(file)
+ def getTargets(cls, files_ids):
+ return [(filename, id) for filename, id in files_ids if cls.isArchive(filename)]
- def verify(self):
+ def extract(self, password=None):
try:
- with zipfile.ZipFile(self.file, 'r', allowZip64=True) as z:
+ with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z:
z.setpassword(self.password)
- badcrc = z.testzip()
+ if not z.testzip():
+ z.extractall(self.out)
+ self.files = z.namelist()
+ else:
+ raise CRCError
except (BadZipfile, LargeZipFile), e:
raise ArchiveError(e)
except RuntimeError, e:
- if 'encrypted' in e:
+ if "encrypted" in e:
raise PasswordError
else:
raise ArchiveError(e)
-
- else:
- if badcrc:
- raise CRCError
-
- if not self.list():
- raise ArchiveError("Empty archive")
-
-
- def list(self):
- try:
- with zipfile.ZipFile(self.file, 'r', allowZip64=True) as z:
- z.setpassword(self.password)
- return z.namelist()
- except Exception:
- return list()
-
-
- def extract(self, progress=lambda x: None):
- try:
- with zipfile.ZipFile(self.file, 'r', allowZip64=True) as z:
- progress(0)
- z.extractall(self.out, pwd=self.password)
- progress(100)
-
- except (BadZipfile, LargeZipFile), e:
- raise ArchiveError(e)
-
- except RuntimeError, e:
- if e is "Bad password for file":
- raise PasswordError
- else:
- raise ArchiveError(e)
-
- finally:
- self.files = self.list()
-
-
- def getDeleteFiles(self):
- return [self.file]
diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py
index c350729ac..c3bd91e52 100644
--- a/module/plugins/internal/XFSAccount.py
+++ b/module/plugins/internal/XFSAccount.py
@@ -23,7 +23,7 @@ class XFSAccount(Account):
HOSTER_DOMAIN = None
HOSTER_URL = None
- COOKIES = [(HOSTER_DOMAIN, "lang", "english")]
+ COOKIES = True
PREMIUM_PATTERN = r'\(Premium only\)'
@@ -46,9 +46,15 @@ class XFSAccount(Account):
def init(self):
if not self.HOSTER_DOMAIN:
self.logError(_("Missing HOSTER_DOMAIN"))
+ self.COOKIES = False
- if not self.HOSTER_URL:
- self.HOSTER_URL = "http://www.%s/" % (self.HOSTER_DOMAIN or "")
+ else:
+ if not self.HOSTER_URL:
+ self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN
+
+ if isinstance(self.COOKIES, list):
+ self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english"))
+ set_cookies(req.cj, self.COOKIES)
def loadAccountInfo(self, user, req):
@@ -57,6 +63,12 @@ class XFSAccount(Account):
leechtraffic = None
premium = None
+ if not self.HOSTER_URL: #@TODO: Remove in 0.4.10
+ return {'validuntil' : validuntil,
+ 'trafficleft' : trafficleft,
+ 'leechtraffic': leechtraffic,
+ 'premium' : premium}
+
html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True)
premium = True if re.search(self.PREMIUM_PATTERN, html) else False
@@ -136,22 +148,25 @@ class XFSAccount(Account):
else:
self.logDebug("LEECH_TRAFFIC_PATTERN not found")
- return {'validuntil': validuntil, 'trafficleft': trafficleft, 'leechtraffic': leechtraffic, 'premium': premium}
+ return {'validuntil' : validuntil,
+ 'trafficleft' : trafficleft,
+ 'leechtraffic': leechtraffic,
+ 'premium' : premium}
def login(self, user, data, req):
- if isinstance(self.COOKIES, list):
- set_cookies(req.cj, self.COOKIES)
+ if not self.HOSTER_URL: #@TODO: Remove in 0.4.10
+ raise Exception(_("Missing HOSTER_DOMAIN"))
url = urljoin(self.HOSTER_URL, "login.html")
html = req.load(url, decode=True)
action, inputs = parseHtmlForm('name="FL"', html)
if not inputs:
- inputs = {'op': "login",
+ inputs = {'op' : "login",
'redirect': self.HOSTER_URL}
- inputs.update({'login': user,
+ inputs.update({'login' : user,
'password': data['password']})
html = req.load(self.HOSTER_URL, post=inputs, decode=True)
diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py
index 3cb837aac..665e13b18 100644
--- a/module/plugins/internal/XFSCrypter.py
+++ b/module/plugins/internal/XFSCrypter.py
@@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo
class XFSCrypter(SimpleCrypter):
__name__ = "XFSCrypter"
__type__ = "crypter"
- __version__ = "0.05"
+ __version__ = "0.06"
__pattern__ = r'^unmatchable$'
@@ -19,10 +19,27 @@ class XFSCrypter(SimpleCrypter):
URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')]
- COOKIES = [(HOSTER_DOMAIN, "lang", "english")]
-
LINK_PATTERN = r'<(?:td|TD).*?>\s*<a href="(.+?)".*?>.+?(?:</a>)?\s*</(?:td|TD)>'
NAME_PATTERN = r'<[tT]itle>.*?\: (?P<N>.+) folder</[tT]itle>'
OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)'
TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)'
+
+
+ def prepare(self):
+ if not self.HOSTER_DOMAIN:
+ if self.account:
+ account = self.account
+ else:
+ account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "")
+ account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name)
+
+ if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN:
+ self.HOSTER_DOMAIN = account.HOSTER_DOMAIN
+ else:
+ self.fail(_("Missing HOSTER_DOMAIN"))
+
+ if isinstance(self.COOKIES, list):
+ self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english"))
+
+ return super(XFSCrypter, self).prepare()
diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py
index b32f5978f..2054b1824 100644
--- a/module/plugins/internal/XFSHoster.py
+++ b/module/plugins/internal/XFSHoster.py
@@ -4,6 +4,7 @@ import re
from random import random
from time import sleep
+from urlparse import urljoin, urlparse
from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME
@@ -15,7 +16,7 @@ from module.utils import html_unescape
class XFSHoster(SimpleHoster):
__name__ = "XFSHoster"
__type__ = "hoster"
- __version__ = "0.35"
+ __version__ = "0.41"
__pattern__ = r'^unmatchable$'
@@ -29,7 +30,6 @@ class XFSHoster(SimpleHoster):
HOSTER_DOMAIN = None
TEXT_ENCODING = False
- COOKIES = [(HOSTER_DOMAIN, "lang", "english")]
DIRECT_LINK = None
MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy...
@@ -56,17 +56,28 @@ class XFSHoster(SimpleHoster):
def setup(self):
- self.chunkLimit = 1
+ self.chunkLimit = -1 if self.premium else 1
self.resumeDownload = self.multiDL = self.premium
def prepare(self):
""" Initialize important variables """
if not self.HOSTER_DOMAIN:
- self.fail(_("Missing HOSTER_DOMAIN"))
+ if self.account:
+ account = self.account
+ else:
+ account = self.pyfile.m.core.accountManager.getAccountPlugin(self.__name__)
+
+ if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN:
+ self.HOSTER_DOMAIN = account.HOSTER_DOMAIN
+ else:
+ self.fail(_("Missing HOSTER_DOMAIN"))
+
+ if isinstance(self.COOKIES, list):
+ self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english"))
if not self.LINK_PATTERN:
- pattern = r'(https?://(www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]'
+ pattern = r'(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]'
self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.')
self.captcha = None
@@ -75,15 +86,19 @@ class XFSHoster(SimpleHoster):
super(XFSHoster, self).prepare()
if self.DIRECT_LINK is None:
- self.directDL = bool(self.premium)
+ self.directDL = self.premium
- def downloadLink(self, link):
+ def downloadLink(self, link, disposition=False): #@TODO: Set `disposition=True` in 0.4.10
if link and isinstance(link, basestring):
- if self.captcha:
- self.correctCaptcha()
+ self.correctCaptcha()
+
+ if not urlparse(link).scheme:
+ url_p = urlparse(self.pyfile.url)
+ baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
+ link = urljoin(baseurl, link)
- self.download(link, ref=True, cookies=True, disposition=False) #@TODO: Set `disposition=True` in 0.4.10
+ self.download(link, ref=False, disposition=disposition)
elif self.errmsg:
if 'captcha' in self.errmsg:
@@ -311,25 +326,25 @@ class XFSHoster(SimpleHoster):
recaptcha = ReCaptcha(self)
try:
captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1)
- except:
+ except Exception:
captcha_key = recaptcha.detect_key()
else:
self.logDebug("ReCaptcha key: %s" % captcha_key)
if captcha_key:
- inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key)
+ inputs['recaptcha_response_field'], inputs['recaptcha_challenge_field'] = recaptcha.challenge(captcha_key)
return 3
solvemedia = SolveMedia(self)
try:
captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1)
- except:
+ except Exception:
captcha_key = solvemedia.detect_key()
else:
self.logDebug("SolveMedia key: %s" % captcha_key)
if captcha_key:
- inputs['adcopy_challenge'], inputs['adcopy_response'] = solvemedia.challenge(captcha_key)
+ inputs['adcopy_response'], inputs['adcopy_challenge'] = solvemedia.challenge(captcha_key)
return 4
return 0
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-08 05:08:44 +0100