diff options
Diffstat (limited to 'module/plugins/internal')
| -rw-r--r-- | module/plugins/internal/Addon.py | 8 | ||||
| -rw-r--r-- | module/plugins/internal/Captcha.py | 14 | ||||
| -rw-r--r-- | module/plugins/internal/Container.py | 2 | ||||
| -rw-r--r-- | module/plugins/internal/Hoster.py | 62 | ||||
| -rw-r--r-- | module/plugins/internal/Plugin.py | 31 | ||||
| -rw-r--r-- | module/plugins/internal/SimpleCrypter.py | 5 | ||||
| -rw-r--r-- | module/plugins/internal/SimpleHoster.py | 27 | ||||
| -rw-r--r-- | module/plugins/internal/XFSAccount.py | 15 |
8 files changed, 112 insertions, 52 deletions
diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 45ca98eac..4ccaaba8b 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -25,7 +25,7 @@ def threaded(fn): class Addon(Plugin): __name__ = "Addon" __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 - __version__ = "0.04" + __version__ = "0.05" __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] @@ -57,6 +57,12 @@ class Addon(Plugin): self.init_events() + #@TODO: Remove in 0.4.10 + def _log(self, level, plugintype, pluginname, messages): + plugintype = "addon" if plugintype is "hook" else plugintype + return super(Addon, self)._log(level, plugintype, pluginname, messages) + + def init_events(self): if self.event_map: for event, funcs in self.event_map.items(): diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index c08050ee8..b864fd2d8 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -12,7 +12,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.42" + __version__ = "0.43" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -50,18 +50,18 @@ class Captcha(Plugin): pass - def decrypt(self, url, get={}, post={}, ref=False, cookies=False, decode=False, + def decrypt(self, url, get={}, post={}, ref=False, cookies=True, decode=False, input_type='jpg', output_type='textual', ocr=True, timeout=120): img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode) return self._decrypt(img, input_type, output_type, ocr, timeout) #@TODO: Definitely choose a better name for this method! - def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=False, timeout=120): + def _decrypt(self, data, input_type='jpg', output_type='textual', ocr=False, timeout=120): """ Loads a captcha and decrypts it with ocr, plugin, user input - :param raw: image raw data + :param data: image raw data :param get: get part for request :param post: post part for request :param cookies: True if cookies should be enabled @@ -77,7 +77,7 @@ class Captcha(Plugin): time_ref = ("%.2f" % time.time())[-6:].replace(".", "") with open(os.path.join("tmp", "captcha_image_%s_%s.%s" % (self.plugin.__name__, time_ref, input_type)), "wb") as tmp_img: - tmp_img.write(raw) + tmp_img.write(data) if ocr: if isinstance(ocr, basestring): @@ -90,7 +90,7 @@ class Captcha(Plugin): captchaManager = self.pyload.captchaManager try: - self.task = captchaManager.newTask(raw, input_type, tmp_img.name, output_type) + self.task = captchaManager.newTask(data, input_type, tmp_img.name, output_type) captchaManager.handleCaptcha(self.task) @@ -120,7 +120,7 @@ class Captcha(Plugin): self.log_warning(_("Error removing: %s") % tmp_img.name, e) traceback.print_exc() - self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? + #self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? return result diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 729592a0d..66e1ad904 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -63,7 +63,7 @@ class Container(Crypter): f.write(content) except IOError, e: - self.fail(str(e)) #@TODO: Remove `str` in 0.4.10 + self.fail(e) else: self.pyfile.name = os.path.basename(self.pyfile.url) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index b397a92a6..ac42182c4 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.20" + __version__ = "0.22" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -368,7 +368,7 @@ class Hoster(Plugin): url = self.fixurl(url) if not url or not isinstance(url, basestring): - self.fail(_("No url given")) + self.fail(_("No given url")) if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, @@ -412,7 +412,7 @@ class Hoster(Plugin): if disposition and newname: finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] - if finalname != newname != self.pyfile.name: + if finalname != newname: try: os.rename(fs_join(location, newname), fs_join(location, finalname)) @@ -421,8 +421,9 @@ class Hoster(Plugin): finalname = newname self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname)) - self.pyfile.name = finalname - filename = os.path.join(location, finalname) + + self.pyfile.name = finalname + filename = os.path.join(location, finalname) self.set_permissions(fs_encode(filename)) @@ -431,9 +432,35 @@ class Hoster(Plugin): return self.last_download - def check_download(self, rules, delete=False, file_size=0, size_tolerance=1024, read_size=1048576): + def check_filesize(self, file_size, size_tolerance=1024): + """ + Checks the file size of the last downloaded file + + :param file_size: expected file size + :param size_tolerance: size check tolerance """ - Checks the content of the last downloaded file, re match is saved to `lastCheck` + if not self.last_download: + return + + download_size = os.stat(fs_encode(self.last_download)).st_size + + if download_size < 1: + self.fail(_("Empty file")) + + elif file_size > 0: + diff = abs(file_size - download_size) + + if diff > size_tolerance: + self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") + % (file_size, download_size)) + + elif diff != 0: + self.log_warning(_("File size is not equal to expected size")) + + + def check_download(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): + """ + Checks the content of the last downloaded file, re match is saved to `last_check` :param rules: dict with names and rules to match (compiled regexp or strings) :param delete: delete if matched @@ -446,26 +473,11 @@ class Hoster(Plugin): last_download = fs_encode(self.last_download) if not self.last_download or not exists(last_download): - self.last_download = "" + self.last_download = "" #@NOTE: Bad place... self.fail(self.pyfile.error or _("No file downloaded")) try: - download_size = os.stat(last_download).st_size - - if download_size < 1: - do_delete = True - self.fail(_("Empty file")) - - elif file_size > 0: - diff = abs(file_size - download_size) - - if diff > size_tolerance: - do_delete = True - self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") - % (file_size, download_size)) - - elif diff != 0: - self.log_warning(_("File size is not equal to expected size")) + self.check_filesize(file_size, size_tolerance) with open(last_download, "rb") as f: content = f.read(read_size) @@ -495,8 +507,8 @@ class Hoster(Plugin): traceback.print_exc() else: + self.log_info(_("File deleted: ") + self.last_download) self.last_download = "" - self.log_info(_("File deleted")) def direct_link(self, url, follow_location=None): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 8bc5d4a65..feabe9f1d 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -52,11 +52,40 @@ def fixurl(url): return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip().rstrip('/') +def fixname(m): + m = unicodedata.normalize('NFKD', m) + output = '' + for c in m: + if not unicodedata.combining(c): + output += c + return output + + #@TODO: Move to utils in 0.4.10 def timestamp(): return int(time.time() * 1000) +#@TODO: Move to utils in 0.4.10 +def which(program): + """ + Works exactly like the unix command which + Courtesy of http://stackoverflow.com/a/377028/675646 + """ + isExe = lambda x: os.path.isfile(x) and os.access(x, os.X_OK) + + fpath, fname = os.path.split(program) + + if fpath: + if isExe(program): + return program + else: + for path in os.environ['PATH'].split(os.pathsep): + exe_file = os.path.join(path.strip('"'), program) + if isExe(exe_file): + return exe_file + + def seconds_to_midnight(gmt=0): now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) @@ -325,7 +354,7 @@ class Plugin(object): url = fixurl(url) if not url or not isinstance(url, basestring): - self.fail(_("No url given")) + self.fail(_("No given url")) if self.pyload.debug: self.log_debug("LOAD URL " + url, diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 6a3f91a5b..d348f92e2 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,11 +10,11 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.60" + __version__ = "0.61" __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides pyload.config['general']['folder_per_package'] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Simple decrypter plugin""" @@ -91,6 +91,7 @@ class SimpleCrypter(Crypter, SimpleHoster): def decrypt(self, pyfile): + self.links = [] #@TODO: Recheck in 0.4.10 self.prepare() self.check_info() #@TODO: Remove in 0.4.10 diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 69f88081a..89af22c87 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -11,7 +11,7 @@ from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, encode, fixurl, replace_patterns, seconds_to_midnight, set_cookie, set_cookies +from module.plugins.internal.Plugin import Fail, encode, fixurl, fixname, replace_patterns, seconds_to_midnight, set_cookie, set_cookies from module.utils import fixup, fs_encode, parseFileSize as parse_size @@ -22,12 +22,13 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.81" + __version__ = "1.82" __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("use_premium", "bool", "Use premium account if available" , True), - ("fallback" , "bool", "Fallback to free download if premium fails", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("premium_fallback", "bool", "Fallback to free download if premium fails", True), + ("chk_filesize" , "bool", "Check file size" , True)] __description__ = """Simple hoster plugin""" __license__ = "GPLv3" @@ -174,7 +175,8 @@ class SimpleHoster(Hoster): info['status'] = 2 if 'N' in info['pattern']: - info['name'] = replace_patterns(fixurl(info['pattern']['N']), + name = fixname(info['pattern']['N']) + info['name'] = replace_patterns(fixurl(name), cls.NAME_REPLACEMENTS) if 'S' in info['pattern']: @@ -294,7 +296,7 @@ class SimpleHoster(Hoster): self.check_file() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 - if self.get_config('fallback', True) and self.premium: + if self.get_config('premium_fallback', True) and self.premium: self.log_warning(_("Premium download failed"), e) self.restart(nopremium=True) @@ -309,15 +311,16 @@ class SimpleHoster(Hoster): self.captcha.invalid() self.retry(10, reason=_("Wrong captcha")) - # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size - # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, - file_size=self.info['size'] if 'size' in self.info else 0, - size_tolerance=10485760, - delete=False): #@TODO: Make `delete` settable in 0.4.10 + delete=True): self.error(_("Empty file")) else: + if self.get_config('chk_filesize', False) and 'size' in self.info: + # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size + # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB + self.check_filesize(self.info['size'], size_tolerance=10485760) + self.log_debug("Using default check rules...") for r, p in self.FILE_ERRORS: errmsg = self.check_download({r: re.compile(p)}) @@ -340,7 +343,7 @@ class SimpleHoster(Hoster): self.check_errors() self.log_info(_("No errors found")) - self.pyfile.error = "" + self.pyfile.error = "" #@TODO: Recheck in 0.4.10 def check_errors(self): diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 5a4cc35fb..3f2158a0b 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -4,6 +4,7 @@ import re import time import urlparse +from module.common.json_layer import json_loads from module.plugins.internal.Account import Account from module.plugins.internal.Plugin import parse_html_form, set_cookie @@ -11,7 +12,7 @@ from module.plugins.internal.Plugin import parse_html_form, set_cookie class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.43" + __version__ = "0.44" __status__ = "testing" __description__ = """XFileSharing account plugin""" @@ -174,5 +175,13 @@ class XFSAccount(Account): html = self.load(url, post=inputs, cookies=self.COOKIES) - if re.search(self.LOGIN_FAIL_PATTERN, html): - self.login_fail() + try: + json = json_loads(html) + + except ValueError: + if re.search(self.LOGIN_FAIL_PATTERN, html): + self.login_fail() + + else: + if not 'success' in json or not json['success']: + self.login_fail() |