From 4cdc03879a8ac9b304cd4e59498ff6646fecefcb Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Sat, 8 Aug 2015 21:41:32 +0300 Subject: Update SimpleHoster.py --- module/plugins/internal/SimpleHoster.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 9c310ca27..544e71d1e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -23,7 +23,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.80" + __version__ = "1.81" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -209,12 +209,11 @@ class SimpleHoster(Hoster): self.direct_dl = False self.leech_dl = False - if not self.get_config('use_premium', True): + if not self.get_config('use_premium', True) and self.premium: self.restart(nopremium=True) if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) - self.LOGIN_ACCOUNT = True if self.LOGIN_ACCOUNT and not self.account: self.fail(_("Required account not found")) -- cgit v1.2.3 From 28e66d852a66c4db4df7142f641900a0fe996c67 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 9 Aug 2015 03:45:36 +0200 Subject: Fix missing import statement --- module/plugins/internal/Hoster.py | 3 ++- module/plugins/internal/SimpleHoster.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index a0cdb1e2e..b397a92a6 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -3,6 +3,7 @@ from __future__ import with_statement import inspect +import mimetypes import os import random import time @@ -43,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.19" + __version__ = "0.20" __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 9c310ca27..d50cbdf9b 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -2,7 +2,6 @@ from __future__ import with_statement -import mimetypes import os import re import time -- cgit v1.2.3 From f4d40b4226872cfe5cc7de2c83a88b4bbe2ad408 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 19 Aug 2015 02:37:10 +0200 Subject: Temp fix https://github.com/pyload/pyload/pull/1708 --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index d50cbdf9b..69f88081a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -22,7 +22,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.80" + __version__ = "1.81" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -314,7 +314,7 @@ class SimpleHoster(Hoster): elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, file_size=self.info['size'] if 'size' in self.info else 0, size_tolerance=10485760, - delete=True): + delete=False): #@TODO: Make `delete` settable in 0.4.10 self.error(_("Empty file")) else: -- cgit v1.2.3 From 52dafac35c3cbfd110746b981afd80da1632a373 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 19 Aug 2015 02:38:11 +0200 Subject: Spare plugin updates --- module/plugins/internal/XFSAccount.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index e0f6b1ee8..5a4cc35fb 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -11,7 +11,7 @@ from module.plugins.internal.Plugin import parse_html_form, set_cookie class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.42" + __version__ = "0.43" __status__ = "testing" __description__ = """XFileSharing account plugin""" @@ -151,6 +151,8 @@ class XFSAccount(Account): if not self.HOSTER_URL: self.login_fail(_("Missing HOSTER_URL")) + else: + self.HOSTER_URL = self.HOSTER_URL.rstrip('/') + "/" if not self.LOGIN_URL: self.LOGIN_URL = urlparse.urljoin(self.HOSTER_URL, "login.html") -- cgit v1.2.3 From 0e87bb35a0fe1317998153bc8ef9e5072248ab2a Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Fri, 21 Aug 2015 01:03:21 +0300 Subject: fix #1765 --- module/plugins/internal/SimpleCrypter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 6a3f91a5b..9597116cc 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.60" + __version__ = "0.61" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -91,6 +91,7 @@ class SimpleCrypter(Crypter, SimpleHoster): def decrypt(self, pyfile): + self.links = [] #@TODO: Recheck in 0.4.10 self.prepare() self.check_info() #@TODO: Remove in 0.4.10 -- cgit v1.2.3 From 45feced4c09605dc5e80c4d96e2a4c5e9ee416d4 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Wed, 26 Aug 2015 15:32:53 +0300 Subject: Update Plugin.py --- module/plugins/internal/Plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 7b45c40a8..8bc5d4a65 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -7,6 +7,7 @@ import inspect import os import re import urllib +import sys if os.name != "nt": import grp @@ -146,7 +147,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.30" + __version__ = "0.32" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -345,7 +346,7 @@ class Plugin(object): #@TODO: Move to network in 0.4.10 if isinstance(decode, basestring): - res = decode(res, decode) + res = sys.modules[__name__].decode(res, decode) #@TODO: See #1787, use utils.decode() in 0.4.10 if self.pyload.debug: frame = inspect.currentframe() -- cgit v1.2.3 From 47ce7cd62c62d476094070e8d833033209c035e1 Mon Sep 17 00:00:00 2001 From: Bambie42 Date: Wed, 26 Aug 2015 14:36:59 +0200 Subject: partly fixes #1756 --- module/plugins/internal/Captcha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index c08050ee8..600fd1d34 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -120,7 +120,7 @@ class Captcha(Plugin): self.log_warning(_("Error removing: %s") % tmp_img.name, e) traceback.print_exc() - self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? + #self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? return result -- cgit v1.2.3 From dedce876f1f9cb9ad9f8b135ced26bf19fc1dbc7 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Sat, 29 Aug 2015 16:28:59 +0300 Subject: [Hoster] fix pyfile.name not updated if disposition --- module/plugins/internal/Hoster.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index b397a92a6..a8b8922bb 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.20" + __version__ = "0.21" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -412,7 +412,7 @@ class Hoster(Plugin): if disposition and newname: finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] - if finalname != newname != self.pyfile.name: + if finalname != newname try: os.rename(fs_join(location, newname), fs_join(location, finalname)) @@ -421,8 +421,9 @@ class Hoster(Plugin): finalname = newname self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname)) - self.pyfile.name = finalname - filename = os.path.join(location, finalname) + + self.pyfile.name = finalname + filename = os.path.join(location, finalname) self.set_permissions(fs_encode(filename)) -- cgit v1.2.3 From 2a3405db57862c5644643fd9b8bad0a74b0c1cbe Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Sat, 29 Aug 2015 23:02:36 +0300 Subject: Update Hoster.py --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index a8b8922bb..924672acf 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -412,7 +412,7 @@ class Hoster(Plugin): if disposition and newname: finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] - if finalname != newname + if finalname != newname: try: os.rename(fs_join(location, newname), fs_join(location, finalname)) -- cgit v1.2.3 From f44acc14fc6492f9fefbc707618c774d34ac1aeb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 30 Aug 2015 03:38:54 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1793 --- module/plugins/internal/Plugin.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 7b45c40a8..1e04d388e 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -54,6 +54,26 @@ def fixurl(url): #@TODO: Move to utils in 0.4.10 def timestamp(): return int(time.time() * 1000) + + +#@TODO: Move to utils in 0.4.10 +def which(program): + """ + Works exactly like the unix command which + Courtesy of http://stackoverflow.com/a/377028/675646 + """ + isExe = lambda x: os.path.isfile(x) and os.access(x, os.X_OK) + + fpath, fname = os.path.split(program) + + if fpath: + if isExe(program): + return program + else: + for path in os.environ['PATH'].split(os.pathsep): + exe_file = os.path.join(path.strip('"'), program) + if isExe(exe_file): + return exe_file def seconds_to_midnight(gmt=0): @@ -146,7 +166,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.30" + __version__ = "0.31" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 4cd2b7390dd97dc2016ab71f954f191de12f2f46 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 3 Sep 2015 20:37:17 +0200 Subject: Spare fixes (2) --- module/plugins/internal/Addon.py | 8 ++++- module/plugins/internal/Hoster.py | 53 +++++++++++++++++++------------- module/plugins/internal/Plugin.py | 6 ++-- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 22 +++++++------ 5 files changed, 55 insertions(+), 36 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 45ca98eac..4ccaaba8b 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -25,7 +25,7 @@ def threaded(fn): class Addon(Plugin): __name__ = "Addon" __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 - __version__ = "0.04" + __version__ = "0.05" __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] @@ -57,6 +57,12 @@ class Addon(Plugin): self.init_events() + #@TODO: Remove in 0.4.10 + def _log(self, level, plugintype, pluginname, messages): + plugintype = "addon" if plugintype is "hook" else plugintype + return super(Addon, self)._log(level, plugintype, pluginname, messages) + + def init_events(self): if self.event_map: for event, funcs in self.event_map.items(): diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 924672acf..e6bd14329 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.21" + __version__ = "0.22" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -432,9 +432,35 @@ class Hoster(Plugin): return self.last_download - def check_download(self, rules, delete=False, file_size=0, size_tolerance=1024, read_size=1048576): + def check_filesize(self, file_size, size_tolerance=1024): """ - Checks the content of the last downloaded file, re match is saved to `lastCheck` + Checks the file size of the last downloaded file + + :param file_size: expected file size + :param size_tolerance: size check tolerance + """ + if not self.last_download: + return + + download_size = os.stat(fs_encode(self.last_download)).st_size + + if download_size < 1: + self.fail(_("Empty file")) + + elif file_size > 0: + diff = abs(file_size - download_size) + + if diff > size_tolerance: + self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") + % (file_size, download_size)) + + elif diff != 0: + self.log_warning(_("File size is not equal to expected size")) + + + def check_download(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): + """ + Checks the content of the last downloaded file, re match is saved to `last_check` :param rules: dict with names and rules to match (compiled regexp or strings) :param delete: delete if matched @@ -447,26 +473,11 @@ class Hoster(Plugin): last_download = fs_encode(self.last_download) if not self.last_download or not exists(last_download): - self.last_download = "" + self.last_download = "" #@NOTE: Bad place... self.fail(self.pyfile.error or _("No file downloaded")) try: - download_size = os.stat(last_download).st_size - - if download_size < 1: - do_delete = True - self.fail(_("Empty file")) - - elif file_size > 0: - diff = abs(file_size - download_size) - - if diff > size_tolerance: - do_delete = True - self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") - % (file_size, download_size)) - - elif diff != 0: - self.log_warning(_("File size is not equal to expected size")) + self.check_filesize(file_size, size_tolerance) with open(last_download, "rb") as f: content = f.read(read_size) @@ -496,8 +507,8 @@ class Hoster(Plugin): traceback.print_exc() else: + self.log_info(_("File deleted: ") + self.last_download) self.last_download = "" - self.log_info(_("File deleted")) def direct_link(self, url, follow_location=None): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 1e04d388e..b0c0bb66e 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -54,9 +54,9 @@ def fixurl(url): #@TODO: Move to utils in 0.4.10 def timestamp(): return int(time.time() * 1000) - - -#@TODO: Move to utils in 0.4.10 + + +#@TODO: Move to utils in 0.4.10 def which(program): """ Works exactly like the unix command which diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 9597116cc..d348f92e2 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -14,7 +14,7 @@ class SimpleCrypter(Crypter, SimpleHoster): __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides pyload.config['general']['folder_per_package'] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Simple decrypter plugin""" diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 69f88081a..968a80b63 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -22,12 +22,13 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.81" + __version__ = "1.82" __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("use_premium", "bool", "Use premium account if available" , True), - ("fallback" , "bool", "Fallback to free download if premium fails", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("premium_fallback", "bool", "Fallback to free download if premium fails", True), + ("chk_filesize" , "bool", "Check file size" , True)] __description__ = """Simple hoster plugin""" __license__ = "GPLv3" @@ -294,7 +295,7 @@ class SimpleHoster(Hoster): self.check_file() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 - if self.get_config('fallback', True) and self.premium: + if self.get_config('premium_fallback', True) and self.premium: self.log_warning(_("Premium download failed"), e) self.restart(nopremium=True) @@ -309,15 +310,16 @@ class SimpleHoster(Hoster): self.captcha.invalid() self.retry(10, reason=_("Wrong captcha")) - # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size - # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, - file_size=self.info['size'] if 'size' in self.info else 0, - size_tolerance=10485760, - delete=False): #@TODO: Make `delete` settable in 0.4.10 + delete=True): self.error(_("Empty file")) else: + if self.get_config('chk_filesize', False) and 'size' in self.info: + # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size + # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB + self.check_filesize(self.info['size'], size_tolerance=10485760) + self.log_debug("Using default check rules...") for r, p in self.FILE_ERRORS: errmsg = self.check_download({r: re.compile(p)}) @@ -340,7 +342,7 @@ class SimpleHoster(Hoster): self.check_errors() self.log_info(_("No errors found")) - self.pyfile.error = "" + self.pyfile.error = "" #@TODO: Recheck in 0.4.10 def check_errors(self): -- cgit v1.2.3 From 6e6d464f35609828a14dcc09e4045b56686eb90c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Broz=CC=8C?= Date: Sun, 13 Sep 2015 08:53:26 +0200 Subject: Remove accents from file name #1817 issue: #1817 --- module/plugins/internal/Plugin.py | 10 ++++++++++ module/plugins/internal/SimpleHoster.py | 5 +++-- 2 files changed, 13 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index b0c0bb66e..fd8712b44 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -7,6 +7,7 @@ import inspect import os import re import urllib +import unicodedata if os.name != "nt": import grp @@ -51,6 +52,15 @@ def fixurl(url): return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip().rstrip('/') +def fixname(m): + m = unicodedata.normalize('NFKD', m) + output = '' + for c in m: + if not unicodedata.combining(c): + output += c + return output + + #@TODO: Move to utils in 0.4.10 def timestamp(): return int(time.time() * 1000) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 968a80b63..89af22c87 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -11,7 +11,7 @@ from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, encode, fixurl, replace_patterns, seconds_to_midnight, set_cookie, set_cookies +from module.plugins.internal.Plugin import Fail, encode, fixurl, fixname, replace_patterns, seconds_to_midnight, set_cookie, set_cookies from module.utils import fixup, fs_encode, parseFileSize as parse_size @@ -175,7 +175,8 @@ class SimpleHoster(Hoster): info['status'] = 2 if 'N' in info['pattern']: - info['name'] = replace_patterns(fixurl(info['pattern']['N']), + name = fixname(info['pattern']['N']) + info['name'] = replace_patterns(fixurl(name), cls.NAME_REPLACEMENTS) if 'S' in info['pattern']: -- cgit v1.2.3 From 36e54a3ee89aafeffdc7eab789502decf11ca2ac Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 14 Sep 2015 02:08:19 +0200 Subject: Update main Captcha plugin --- module/plugins/internal/Captcha.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 600fd1d34..b864fd2d8 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -12,7 +12,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.42" + __version__ = "0.43" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -50,18 +50,18 @@ class Captcha(Plugin): pass - def decrypt(self, url, get={}, post={}, ref=False, cookies=False, decode=False, + def decrypt(self, url, get={}, post={}, ref=False, cookies=True, decode=False, input_type='jpg', output_type='textual', ocr=True, timeout=120): img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode) return self._decrypt(img, input_type, output_type, ocr, timeout) #@TODO: Definitely choose a better name for this method! - def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=False, timeout=120): + def _decrypt(self, data, input_type='jpg', output_type='textual', ocr=False, timeout=120): """ Loads a captcha and decrypts it with ocr, plugin, user input - :param raw: image raw data + :param data: image raw data :param get: get part for request :param post: post part for request :param cookies: True if cookies should be enabled @@ -77,7 +77,7 @@ class Captcha(Plugin): time_ref = ("%.2f" % time.time())[-6:].replace(".", "") with open(os.path.join("tmp", "captcha_image_%s_%s.%s" % (self.plugin.__name__, time_ref, input_type)), "wb") as tmp_img: - tmp_img.write(raw) + tmp_img.write(data) if ocr: if isinstance(ocr, basestring): @@ -90,7 +90,7 @@ class Captcha(Plugin): captchaManager = self.pyload.captchaManager try: - self.task = captchaManager.newTask(raw, input_type, tmp_img.name, output_type) + self.task = captchaManager.newTask(data, input_type, tmp_img.name, output_type) captchaManager.handleCaptcha(self.task) -- cgit v1.2.3 From f23c2ed39b842c115751937ebc3d1b1138a9ad88 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 19 Sep 2015 06:48:27 +0200 Subject: [XFSAccount] Update --- module/plugins/internal/Container.py | 2 +- module/plugins/internal/Hoster.py | 2 +- module/plugins/internal/Plugin.py | 2 +- module/plugins/internal/XFSAccount.py | 15 ++++++++++++--- 4 files changed, 15 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 729592a0d..66e1ad904 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -63,7 +63,7 @@ class Container(Crypter): f.write(content) except IOError, e: - self.fail(str(e)) #@TODO: Remove `str` in 0.4.10 + self.fail(e) else: self.pyfile.name = os.path.basename(self.pyfile.url) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index e6bd14329..ac42182c4 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -368,7 +368,7 @@ class Hoster(Plugin): url = self.fixurl(url) if not url or not isinstance(url, basestring): - self.fail(_("No url given")) + self.fail(_("No given url")) if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index fd8712b44..fc05e2a5b 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -354,7 +354,7 @@ class Plugin(object): url = fixurl(url) if not url or not isinstance(url, basestring): - self.fail(_("No url given")) + self.fail(_("No given url")) if self.pyload.debug: self.log_debug("LOAD URL " + url, diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 5a4cc35fb..3f2158a0b 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -4,6 +4,7 @@ import re import time import urlparse +from module.common.json_layer import json_loads from module.plugins.internal.Account import Account from module.plugins.internal.Plugin import parse_html_form, set_cookie @@ -11,7 +12,7 @@ from module.plugins.internal.Plugin import parse_html_form, set_cookie class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.43" + __version__ = "0.44" __status__ = "testing" __description__ = """XFileSharing account plugin""" @@ -174,5 +175,13 @@ class XFSAccount(Account): html = self.load(url, post=inputs, cookies=self.COOKIES) - if re.search(self.LOGIN_FAIL_PATTERN, html): - self.login_fail() + try: + json = json_loads(html) + + except ValueError: + if re.search(self.LOGIN_FAIL_PATTERN, html): + self.login_fail() + + else: + if not 'success' in json or not json['success']: + self.login_fail() -- cgit v1.2.3 From 97d25752691aa561c29a91166fdd30302bef2db2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 21 Sep 2015 00:54:03 +0200 Subject: [Account] parse_info -> grab_info --- module/plugins/internal/Account.py | 10 +++++----- module/plugins/internal/XFSAccount.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 2713e8da4..07a4a2f81 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -190,7 +190,7 @@ class Account(Plugin): def get_info(self, user, reload=False): """ Retrieve account infos for an user, do **not** overwrite this method!\\ - just use it to retrieve infos in hoster plugins. see `parse_info` + just use it to retrieve infos in hoster plugins. see `grab_info` :param user: username :param reload: reloads cached account information @@ -235,7 +235,7 @@ class Account(Plugin): try: self.req = self.get_request(user) - extra_info = self.parse_info(user, info['login']['password'], info, self.req) + extra_info = self.grab_info(user, info['login']['password'], info, self.req) if extra_info and isinstance(extra_info, dict): info['data'].update(extra_info) @@ -253,7 +253,7 @@ class Account(Plugin): return info - def parse_info(self, user, password, info, req): + def grab_info(self, user, password, info, req): """ This should be overwritten in account plugin and retrieving account information for user @@ -270,8 +270,8 @@ class Account(Plugin): return [self.getAccountData(user, *args, **kwargs) for user, info in self.info.items()] - def login_fail(self, reason=_("Login handshake has failed")): - return self.fail(reason) + def fail_login(self, msg=_("Login handshake has failed")): + return self.fail(msg) def get_request(self, user=None): diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 3f2158a0b..bef8fc828 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -40,7 +40,7 @@ class XFSAccount(Account): LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<' - def parse_info(self, user, password, data, req): + def grab_info(self, user, password, data, req): validuntil = None trafficleft = None leechtraffic = None @@ -151,7 +151,7 @@ class XFSAccount(Account): set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") if not self.HOSTER_URL: - self.login_fail(_("Missing HOSTER_URL")) + self.fail_login(_("Missing HOSTER_URL")) else: self.HOSTER_URL = self.HOSTER_URL.rstrip('/') + "/" @@ -180,8 +180,8 @@ class XFSAccount(Account): except ValueError: if re.search(self.LOGIN_FAIL_PATTERN, html): - self.login_fail() + self.fail_login() else: if not 'success' in json or not json['success']: - self.login_fail() + self.fail_login() -- cgit v1.2.3 From 59d2ad3541bf133ddd69fd3b7c633e7e226e4829 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 21 Sep 2015 01:08:35 +0200 Subject: Spare improvements and fixes --- module/plugins/internal/Addon.py | 9 +- module/plugins/internal/Captcha.py | 7 +- module/plugins/internal/Extractor.py | 56 +++------- module/plugins/internal/Hoster.py | 176 ++++++++++++++++++++----------- module/plugins/internal/Plugin.py | 41 +++---- module/plugins/internal/SevenZip.py | 32 ++---- module/plugins/internal/SimpleCrypter.py | 8 +- module/plugins/internal/SimpleHoster.py | 21 ++-- module/plugins/internal/UnRar.py | 43 ++++---- module/plugins/internal/UnZip.py | 14 +-- module/plugins/internal/XFSHoster.py | 8 +- 11 files changed, 210 insertions(+), 205 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 4ccaaba8b..35375046c 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -113,18 +113,15 @@ class Addon(Plugin): pass - def __repr__(self): - return "" % self.__name__ - - - def is_activated(self): + @property + def activated(self): """ Checks if addon is activated """ return self.get_config("activated") - #: Deprecated method, use `is_activated` instead (Remove in 0.4.10) + #: Deprecated method, use `activated` property instead (Remove in 0.4.10) def isActivated(self, *args, **kwargs): return self.is_activated(*args, **kwargs) diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index b864fd2d8..dd5493ce9 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -12,7 +12,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.43" + __version__ = "0.44" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -96,8 +96,7 @@ class Captcha(Plugin): self.task.setWaiting(max(timeout, 50)) #@TODO: Move to `CaptchaManager` in 0.4.10 while self.task.isWaiting(): - if self.plugin.pyfile.abort: - self.plugin.abort() + self.plugin.check_abort() time.sleep(1) finally: @@ -108,7 +107,7 @@ class Captcha(Plugin): elif not self.task.result: self.invalid() - self.plugin.retry(reason=_("No captcha result obtained in appropiate time")) + self.plugin.retry(msg=_("No captcha result obtained in appropiate time")) result = self.task.result diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 7f5212090..f21fe473c 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.33" + __version__ = "0.34" __status__ = "testing" __description__ = """Base extractor plugin""" @@ -42,16 +42,10 @@ class Extractor(Plugin): return any(name.endswith(ext) for ext in cls.EXTENSIONS) - @classmethod - def is_multipart(cls, filename): - return False - - @classmethod def find(cls): """ Check if system statisfy dependencies - :return: boolean """ pass @@ -72,9 +66,15 @@ class Extractor(Plugin): if pname not in processed: processed.append(pname) targets.append((fname, id, fout)) + return targets + @property + def target(self): + return fs_encode(self.filename) + + def __init__(self, plugin, filename, out, fullpath=True, overwrite=False, @@ -119,53 +119,29 @@ class Extractor(Plugin): (self.__name__,) + messages) - def check(self): + def verify(self, password=None): """ - Quick Check by listing content of archive. - Raises error if password is needed, integrity is questionable or else. - - :raises PasswordError - :raises CRCError - :raises ArchiveError + Testing with Extractors built-in method + Raise error if password is needed, integrity is questionable or else """ - raise NotImplementedError - - - def verify(self): - """ - Testing with Extractors buildt-in method - Raises error if password is needed, integrity is questionable or else. - - :raises PasswordError - :raises CRCError - :raises ArchiveError - """ - raise NotImplementedError + pass def repair(self): - return None + return False def extract(self, password=None): """ - Extract the archive. Raise specific errors in case of failure. - - :param progress: Progress function, call this to update status - :param password password to use - :raises PasswordError - :raises CRCError - :raises ArchiveError - :return: + Extract the archive + Raise specific errors in case of failure """ raise NotImplementedError - def get_delete_files(self): + def items(self): """ - Return list of files to delete, do *not* delete them here. - - :return: List with paths of files to delete + Return list of archive parts """ return [self.filename] diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index ac42182c4..3449c8128 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.22" + __version__ = "0.23" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -105,6 +105,16 @@ class Hoster(Plugin): self.init() + def _log(self, level, plugintype, pluginname, messages): + log = getattr(self.pyload.log, level) + msg = " | ".join(encode(a).strip() for a in messages if a) + log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" + % {'plugintype': plugintype.upper(), + 'pluginname': pluginname, + 'id' : self.pyfile.id, + 'msg' : msg}) + + @classmethod def get_info(cls, url="", html=""): url = _fixurl(url) @@ -132,6 +142,11 @@ class Hoster(Plugin): def _setup(self): + #@TODO: Remove in 0.4.10 + self.html = "" + self.last_download = "" + self.pyfile.error = "" + if self.account: self.req = self.pyload.requestFactory.getRequest(self.__name__, self.user) self.chunk_limit = -1 #: -1 for unlimited @@ -143,23 +158,10 @@ class Hoster(Plugin): self.resume_download = False self.premium = False - - def load_account(self): - if self.req: - self.req.close() - - if not self.account: - self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - - if self.account: - if not self.user: - self.user = self.account.select()[0] - - if not self.user or not self.account.is_logged(self.user, True): - self.account = False + return self.setup() - def preprocessing(self, thread): + def _process(self, thread): """ Handles important things to do before starting """ @@ -172,19 +174,36 @@ class Hoster(Plugin): self.retry_free = False self._setup() - self.setup() + self.pyfile.setStatus("starting") self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 - if self.pyfile.abort: - self.abort() + self.check_abort() - self.pyfile.setStatus("starting") self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) - return self.process(self.pyfile) + #: Deprecated method, use `_process` instead (Remove in 0.4.10) + def preprocessing(self, *args, **kwargs): + return self._process(*args, **kwargs) + + + def load_account(self): + if self.req: + self.req.close() + + if not self.account: + self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) + + if self.account: + if not self.user: + self.user = self.account.select()[0] + + if not self.user or not self.account.is_logged(self.user, True): + self.account = False + + def process(self, pyfile): """ The 'main' method of every plugin, you **have to** overwrite it @@ -242,15 +261,12 @@ class Hoster(Plugin): self.log_warning("Ignore reconnection due logged account") while pyfile.waitUntil > time.time(): - if pyfile.abort: - self.abort() - + self.check_abort() time.sleep(2) else: while pyfile.waitUntil > time.time(): - if pyfile.abort: - self.abort() + self.check_abort() if self.thread.m.reconnecting.isSet(): self.waiting = False @@ -264,78 +280,94 @@ class Hoster(Plugin): pyfile.status = status #@NOTE: Remove in 0.4.10 - def skip(self, reason=""): + def skip(self, msg=""): """ - Skip and give reason + Skip and give msg """ - raise Skip(encode(reason)) #@TODO: Remove `encode` in 0.4.10 + raise Skip(encode(msg or self.pyfile.error)) #@TODO: Remove `encode` in 0.4.10 - def abort(self, reason=""): + #@TODO: Remove in 0.4.10 + def fail(self, msg): """ - Abort and give reason + Fail and give msg """ - #@TODO: Remove in 0.4.10 - if reason: - self.pyfile.error = encode(reason) + msg = msg.strip() + + if msg: + self.pyfile.error = msg + else: + msg = self.pyfile.error + + raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 + + + def error(self, msg="", type=_("Parse")): + type = _("%s error") % type.strip().capitalize() if type else _("Unknown") + msg = _("%(type)s: %(msg)s | Plugin may be out of date" + % {'type': type, 'msg': msg or self.pyfile.error}) + + self.fail(msg) + + + def abort(self, msg=""): + """ + Abort and give msg + """ + if msg: #@TODO: Remove in 0.4.10 + self.pyfile.error = encode(msg) raise Abort - def offline(self, reason=""): + #@TODO: Recheck in 0.4.10 + def offline(self, msg=""): """ Fail and indicate file is offline """ - #@TODO: Remove in 0.4.10 - if reason: - self.pyfile.error = encode(reason) - - raise Fail("offline") + self.fail("offline") - def temp_offline(self, reason=""): + #@TODO: Recheck in 0.4.10 + def temp_offline(self, msg=""): """ Fail and indicates file ist temporary offline, the core may take consequences """ - #@TODO: Remove in 0.4.10 - if reason: - self.pyfile.error = encode(reason) - - raise Fail("temp. offline") + self.fail("temp. offline") - def retry(self, max_tries=5, wait_time=1, reason=""): + def retry(self, max_tries=5, wait_time=1, msg=""): """ Retries and begin again from the beginning :param max_tries: number of maximum retries :param wait_time: time to wait in seconds - :param reason: reason for retrying, will be passed to fail if max_tries reached + :param msg: msg for retrying, will be passed to fail if max_tries reached """ id = inspect.currentframe().f_back.f_lineno if id not in self.retries: self.retries[id] = 0 if 0 < max_tries <= self.retries[id]: - self.fail(reason or _("Max retries reached")) + self.fail(msg or _("Max retries reached")) self.wait(wait_time, False) self.retries[id] += 1 - raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 + raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def restart(self, reason=None, nopremium=False): - if not reason: - reason = _("Fallback to free download") if nopremium else _("Restart") + def restart(self, msg=None, nopremium=False): + if not msg: + msg = _("Fallback to free download") if nopremium else _("Restart") if nopremium: if self.premium: self.retry_free = True else: - self.fail("%s | %s" % (reason, _("Download was already free"))) + self.fail("%s | %s" % (msg, _("Download was already free"))) - raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 + raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 def fixurl(self, url): @@ -349,6 +381,11 @@ class Hoster(Plugin): return url + def load(self, *args, **kwargs): + self.check_abort() + return super(Hoster, self).load(*args, **kwargs) + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ Downloads the content at url to download folder @@ -362,8 +399,7 @@ class Hoster(Plugin): the filename will be changed if needed :return: The location where the file was saved """ - if self.pyfile.abort: - self.abort() + self.check_abort() url = self.fixurl(url) @@ -398,8 +434,7 @@ class Hoster(Plugin): self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename) - if self.pyfile.abort: - self.abort() + self.check_abort() try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, @@ -432,6 +467,26 @@ class Hoster(Plugin): return self.last_download + def check_abort(self): + if not self.pyfile.abort: + return + + if self.pyfile.hasStatus("failed"): + self.fail() + + elif self.pyfile.hasStatus("skipped"): + self.skip(self.pyfile.statusname) + + elif self.pyfile.hasStatus("offline"): + self.offline() + + elif self.pyfile.hasStatus("temp. offline"): + self.temp_offline() + + else: + self.abort() + + def check_filesize(self, file_size, size_tolerance=1024): """ Checks the file size of the last downloaded file @@ -473,7 +528,6 @@ class Hoster(Plugin): last_download = fs_encode(self.last_download) if not self.last_download or not exists(last_download): - self.last_download = "" #@NOTE: Bad place... self.fail(self.pyfile.error or _("No file downloaded")) try: @@ -609,8 +663,10 @@ class Hoster(Plugin): if traffic is None: return False + elif traffic == -1: return True + else: size = self.pyfile.size / 1024 self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic)) diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index feabe9f1d..e4c16846c 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -6,8 +6,9 @@ import datetime import inspect import os import re -import urllib import sys +import urllib +import unicodedata if os.name != "nt": import grp @@ -23,7 +24,7 @@ def decode(string, encoding='utf8'): if type(string) is str: return string.decode(encoding, "replace") else: - return string + return unicode(string) #@TODO: Move to utils in 0.4.10 @@ -32,7 +33,7 @@ def encode(string, encoding='utf8'): if type(string) is unicode: return string.encode(encoding, "replace") else: - return string + return str(string) #@TODO: Move to utils in 0.4.10 @@ -175,8 +176,8 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" - __type__ = "hoster" - __version__ = "0.32" + __type__ = "plugin" + __version__ = "0.33" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -195,6 +196,11 @@ class Plugin(object): self.init() + def __repr__(self): + return "<%(type)s %(name)s>" % {'type': self.__type__.capitalize() + 'name': self.__name__} + + def _init(self, core): self.pyload = core self.info = {} #: Provide information in dict here @@ -210,11 +216,10 @@ class Plugin(object): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) - msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in messages if a)) - log("%(plugintype)s %(pluginname)s%(id)s: %(msg)s" + msg = " | ".join(encode(a).strip() for a in messages if a) + log("%(plugintype)s %(pluginname)s%: %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, - 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", 'msg' : msg}) @@ -317,21 +322,10 @@ class Plugin(object): self.pyload.db.delStorage(self.__name__, key) - def fail(self, reason): + def fail(self, msg): """ - Fail and give reason + Fail and give msg """ - raise Fail(encode(reason)) #@TODO: Remove `encode` in 0.4.10 - - - def error(self, reason="", type=_("Parse")): - if not reason: - type = _("Unknown") - - msg = _("%s error") % type.strip().capitalize() if type else _("Error") - msg += (": %s" % reason.strip()) if reason else "" - msg += _(" | Plugin may be out of date") - raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 @@ -348,9 +342,6 @@ class Plugin(object): :param decode: Wether to decode the output according to http header, should be True in most cases :return: Loaded content """ - if hasattr(self, 'pyfile') and self.pyfile.abort: - self.abort() - url = fixurl(url) if not url or not isinstance(url, basestring): @@ -375,7 +366,7 @@ class Plugin(object): #@TODO: Move to network in 0.4.10 if isinstance(decode, basestring): - res = sys.modules[__name__].decode(res, decode) #@TODO: See #1787, use utils.decode() in 0.4.10 + res = sys.modules[self.__name__].decode(res, decode) #@TODO: See #1787, use utils.decode() in 0.4.10 if self.pyload.debug: frame = inspect.currentframe() diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index 5811c28de..b79256536 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -10,7 +10,7 @@ from module.utils import fs_encode, save_join as fs_join class SevenZip(UnRar): __name__ = "SevenZip" - __version__ = "0.14" + __version__ = "0.15" __status__ = "testing" __description__ = """7-Zip extractor plugin""" @@ -55,42 +55,28 @@ class SevenZip(UnRar): return True - def verify(self, password): + def verify(self, password=None): #: 7z can't distinguish crc and pw error in test - p = self.call_cmd("l", "-slt", fs_encode(self.filename)) + p = self.call_cmd("l", "-slt", self.target) out, err = p.communicate() if self.re_wrongpwd.search(out): raise PasswordError - if self.re_wrongpwd.search(err): + elif self.re_wrongpwd.search(err): raise PasswordError - if self.re_wrongcrc.search(err): - raise CRCError(err) - - - - def check(self, password): - p = self.call_cmd("l", "-slt", fs_encode(self.filename)) - out, err = p.communicate() - - #: Check if output or error macthes the 'wrong password'-Regexp - if self.re_wrongpwd.search(out): - raise PasswordError - - if self.re_wrongcrc.search(out): + elif self.re_wrongcrc.search(out): raise CRCError(_("Header protected")) - - def repair(self): - return False + elif self.re_wrongcrc.search(err): + raise CRCError(err) def extract(self, password=None): command = "x" if self.fullpath else "e" - p = self.call_cmd(command, '-o' + self.out, fs_encode(self.filename), password=password) + p = self.call_cmd(command, '-o' + self.out, self.target, password=password) renice(p.pid, self.renice) @@ -117,7 +103,7 @@ class SevenZip(UnRar): def list(self, password=None): command = "l" if self.fullpath else "l" - p = self.call_cmd(command, fs_encode(self.filename), password=password) + p = self.call_cmd(command, self.target, password=password) out, err = p.communicate() if "Can not open" in err: diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index d348f92e2..488578bc0 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.61" + __version__ = "0.62" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -90,8 +90,12 @@ class SimpleCrypter(Crypter, SimpleHoster): self.log_error(_("Too many redirects")) + def prepare(self): + self.links = [] + return super(SimpleCrypter, self).prepare() + + def decrypt(self, pyfile): - self.links = [] #@TODO: Recheck in 0.4.10 self.prepare() self.check_info() #@TODO: Remove in 0.4.10 diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 89af22c87..a23691220 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -22,7 +22,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.82" + __version__ = "1.83" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -203,12 +203,9 @@ class SimpleHoster(Hoster): def prepare(self): - self.pyfile.error = "" #@TODO: Remove in 0.4.10 - self.html = "" #@TODO: Recheck in 0.4.10 - self.link = "" #@TODO: Recheck in 0.4.10 - self.last_download = "" - self.direct_dl = False - self.leech_dl = False + self.link = "" + self.direct_dl = False + self.leech_dl = False if not self.get_config('use_premium', True): self.restart(nopremium=True) @@ -309,7 +306,7 @@ class SimpleHoster(Hoster): if self.captcha.task and not self.last_download: self.captcha.invalid() - self.retry(10, reason=_("Wrong captcha")) + self.retry(10, msg=_("Wrong captcha")) elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, delete=True): @@ -334,7 +331,7 @@ class SimpleHoster(Hoster): self.log_warning(_("Check result: ") + errmsg, _("Waiting 1 minute and retry")) self.wantReconnect = True - self.retry(wait_time=60, reason=errmsg) + self.retry(wait_time=60, msg=errmsg) else: if self.CHECK_FILE: self.log_debug("Using custom check rules...") @@ -409,7 +406,7 @@ class SimpleHoster(Hoster): elif re.search('captcha|code', errmsg, re.I): self.captcha.invalid() - self.retry(10, reason=_("Wrong captcha")) + self.retry(10, msg=_("Wrong captcha")) elif re.search('countdown|expired', errmsg, re.I): self.retry(10, 60, _("Link expired")) @@ -426,14 +423,14 @@ class SimpleHoster(Hoster): elif re.search('filename', errmsg, re.I): url_p = urlparse.urlparse(self.pyfile.url) self.pyfile.url = "%s://%s/%s" % (url_p.scheme, url_p.netloc, url_p.path.split('/')[0]) - self.retry(1, reason=_("Wrong url")) + self.retry(1, msg=_("Wrong url")) elif re.search('premium', errmsg, re.I): self.fail(_("File can be downloaded by premium users only")) else: self.wantReconnect = True - self.retry(wait_time=60, reason=errmsg) + self.retry(wait_time=60, msg=errmsg) elif hasattr(self, 'WAIT_PATTERN'): m = re.search(self.WAIT_PATTERN, self.html) diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 0386991d9..88c490750 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.25" + __version__ = "1.26" __status__ = "testing" __description__ = """Rar extractor plugin""" @@ -84,20 +84,8 @@ class UnRar(Extractor): return True if cls.re_multipart.search(filename) else False - def verify(self, password): - p = self.call_cmd("t", "-v", fs_encode(self.filename), password=password) - self._progress(p) - err = p.stderr.read().strip() - - if self.re_wrongpwd.search(err): - raise PasswordError - - if self.re_wrongcrc.search(err): - raise CRCError(err) - - - def check(self, password): - p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password) + def verify(self, password=None): + p = self.call_cmd("l", "-v", self.target, password=password) out, err = p.communicate() if self.re_wrongpwd.search(err): @@ -113,13 +101,28 @@ class UnRar(Extractor): def repair(self): - p = self.call_cmd("rc", fs_encode(self.filename)) + p = self.call_cmd("rc", self.target) #: Communicate and retrieve stderr self._progress(p) err = p.stderr.read().strip() + if err or p.returncode: - return False + p = self.call_cmd("r", self.target) + + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + + if err or p.returncode: + return False + + else: + dir = os.path.dirname(filename) + name = re_filefixed.search(out).group(1) + + self.filename = os.path.join(dir, name) + return True @@ -145,7 +148,7 @@ class UnRar(Extractor): def extract(self, password=None): command = "x" if self.fullpath else "e" - p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password) + p = self.call_cmd(command, self.target, self.out, password=password) renice(p.pid, self.renice) @@ -169,7 +172,7 @@ class UnRar(Extractor): self.files = self.list(password) - def get_delete_files(self): + def items(self): dir, name = os.path.split(self.filename) #: Actually extracted file @@ -185,7 +188,7 @@ class UnRar(Extractor): def list(self, password=None): command = "vb" if self.fullpath else "lb" - p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password) + p = self.call_cmd(command, "-v", self.target, password=password) out, err = p.communicate() if "Cannot open" in err: diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 9a01611bf..ac197a80d 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -12,7 +12,7 @@ from module.utils import fs_encode class UnZip(Extractor): __name__ = "UnZip" - __version__ = "1.15" + __version__ = "1.16" __status__ = "testing" __description__ = """Zip extractor plugin""" @@ -30,17 +30,13 @@ class UnZip(Extractor): def list(self, password=None): - with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z: z.setpassword(password) return z.namelist() - def check(self, password): - pass - - - def verify(self): - with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + def verify(self, password=None): + with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z: badfile = z.testzip() if badfile: @@ -51,7 +47,7 @@ class UnZip(Extractor): def extract(self, password=None): try: - with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z: z.setpassword(password) badfile = z.testzip() diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 5e0830dc6..1f0bd3a44 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.57" + __version__ = "0.58" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -150,7 +150,7 @@ class XFSHoster(SimpleHoster): action, inputs = self.parse_html_form('F1') if not inputs: - self.retry(reason=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) + self.retry(msg=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) self.log_debug(inputs) @@ -163,7 +163,7 @@ class XFSHoster(SimpleHoster): self.retry(20, 3 * 60, _("Can not leech file")) elif 'today' in stmsg: - self.retry(wait_time=seconds_to_midnight(gmt=2), reason=_("You've used all Leech traffic today")) + self.retry(wait_time=seconds_to_midnight(gmt=2), msg=_("You've used all Leech traffic today")) else: self.fail(stmsg) @@ -188,7 +188,7 @@ class XFSHoster(SimpleHoster): if not inputs: action, inputs = self.parse_html_form('F1') if not inputs: - self.retry(reason=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) + self.retry(msg=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) self.log_debug(inputs) -- cgit v1.2.3 From af896302d7701e2bc63b1523b9ea73f38f5201c8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 21 Sep 2015 14:01:21 +0200 Subject: Fix pyfile.name processing --- module/plugins/internal/Crypter.py | 15 +++++------ module/plugins/internal/Hoster.py | 45 ++++++++++++-------------------- module/plugins/internal/Plugin.py | 30 ++++++++++----------- module/plugins/internal/SimpleCrypter.py | 1 - module/plugins/internal/SimpleHoster.py | 20 ++++++-------- 5 files changed, 45 insertions(+), 66 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index d0e8eb1b4..2033b67df 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -1,15 +1,13 @@ # -*- coding: utf-8 -*- -import urlparse - -from module.plugins.internal.Hoster import Hoster, _fixurl +from module.plugins.internal.Hoster import Hoster, parse_name from module.utils import save_path as safe_filename class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.07" + __version__ = "0.08" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -78,13 +76,14 @@ class Crypter(Hoster): "%d links" % len(links), "Saved to folder: %s" % folder if folder else "Saved to download folder") - pid = self.pyload.api.addPackage(name, map(self.fixurl, links), package_queue) + links = map(self.fixurl, links) + pid = self.pyload.api.addPackage(name, links, package_queue) if package_password: self.pyload.api.setPackageData(pid, {'password': package_password}) #: Workaround to do not break API addPackage method - set_folder = lambda x: self.pyload.api.setPackageData(pid, {'folder': x or ""}) + set_folder = lambda x: self.pyload.api.setPackageData(pid, {'folder': safe_filename(x) or ""}) if use_subfolder: if not subfolder_per_package: @@ -93,9 +92,9 @@ class Crypter(Hoster): elif not folder_per_package or name is not folder: if not folder: - folder = urlparse.urlparse(_fixurl(name)).path.split("/")[-1] + folder = parse_name(name) - set_folder(safe_filename(folder)) + set_folder(folder) self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) elif folder_per_package: diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 3449c8128..8457420c1 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -12,16 +12,17 @@ import urlparse from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, encode, exists, fixurl as _fixurl, replace_patterns, - seconds_to_midnight, set_cookie, set_cookies, parse_html_form, - parse_html_tag_attr_value, timestamp) + chunks, encode, exists, parse_html_form, + parse_html_tag_attr_value, parse_name, + replace_patterns, seconds_to_midnight, + set_cookie, set_cookies, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename #@TODO: Remove in 0.4.10 def parse_fileInfo(klass, url="", html=""): info = klass.get_info(url, html) - return info['name'], info['size'], info['status'], info['url'] + return encode(info['name']), info['size'], info['status'], info['url'] #@TODO: Remove in 0.4.10 @@ -44,7 +45,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.23" + __version__ = "0.24" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -117,13 +118,9 @@ class Hoster(Plugin): @classmethod def get_info(cls, url="", html=""): - url = _fixurl(url) - url_p = urlparse.urlparse(url) - return {'name' : (url_p.path.split('/')[-1] or - url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or - url_p.netloc.split('.', 1)[0]), + return {'name' : parse_name(url), 'size' : 0, - 'status': 3 if url else 8, + 'status': 3 if url.strip() else 8, 'url' : url} @@ -370,11 +367,12 @@ class Hoster(Plugin): raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def fixurl(self, url): - url = _fixurl(url) + def fixurl(self, url, baseurl=None): + if not baseurl: + baseurl = self.pyfile.url if not urlparse.urlparse(url).scheme: - url_p = urlparse.urlparse(self.pyfile.url) + url_p = urlparse.urlparse(baseurl) baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) url = urlparse.urljoin(baseurl, url) @@ -401,17 +399,11 @@ class Hoster(Plugin): """ self.check_abort() - url = self.fixurl(url) - - if not url or not isinstance(url, basestring): - self.fail(_("No given url")) - if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) - name = _fixurl(self.pyfile.name) - self.pyfile.name = urlparse.urlparse(name).path.split('/')[-1] or name + self.pyfile.name = parse_name(self.pyfile.name) #: Safe check self.captcha.correct() self.check_for_same_files() @@ -445,7 +437,7 @@ class Hoster(Plugin): #@TODO: Recheck in 0.4.10 if disposition and newname: - finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] + finalname = parse_name(newname).split(' filename*=')[0] if finalname != newname: try: @@ -609,12 +601,7 @@ class Hoster(Plugin): link = url elif 'location' in header and header['location']: - location = header['location'] - - if not urlparse.urlparse(location).scheme: - url_p = urlparse.urlparse(url) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - location = urlparse.urljoin(baseurl, location) + location = self.fixurl(header['location'], url) if 'code' in header and header['code'] == 302: link = location @@ -624,7 +611,7 @@ class Hoster(Plugin): continue else: - extension = os.path.splitext(urlparse.urlparse(url).path.split('/')[-1])[-1] + extension = os.path.splitext(parse_name(url))[-1] if 'content-type' in header and header['content-type']: mimetype = header['content-type'].split(';')[0].strip() diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index e4c16846c..9494d34a6 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -7,8 +7,9 @@ import inspect import os import re import sys -import urllib import unicodedata +import urllib +import urlparse if os.name != "nt": import grp @@ -49,17 +50,19 @@ def exists(path): #@TODO: Move to utils in 0.4.10 -def fixurl(url): - return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip().rstrip('/') +def parse_name(url): + url = urllib.unquote(url) + url = url.decode('unicode-escape') + url = html_unescape(url) + url = urllib.quote(url) + + url_p = urlparse.urlparse(url.strip().rstrip('/')) + name = (url_p.path.split('/')[-1] or + url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or + url_p.netloc.split('.', 1)[0]) -def fixname(m): - m = unicodedata.normalize('NFKD', m) - output = '' - for c in m: - if not unicodedata.combining(c): - output += c - return output + return urllib.unquote(name) #@TODO: Move to utils in 0.4.10 @@ -177,7 +180,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.33" + __version__ = "0.34" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -342,11 +345,6 @@ class Plugin(object): :param decode: Wether to decode the output according to http header, should be True in most cases :return: Loaded content """ - url = fixurl(url) - - if not url or not isinstance(url, basestring): - self.fail(_("No given url")) - if self.pyload.debug: self.log_debug("LOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 488578bc0..3771956bc 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -4,7 +4,6 @@ import re from module.plugins.internal.Crypter import Crypter from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookie, set_cookies -from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 19263dd8f..7c47d3620 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -5,13 +5,12 @@ from __future__ import with_statement import os import re import time -import urlparse from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, encode, fixurl, fixname, replace_patterns, seconds_to_midnight, set_cookie, set_cookies +from module.plugins.internal.Plugin import Fail, encode, parse_name, replace_patterns, seconds_to_midnight, set_cookie, set_cookies from module.utils import fixup, fs_encode, parseFileSize as parse_size @@ -22,7 +21,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.83" + __version__ = "1.84" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -91,7 +90,7 @@ class SimpleHoster(Hoster): LINK_PREMIUM_PATTERN: (optional) group(1) should be the direct link for premium download example: LINK_PREMIUM_PATTERN = r'
Date: Mon, 21 Sep 2015 14:36:22 +0200 Subject: Spare code cosmetics --- module/plugins/internal/Hoster.py | 2 ++ module/plugins/internal/OCR.py | 2 ++ module/plugins/internal/Plugin.py | 1 + module/plugins/internal/SimpleCrypter.py | 1 + module/plugins/internal/SimpleHoster.py | 10 +++++++--- 5 files changed, 13 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 8457420c1..6ac1e95f6 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -416,6 +416,7 @@ class Hoster(Plugin): if not exists(download_location): try: os.makedirs(download_location) + except Exception, e: self.fail(e) @@ -632,6 +633,7 @@ class Hoster(Plugin): else: try: self.log_error(_("Too many redirects")) + except Exception: pass diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index b24b3058b..5913e604d 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -128,6 +128,7 @@ class OCR(Plugin): try: with open(tmpTxt.name, 'r') as f: self.result_captcha = f.read().replace("\n", "") + except Exception: self.result_captcha = "" @@ -194,6 +195,7 @@ class OCR(Plugin): count += 1 if pixels[x, y - 1] != 255: count += 1 + except Exception: pass diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 9494d34a6..bbccaa48d 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -410,6 +410,7 @@ class Plugin(object): """ try: self.req.close() + except Exception: pass diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 3771956bc..d5a5ce2d7 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -136,6 +136,7 @@ class SimpleCrypter(Crypter, SimpleHoster): def handle_pages(self, pyfile): try: pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) + except Exception: pages = 1 diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 7c47d3620..2a83389b6 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -323,6 +323,7 @@ class SimpleHoster(Hoster): try: errmsg += " | " + self.last_check.group(1).strip() + except Exception: pass @@ -359,7 +360,8 @@ class SimpleHoster(Hoster): m = re.search(self.DL_LIMIT_PATTERN, self.html) try: errmsg = m.group(1).strip() - except Exception: + + except AttributeError: errmsg = m.group(0).strip() self.info['error'] = re.sub(r'<.*?>', " ", errmsg) @@ -382,7 +384,8 @@ class SimpleHoster(Hoster): if m: try: errmsg = m.group(1).strip() - except Exception: + + except AttributeError: errmsg = m.group(0).strip() self.info['error'] = re.sub(r'<.*?>', " ", errmsg) @@ -432,7 +435,8 @@ class SimpleHoster(Hoster): if m: try: waitmsg = m.group(1).strip() - except Exception: + + except AttributeError: waitmsg = m.group(0).strip() wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in -- cgit v1.2.3 From fb45ec44220201268f99b747831df717fa7a83f8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 21 Sep 2015 16:31:48 +0200 Subject: [Account] Fix __version__ --- module/plugins/internal/Account.py | 2 +- module/plugins/internal/Addon.py | 4 ++-- module/plugins/internal/Plugin.py | 4 ++-- module/plugins/internal/XFSAccount.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 07a4a2f81..de338cd33 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -13,7 +13,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.17" + __version__ = "0.18" __status__ = "testing" __description__ = """Base account plugin""" diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 35375046c..2609c6033 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -25,7 +25,7 @@ def threaded(fn): class Addon(Plugin): __name__ = "Addon" __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 - __version__ = "0.05" + __version__ = "0.06" __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] @@ -123,7 +123,7 @@ class Addon(Plugin): #: Deprecated method, use `activated` property instead (Remove in 0.4.10) def isActivated(self, *args, **kwargs): - return self.is_activated(*args, **kwargs) + return self.activated def deactivate(self): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index bbccaa48d..118b68812 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -200,7 +200,7 @@ class Plugin(object): def __repr__(self): - return "<%(type)s %(name)s>" % {'type': self.__type__.capitalize() + return "<%(type)s %(name)s>" % {'type': self.__type__.capitalize(), 'name': self.__name__} @@ -220,7 +220,7 @@ class Plugin(object): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) msg = " | ".join(encode(a).strip() for a in messages if a) - log("%(plugintype)s %(pluginname)s%: %(msg)s" + log("%(plugintype)s %(pluginname)s: %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, 'msg' : msg}) diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index bef8fc828..392d9503c 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.Plugin import parse_html_form, set_cookie class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.44" + __version__ = "0.45" __status__ = "testing" __description__ = """XFileSharing account plugin""" -- cgit v1.2.3 From 34244c7a49b099d30456e0bd80cf26a2be3a3208 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 21 Sep 2015 23:35:58 +0200 Subject: Spare improvements and fixes (2) --- module/plugins/internal/Addon.py | 4 ---- module/plugins/internal/Captcha.py | 2 -- module/plugins/internal/Container.py | 18 ++++------------ module/plugins/internal/Hoster.py | 36 ++++++++++++++------------------ module/plugins/internal/OCR.py | 4 +--- module/plugins/internal/Plugin.py | 22 ++++++++++++------- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 8 +++---- module/plugins/internal/XFSAccount.py | 4 ++-- module/plugins/internal/XFSCrypter.py | 4 ++-- module/plugins/internal/XFSHoster.py | 10 ++++----- 11 files changed, 49 insertions(+), 65 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 2609c6033..5150e88f6 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -import traceback - from module.plugins.internal.Plugin import Plugin @@ -103,8 +101,6 @@ class Addon(Plugin): except Exception, e: self.log_error(_("Error executing periodical task: %s") % e) - if self.pyload.debug: - traceback.print_exc() self.cb = self.pyload.scheduler.addJob(self.interval, self._periodical, [threaded], threaded=threaded) diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index dd5493ce9..d2be21a58 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -4,7 +4,6 @@ from __future__ import with_statement import os import time -import traceback from module.plugins.internal.Plugin import Plugin @@ -117,7 +116,6 @@ class Captcha(Plugin): except OSError, e: self.log_warning(_("Error removing: %s") % tmp_img.name, e) - traceback.print_exc() #self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 66e1ad904..430590421 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -4,7 +4,6 @@ from __future__ import with_statement import os import re -import traceback from module.plugins.internal.Crypter import Crypter from module.plugins.internal.Plugin import exists @@ -14,7 +13,7 @@ from module.utils import save_join as fs_join class Container(Crypter): __name__ = "Container" __type__ = "container" - __version__ = "0.06" + __version__ = "0.07" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -44,11 +43,6 @@ class Container(Crypter): self._create_packages() - #: Deprecated method, use `_load2disk` instead (Remove in 0.4.10) - def loadToDisk(self, *args, **kwargs): - return self._load2disk(*args, **kwargs) - - def _load2disk(self): """ Loads container to disk if its stored remotely and overwrite url, @@ -67,16 +61,14 @@ class Container(Crypter): else: self.pyfile.name = os.path.basename(self.pyfile.url) + if not exists(self.pyfile.url): if exists(fs_join(pypath, self.pyfile.url)): self.pyfile.url = fs_join(pypath, self.pyfile.url) else: self.fail(_("File not exists")) - - - #: Deprecated method, use `delete_tmp` instead (Remove in 0.4.10) - def deleteTmp(self, *args, **kwargs): - return self.delete_tmp(*args, **kwargs) + else: + self.data = self.pyfile.url def delete_tmp(self): @@ -87,5 +79,3 @@ class Container(Crypter): os.remove(self.pyfile.url) except OSError, e: self.log_warning(_("Error removing: %s") % self.pyfile.url, e) - if self.pyload.debug: - traceback.print_exc() diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 6ac1e95f6..a77ef05dd 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -7,12 +7,11 @@ import mimetypes import os import random import time -import traceback import urlparse from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, encode, exists, parse_html_form, + chunks, decode, encode, exists, parse_html_form, parse_html_tag_attr_value, parse_name, replace_patterns, seconds_to_midnight, set_cookie, set_cookies, timestamp) @@ -45,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.24" + __version__ = "0.26" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -75,7 +74,6 @@ class Hoster(Plugin): #: Account handler instance, see :py:class:`Account` self.account = None - self.user = None self.req = None #: Browser instance, see `network.Browser` #: Associated pyfile instance, see `PyFile` @@ -108,7 +106,7 @@ class Hoster(Plugin): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) - msg = " | ".join(encode(a).strip() for a in messages if a) + msg = " | ".join(decode(a).strip() for a in messages if a) log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, @@ -145,10 +143,10 @@ class Hoster(Plugin): self.pyfile.error = "" if self.account: - self.req = self.pyload.requestFactory.getRequest(self.__name__, self.user) + self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) self.chunk_limit = -1 #: -1 for unlimited self.resume_download = True - self.premium = self.account.is_premium(self.user) + self.premium = self.account.premium else: self.req = self.pyload.requestFactory.getRequest(self.__name__) self.chunk_limit = 1 @@ -194,10 +192,10 @@ class Hoster(Plugin): self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) if self.account: - if not self.user: - self.user = self.account.select()[0] + if not self.account.user: #@TODO: Move to `Account` in 0.4.10 + self.account.user = self.account.select()[0] - if not self.user or not self.account.is_logged(self.user, True): + if not self.account.logged: self.account = False @@ -333,22 +331,22 @@ class Hoster(Plugin): self.fail("temp. offline") - def retry(self, max_tries=5, wait_time=1, msg=""): + def retry(self, attemps=5, delay=1, msg=""): """ Retries and begin again from the beginning - :param max_tries: number of maximum retries - :param wait_time: time to wait in seconds - :param msg: msg for retrying, will be passed to fail if max_tries reached + :param attemps: number of maximum retries + :param delay: time to wait in seconds + :param msg: msg for retrying, will be passed to fail if attemps value was reached """ id = inspect.currentframe().f_back.f_lineno if id not in self.retries: self.retries[id] = 0 - if 0 < max_tries <= self.retries[id]: + if 0 < attemps <= self.retries[id]: self.fail(msg or _("Max retries reached")) - self.wait(wait_time, False) + self.wait(delay, False) self.retries[id] += 1 raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 @@ -550,8 +548,6 @@ class Hoster(Plugin): except OSError, e: self.log_warning(_("Error removing: %s") % last_download, e) - if self.pyload.debug: - traceback.print_exc() else: self.log_info(_("File deleted: ") + self.last_download) @@ -648,7 +644,7 @@ class Hoster(Plugin): if not self.account: return True - traffic = self.account.get_data(self.user, True)['trafficleft'] + traffic = self.account.get_data(self.account.user, True)['trafficleft'] if traffic is None: return False @@ -658,7 +654,7 @@ class Hoster(Plugin): else: size = self.pyfile.size / 1024 - self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic)) + self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) return size <= traffic diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 5913e604d..3e5afae69 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -12,7 +12,6 @@ import logging import os import subprocess # import tempfile -import traceback from module.plugins.internal.Plugin import Plugin from module.utils import save_join as fs_join @@ -138,10 +137,9 @@ class OCR(Plugin): os.remove(tmpTxt.name) if subset and (digits or lowercase or uppercase): os.remove(tmpSub.name) + except OSError, e: self.log_warning(e) - if self.pyload.debug: - traceback.print_exc() def recognize(self, name): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 118b68812..e9964b11d 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -7,9 +7,8 @@ import inspect import os import re import sys -import unicodedata +import traceback import urllib -import urlparse if os.name != "nt": import grp @@ -180,7 +179,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.34" + __version__ = "0.35" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -227,24 +226,31 @@ class Plugin(object): def log_debug(self, *args): - if self.pyload.debug: - return self._log("debug", self.__type__, self.__name__, args) + if not self.pyload.debug: + return + self._log("debug", self.__type__, self.__name__, args) def log_info(self, *args): - return self._log("info", self.__type__, self.__name__, args) + self._log("info", self.__type__, self.__name__, args) def log_warning(self, *args): - return self._log("warning", self.__type__, self.__name__, args) + self._log("warning", self.__type__, self.__name__, args) + if self.pyload.debug: + traceback.print_exc() def log_error(self, *args): - return self._log("error", self.__type__, self.__name__, args) + self._log("error", self.__type__, self.__name__, args) + if self.pyload.debug: + traceback.print_exc() def log_critical(self, *args): return self._log("critical", self.__type__, self.__name__, args) + if self.pyload.debug: + traceback.print_exc() def set_permissions(self, path): diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index d5a5ce2d7..8c5d3599d 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -137,7 +137,7 @@ class SimpleCrypter(Crypter, SimpleHoster): try: pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) - except Exception: + except AttributeError: pages = 1 for p in xrange(2, pages + 1): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2a83389b6..fb55ef762 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -21,7 +21,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.84" + __version__ = "1.85" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -124,7 +124,7 @@ class SimpleHoster(Hoster): try: info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: Pattern groups will be saved here - except Exception: + except AttributeError: info['pattern'] = {} if not html and not online: @@ -329,7 +329,7 @@ class SimpleHoster(Hoster): self.log_warning(_("Check result: ") + errmsg, _("Waiting 1 minute and retry")) self.wantReconnect = True - self.retry(wait_time=60, msg=errmsg) + self.retry(delay=60, msg=errmsg) else: if self.CHECK_FILE: self.log_debug("Using custom check rules...") @@ -428,7 +428,7 @@ class SimpleHoster(Hoster): else: self.wantReconnect = True - self.retry(wait_time=60, msg=errmsg) + self.retry(delay=60, msg=errmsg) elif hasattr(self, 'WAIT_PATTERN'): m = re.search(self.WAIT_PATTERN, self.html) diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 392d9503c..bb5cbcf50 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.Plugin import parse_html_form, set_cookie class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.45" + __version__ = "0.46" __status__ = "testing" __description__ = """XFileSharing account plugin""" @@ -145,7 +145,7 @@ class XFSAccount(Account): self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN if self.COOKIES: - if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")): + if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) else: set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 4c059d647..12a48d4a3 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -7,7 +7,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.13" + __version__ = "0.14" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -42,7 +42,7 @@ class XFSCrypter(SimpleCrypter): self.fail(_("Missing HOSTER_DOMAIN")) if self.COOKIES: - if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")): + if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) else: set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 1f0bd3a44..145efcd8e 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.58" + __version__ = "0.59" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -73,7 +73,7 @@ class XFSHoster(SimpleHoster): self.fail(_("Missing HOSTER_DOMAIN")) if self.COOKIES: - if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")): + if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) else: set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") @@ -163,7 +163,7 @@ class XFSHoster(SimpleHoster): self.retry(20, 3 * 60, _("Can not leech file")) elif 'today' in stmsg: - self.retry(wait_time=seconds_to_midnight(gmt=2), msg=_("You've used all Leech traffic today")) + self.retry(delay=seconds_to_midnight(gmt=2), msg=_("You've used all Leech traffic today")) else: self.fail(stmsg) @@ -244,7 +244,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) - except Exception: + except AttributeError: captcha_key = recaptcha.detect_key() else: @@ -258,7 +258,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) - except Exception: + except AttributeError: captcha_key = solvemedia.detect_key() else: -- cgit v1.2.3 From 5748b8ed8222dac07e9d40ce582ca7f8ae591cb8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 22 Sep 2015 00:33:01 +0200 Subject: Auto-detect seconds_to_midnight --- module/plugins/internal/Plugin.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index e9964b11d..b651079c2 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -89,22 +89,15 @@ def which(program): return exe_file -def seconds_to_midnight(gmt=0): - now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) - - if now.hour == 0 and now.minute < 10: - midnight = now +def seconds_to_midnight(gmt=None): + if gmt is None: + now = datetime.datetime.today() else: - midnight = now + datetime.timedelta(days=1) - - td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now + now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) - if hasattr(td, 'total_seconds'): - res = td.total_seconds() - else: #@NOTE: work-around for python 2.5 and 2.6 missing datetime.timedelta.total_seconds - res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 + midnight = now.replace(hour=0, minute=10, second=0, microsecond=0) + datetime.timedelta(days=1) - return int(res) + return (midnight - now).seconds def replace_patterns(string, ruleslist): -- cgit v1.2.3 From 553538287293b8a52eee41f28d57d0b4ae1003aa Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 22 Sep 2015 01:57:37 +0300 Subject: [Plugin] fix missing import --- module/plugins/internal/Plugin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index b651079c2..47648e2fb 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -9,6 +9,7 @@ import re import sys import traceback import urllib +import urlparse if os.name != "nt": import grp @@ -172,7 +173,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.35" + __version__ = "0.36" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From fa5e570bfa4439812b4329f43d6a49e0b5239641 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 22 Sep 2015 03:20:25 +0200 Subject: [Plugin] Fix _log + code cosmetics --- module/plugins/internal/Hoster.py | 13 +++++++------ module/plugins/internal/Plugin.py | 8 ++++---- module/plugins/internal/SimpleHoster.py | 4 ++-- module/plugins/internal/XFSHoster.py | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index a77ef05dd..bdb5c4746 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -106,7 +106,7 @@ class Hoster(Plugin): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) - msg = " | ".join(decode(a).strip() for a in messages if a) + msg = u" | ".join(decode(a).strip() for a in messages if a) log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, @@ -207,12 +207,13 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): - reconnect = bool(reconnect) + if reconnect: + self.log_info(_("Requesting line reconnection...") + else: + self.log_debug("Reconnect: %s" % reconnect) - self.log_info(_("RECONNECT ") + ("enabled" if reconnect else "disabled")) self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) - - self.wantReconnect = reconnect + self.wantReconnect = bool(reconnect) def set_wait(self, seconds, reconnect=None): @@ -225,7 +226,7 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_info(_("WAIT %d seconds") % wait_time) + self.log_info(_("Waiting %d seconds...") % wait_time) self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 47648e2fb..f6f7258fa 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -90,11 +90,11 @@ def which(program): return exe_file -def seconds_to_midnight(gmt=None): - if gmt is None: +def seconds_to_midnight(utc=None): + if utc is None: now = datetime.datetime.today() else: - now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) + now = datetime.datetime.utcnow() + datetime.timedelta(hours=utc) midnight = now.replace(hour=0, minute=10, second=0, microsecond=0) + datetime.timedelta(days=1) @@ -212,7 +212,7 @@ class Plugin(object): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) - msg = " | ".join(encode(a).strip() for a in messages if a) + msg = u" | ".join(decode(a).strip() for a in messages if a) log("%(plugintype)s %(pluginname)s: %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index fb55ef762..365fad35e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -368,7 +368,7 @@ class SimpleHoster(Hoster): self.log_warning(self.info['error']) if re.search('da(il)?y|today', errmsg, re.I): - wait_time = seconds_to_midnight(gmt=2) + wait_time = seconds_to_midnight() else: wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) @@ -393,7 +393,7 @@ class SimpleHoster(Hoster): if re.search('limit|wait|slot', errmsg, re.I): if re.search("da(il)?y|today", errmsg): - wait_time = seconds_to_midnight(gmt=2) + wait_time = seconds_to_midnight() else: wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 145efcd8e..d83e3cb78 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -163,7 +163,7 @@ class XFSHoster(SimpleHoster): self.retry(20, 3 * 60, _("Can not leech file")) elif 'today' in stmsg: - self.retry(delay=seconds_to_midnight(gmt=2), msg=_("You've used all Leech traffic today")) + self.retry(delay=seconds_to_midnight(), msg=_("You've used all Leech traffic today")) else: self.fail(stmsg) -- cgit v1.2.3 From 32bfe05611f9cdeb11f8d13d4e57b68c96ad2b58 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 22 Sep 2015 15:45:39 +0300 Subject: version up --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index bdb5c4746..e655b0544 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.26" + __version__ = "0.27" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 96ce21ce760a6d4601d904446a639b1bef20bc6e Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 22 Sep 2015 15:47:34 +0300 Subject: version up --- module/plugins/internal/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index f6f7258fa..51192d8c9 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -173,7 +173,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.36" + __version__ = "0.37" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From a00bfaa5ea7c319ddfe2797d0c6629af2250c761 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 22 Sep 2015 15:48:29 +0300 Subject: version up --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 365fad35e..0f030e000 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -21,7 +21,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.85" + __version__ = "1.86" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 30c6746dda1709aa7dda57f6dd977319a276b2cc Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 22 Sep 2015 15:49:38 +0300 Subject: version up --- module/plugins/internal/XFSHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index d83e3cb78..729c9a0ee 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.59" + __version__ = "0.60" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 2c61d10fdeeb89199d14ba152490f227f201d67c Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 22 Sep 2015 18:12:57 +0300 Subject: Update Hoster.py --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index e655b0544..90961d529 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -575,7 +575,7 @@ class Hoster(Plugin): except Exception: #: Bad bad bad... rewrite this part in 0.4.10 res = self.load(url, just_header=True, - req=self.pyload.requestFactory.getRequest()) + req=self.pyload.requestFactory.getRequest(self.__name__)) header = {'code': req.code} for line in res.splitlines(): -- cgit v1.2.3 From 3da545b9be94d5da8d247102538e062ea83b0a56 Mon Sep 17 00:00:00 2001 From: "Marcus \"Chaosblog\"" Date: Wed, 23 Sep 2015 12:53:14 +0200 Subject: Add missing closing bracket --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 90961d529..f610023f4 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -208,7 +208,7 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): if reconnect: - self.log_info(_("Requesting line reconnection...") + self.log_info(_("Requesting line reconnection...")) else: self.log_debug("Reconnect: %s" % reconnect) -- cgit v1.2.3 From 625be1490bee7e8dd44af720eb64864799c06487 Mon Sep 17 00:00:00 2001 From: Stefan Reisich Date: Wed, 23 Sep 2015 18:37:27 +0200 Subject: [FIX] AttributeError: x object has no attribute 'user' --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index f610023f4..b853f2815 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -192,10 +192,10 @@ class Hoster(Plugin): self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) if self.account: - if not self.account.user: #@TODO: Move to `Account` in 0.4.10 + if not hasattr(self.account, 'user'): #@TODO: Move to `Account` in 0.4.10 self.account.user = self.account.select()[0] - if not self.account.logged: + if not hasattr(self.account, 'logged'): self.account = False -- cgit v1.2.3 From 8dd484b68a0be6376d0f38a5abc7bcf0107aa349 Mon Sep 17 00:00:00 2001 From: "Marcus \"Chaosblog\"" Date: Wed, 23 Sep 2015 22:53:40 +0200 Subject: Version up Increase version number --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index b853f2815..5d0a64f1a 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.27" + __version__ = "0.28" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 260d307c7c6c3149607a8f6dba143dea9a6b609c Mon Sep 17 00:00:00 2001 From: Jochen Oberreiter Date: Sat, 26 Sep 2015 18:31:31 +0200 Subject: Add captcha pattern for Recaptcha and Solvemedia Fix for #1852, #1865 --- module/plugins/internal/XFSHoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 729c9a0ee..105e8f4ba 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.60" + __version__ = "0.61" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -45,8 +45,8 @@ class XFSHoster(SimpleHoster): CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' CAPTCHA_BLOCK_PATTERN = r'>Enter code.*?(.+?)
' - RECAPTCHA_PATTERN = None - SOLVEMEDIA_PATTERN = None + RECAPTCHA_PATTERN = r'api\.recaptcha\.net/challenge\?k=(.+?)["\']' + SOLVEMEDIA_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']' FORM_PATTERN = None FORM_INPUTS_MAP = None #: Dict passed as input_names to parse_html_form -- cgit v1.2.3 From 96d7371a205c6dfd717f0a6331f741485d5a30fe Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Mon, 28 Sep 2015 03:51:56 +0300 Subject: [Captcha] generic fix for #1756 --- module/plugins/internal/Captcha.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index d2be21a58..93642b32a 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -11,7 +11,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.44" + __version__ = "0.45" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -49,9 +49,9 @@ class Captcha(Plugin): pass - def decrypt(self, url, get={}, post={}, ref=False, cookies=True, decode=False, + def decrypt(self, url, get={}, post={}, ref=False, cookies=True, decode=False, req=None, input_type='jpg', output_type='textual', ocr=True, timeout=120): - img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode) + img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode, req=req or self.plugin.req) return self._decrypt(img, input_type, output_type, ocr, timeout) -- cgit v1.2.3 From ec6e5dc7fcdcefee10e37d22473c9accae1104cf Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 29 Sep 2015 20:05:07 +0200 Subject: Account class completely rewritten + plugins updated --- module/plugins/internal/Account.py | 458 +++++++++++++++++----------------- module/plugins/internal/XFSAccount.py | 43 ++-- 2 files changed, 257 insertions(+), 244 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index de338cd33..5c76d4ccb 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- -import copy import random import time import threading -import traceback +from module.plugins.Plugin import SkipDownload as Skip from module.plugins.internal.Plugin import Plugin from module.utils import compare_time, lock, parseFileSize as parse_size @@ -13,7 +12,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.18" + __version__ = "0.50" __status__ = "testing" __description__ = """Base account plugin""" @@ -21,18 +20,23 @@ class Account(Plugin): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - LOGIN_TIMEOUT = 10 * 60 #: After that time (in minutes) pyload will relogin the account - INFO_THRESHOLD = 30 * 60 #: After that time (in minutes) account data will be reloaded + LOGIN_TIMEOUT = 10 * 60 #: Relogin accounts every 10 minutes + AUTO_TIMEOUT = True #: Automatically adjust relogin interval def __init__(self, manager, accounts): self._init(manager.core) - self.lock = threading.RLock() - self.accounts = accounts #@TODO: Remove in 0.4.10 + self.manager = manager + self.lock = threading.RLock() + + self.accounts = accounts #@TODO: Recheck in 0.4.10 + self.user = None + + self.interval = self.LOGIN_TIMEOUT + self.auto_timeout = self.interval if self.AUTO_TIMEOUT else False self.init() - self.init_accounts(accounts) def init(self): @@ -42,218 +46,172 @@ class Account(Plugin): pass - def login(self, user, password, data, req): + @property + def logged(self): """ - Login into account, the cookies will be saved so user can be recognized + Checks if user is still logged in """ - pass - - - @lock - def _login(self, user): - try: - info = self.info[user] - info['login']['timestamp'] = time.time() #: Set timestamp for login - - self.req = self.get_request(user) - self.login(user, info['login']['password'], info['data'], self.req) - - except Exception, e: - self.log_warning(_("Could not login user `%s`") % user, e) - res = info['login']['valid'] = False - self.accounts[user]['valid'] = False #@TODO: Remove in 0.4.10 + if not self.user: + return False - if self.pyload.debug: - traceback.print_exc() + self.sync() + if self.info['login']['timestamp'] + self.interval < time.time(): + self.log_debug("Reached login timeout for user `%s`" % self.user) + return False else: - res = info['login']['valid'] = True - self.accounts[user]['valid'] = True #@TODO: Remove in 0.4.10 - - finally: - self.clean() - return res + return True - def relogin(self, user): - self.log_info(_("Relogin user `%s`...") % user) + @property + def premium(self): + return bool(self.get_data('premium')) - req = self.get_request(user) - if req: - req.clearCookies() - self.clean() - return self._login(user) + def signin(self, user, password, data): + """ + Login into account, the cookies will be saved so user can be recognized + """ + pass - #@TODO: Rewrite in 0.4.10 - def init_accounts(self, accounts): - for user, data in accounts.items(): - self.add(user, data['password'], data['options']) + def login(self): + if not self.req: + self.log_info(_("Login user `%s`...") % self.user) + else: + self.log_info(_("Relogin user `%s`...") % self.user) + self.clean() + self.req = self.pyload.requestFactory.getRequest(self.__name__, self.user) - @lock - def add(self, user, password=None, options={}): - if user not in self.info: - self.info[user] = {'login': {'valid' : None, - 'password' : password or "", - 'timestamp': 0}, - 'data' : {'options' : options, - 'premium' : None, - 'validuntil' : None, - 'trafficleft': None, - 'maxtraffic' : None}} - - #@TODO: Remove in 0.4.10 - self.accounts[user] = self.info[user]['data'] - self.accounts[user].update({'login' : user, - 'type' : self.__name__, - 'valid' : self.info[user]['login']['valid'], - 'password': self.info[user]['login']['password']}) - - self.log_info(_("Login user `%s`...") % user) - self._login(user) - return True + self.sync() - else: - self.log_error(_("Error adding user `%s`") % user, _("User already exists")) + try: + self.info['login']['timestamp'] = time.time() #: Set timestamp for login + self.signin(self.user, self.info['login']['password'], self.info['data']) + except Skip: + self.info['login']['valid'] = True + if self.auto_timeout: + self.auto_timeout *= 2 + self.interval = self.auto_timeout - @lock - def update(self, user, password=None, options={}): - """ - Updates account and return true if anything changed - """ - if not (password or options): - return - - if user not in self.info: - return self.add(user, password, options) + except Exception, e: + self.log_error(_("Could not login user `%s`") % user, e) + self.info['login']['valid'] = False else: - if password: - self.info[user]['login']['password'] = password - self.accounts[user]['password'] = password #@TODO: Remove in 0.4.10 - self.relogin(user) + self.info['login']['valid'] = True + if self.interval is self.auto_timeout: + self.interval = self.auto_timeout / 2 + self.auto_timeout = False - if options: - before = self.info[user]['data']['options'] - self.info[user]['data']['options'].update(options) - return self.info[user]['data']['options'] != before + finally: + self.syncback() + return bool(self.info['login']['valid']) - return True + #@TODO: Recheck in 0.4.10 + def syncback(self): + return self.sync(reverse=True) - #: Deprecated method, use `update` instead (Remove in 0.4.10) - def updateAccounts(self, *args, **kwargs): - return self.update(*args, **kwargs) + #@TODO: Recheck in 0.4.10 + def sync(self, reverse=False): + if not self.user: + return - def remove(self, user=None): # -> def remove - if not user: - self.info.clear() - self.accounts.clear() #@TODO: Remove in 0.4.10 + u = self.accounts[self.user] - elif user in self.info: - self.info.pop(user, None) - self.accounts.pop(user, None) #@TODO: Remove in 0.4.10 + if reverse: + u.update(self.info['data']) + u.update(self.info['login']) + else: + d = {'login': {'password' : u['password'], + 'timestamp': u['timestamp'], + 'valid' : u['valid']}, + 'data' : {'maxtraffic' : u['maxtraffic'], + 'options' : u['options'], + 'premium' : u['premium'], + 'trafficleft': u['trafficleft'], + 'validuntil' : u['validuntil']}} - #: Deprecated method, use `remove` instead (Remove in 0.4.10) - def removeAccount(self, *args, **kwargs): - return self.remove(*args, **kwargs) + self.info.update(d) - #@NOTE: Remove in 0.4.10? - def get_data(self, user, reload=False): - if not user: - return + def relogin(self): + return self.login() - info = self.get_info(user, reload) - if info and 'data' in info: - return info['data'] + def reset(self): + self.sync() - #: Deprecated method, use `get_data` instead (Remove in 0.4.10) - def getAccountData(self, *args, **kwargs): - if 'force' in kwargs: - kwargs['reload'] = kwargs['force'] - kwargs.pop('force', None) + d = {'maxtraffic' : None, + 'options' : {'limitdl': ['0']}, + 'premium' : None, + 'trafficleft': None, + 'validuntil' : None} - data = self.get_data(*args, **kwargs) or {} - if 'options' not in data: - data['options'] = {'limitdl': ['0']} + self.info['data'].update(d) - return data + self.syncback() - def get_info(self, user, reload=False): + def get_info(self, refresh=True): """ - Retrieve account infos for an user, do **not** overwrite this method!\\ + Retrieve account infos for an user, do **not** overwrite this method! just use it to retrieve infos in hoster plugins. see `grab_info` :param user: username - :param reload: reloads cached account information + :param relogin: reloads cached account information :return: dictionary with information """ - if user not in self.info: - self.log_error(_("User `%s` not found while retrieving account info") % user) - return - - elif reload: - self.log_info(_("Parsing account info for user `%s`...") % user) - info = self._parse_info(user) - - safe_info = copy.deepcopy(info) - safe_info['login']['password'] = "**********" - safe_info['data']['password'] = "**********" #@TODO: Remove in 0.4.10 - self.log_debug("Account info for user `%s`: %s" % (user, safe_info)) + if not self.logged: + if self.relogin(): + refresh = True + else: + refresh = False + self.reset() - elif self.INFO_THRESHOLD > 0 and self.info[user]['login']['timestamp'] + self.INFO_THRESHOLD < time.time(): - self.log_debug("Reached data timeout for %s" % user) - info = self.get_info(user, True) + if refresh: + self.log_info(_("Grabbing account info for user `%s`...") % self.user) + self.info = self._grab_info() - else: - info = self.info[user] + self.syncback() - return info + safe_info = dict(self.info) + safe_info['login']['password'] = "**********" + self.log_debug("Account info for user `%s`: %s" % (self.user, safe_info)) + return self.info - def is_premium(self, user): - if not user: - return False - info = self.get_info(user) - return info['data']['premium'] + def get_login(self, key=None): + d = self.get_info()['login'] + return d.get(key) or d - def _parse_info(self, user): - info = self.info[user] + def get_data(self, key=None): + d = self.get_info()['data'] + return d.get(key) or d - if not info['login']['valid']: - return info + def _grab_info(self): try: - self.req = self.get_request(user) - extra_info = self.grab_info(user, info['login']['password'], info, self.req) + data = self.grab_info(self.user, self.info['login']['password'], self.info['data']) - if extra_info and isinstance(extra_info, dict): - info['data'].update(extra_info) + if data and isinstance(data, dict): + self.info['data'].update(data) - except (Fail, Exception), e: - self.log_warning(_("Error loading info for user `%s`") % user, e) - - if self.pyload.debug: - traceback.print_exc() + except Exception, e: + self.log_warning(_("Error loading info for user `%s`") % self.user, e) finally: - self.clean() - - self.info[user].update(info) - return info + return self.info - def grab_info(self, user, password, info, req): + def grab_info(self, user, password, data): """ This should be overwritten in account plugin and retrieving account information for user @@ -265,43 +223,104 @@ class Account(Plugin): pass - #: Remove in 0.4.10 - def getAllAccounts(self, *args, **kwargs): - return [self.getAccountData(user, *args, **kwargs) for user, info in self.info.items()] + ########################################################################### + #@TODO: Recheck and move to `AccountManager` in 0.4.10 #################### + ########################################################################### + @lock + def init_accounts(self): + accounts = dict(self.accounts) + self.accounts.clear() - def fail_login(self, msg=_("Login handshake has failed")): - return self.fail(msg) + for user, info in accounts.items(): + self.add(user, info['password'], info['options']) - def get_request(self, user=None): - if not user: - user, info = self.select() + @lock + def getAccountData(self, user, force=False): + self.accounts[user]['plugin'].get_info() + return self.accounts[user] - return self.pyload.requestFactory.getRequest(self.__name__, user) + @lock + def getAllAccounts(self, force=False): + if force: + self.init_accounts() #@TODO: Recheck in 0.4.10 - def get_cookies(self, user=None): - if not user: - user, info = self.select() + return [self.getAccountData(user, force) for user in self.accounts] - return self.pyload.requestFactory.getCookieJar(self.__name__, user) + #@TODO: Remove in 0.4.10 + @lock + def scheduleRefresh(self, user, force=False): + pass - def select(self): + + @lock + def add(self, user, password=None, options={}): + self.log_info(_("Adding user `%s`...") % user) + + if user in self.accounts: + self.log_error(_("Error adding user `%s`") % user, _("User already exists")) + return False + + d = {'login' : user, + 'maxtraffic' : None, + 'options' : options or {'limitdl': ['0']}, + 'password' : password or "", + 'plugin' : self.__class__(self.manager, self.accounts), + 'premium' : None, + 'timestamp' : 0, + 'trafficleft': None, + 'type' : self.__name__, + 'valid' : None, + 'validuntil' : None} + + u = self.accounts[user] = d + return u['plugin'].choose(user) + + + @lock + def updateAccounts(self, user, password=None, options={}): """ - Returns a valid account name and info + Updates account and return true if anything changed """ + if user in self.accounts: + self.log_info(_("Updating account info for user `%s`...") % user) + + u = self.accounts[user] + if password: + u['password'] = password + + if options: + u['options'].update(options) + + u['plugin'].relogin() + + else: + self.add(user, password, options) + + + @lock + def removeAccount(self, user): + self.accounts.pop(user, None) + if user is self.user: + self.choose() + + + @lock + def select(self): free_accounts = {} premium_accounts = {} - for user, info in self.info.items(): + for user in self.accounts: + info = self.accounts[user]['plugin'].get_info() + data = info['data'] + if not info['login']['valid']: continue - data = info['data'] - - if "time" in data['options'] and data['options']['time']: + if data['options'].get('time'): time_data = "" try: time_data = data['options']['time'][0] @@ -311,7 +330,8 @@ class Account(Plugin): continue except Exception: - self.log_warning(_("Wrong time format `%s` for account `%s`, use 1:22-3:44") % (user, time_data)) + self.log_warning(_("Wrong time format `%s` for account `%s`, use 1:22-3:44") + % (user, time_data)) if data['trafficleft'] == 0: continue @@ -330,68 +350,56 @@ class Account(Plugin): if not account_list: return None, None - validuntil_list = [(user, info) for user, info in account_list if info['data']['validuntil']] + validuntil_list = [(user, info) for user, info in account_list \ + if info['data']['validuntil']] if not validuntil_list: - return random.choice(account_list) #@TODO: Random account?! Recheck in 0.4.10 + return random.choice(account_list) #@TODO: Random account?! Rewrite in 0.4.10 return sorted(validuntil_list, key=lambda a: a[1]['data']['validuntil'], reverse=True)[0] - def parse_traffic(self, value, unit=None): #: Return kilobytes - if not unit and not isinstance(value, basestring): - unit = "KB" - - return parse_size(value, unit) - + @lock + def choose(self, user=None): + """ + Choose a valid account + """ + if not user: + user = self.select()[0] - def empty(self, user): - if user not in self.info: - return + elif user not in self.accounts: + self.log_error(_("Error choosing user `%s`") % user, _("User not exists")) + return False - self.log_warning(_("Account `%s` has not enough traffic") % user, _("Checking again in 30 minutes")) + if user is self.user: + return True - self.info[user]['data']['trafficleft'] = 0 - self.schedule_refresh(user, 30 * 60) + self.user = user + self.info.clear() + self.clean() + if self.user is not None: + self.login() + return True - def expired(self, user): - if user not in self.info: - return + else: + return False - self.log_warning(_("Account `%s` is expired") % user, _("Checking again in 60 minutes")) - self.info[user]['data']['validuntil'] = time.time() - 1 - self.schedule_refresh(user, 60 * 60) + ########################################################################### + def parse_traffic(self, value, unit=None): #@NOTE: Returns kilobytes in 0.4.9 + if not isinstance(unit, basestring): + unit = "KB" - def schedule_refresh(self, user, time=0): - """ - Add task to refresh account info to sheduler - """ - self.log_debug("Scheduled refresh for user `%s` in %s seconds" % (user, time)) - self.pyload.scheduler.addJob(time, self.get_info, [user, True]) + return parse_size(value, unit) / 1024 #@TODO: Remove `/ 1024` in 0.4.10 - #: Deprecated method, use `schedule_refresh` instead (Remove in 0.4.10) - def scheduleRefresh(self, *args, **kwargs): - if 'force' in kwargs: - kwargs.pop('force', None) #@TODO: Recheck in 0.4.10 - return self.schedule_refresh(*args, **kwargs) + def fail_login(self, msg=_("Login handshake has failed")): + return self.fail(msg) - @lock - def is_logged(self, user, relogin=False): - """ - Checks if user is still logged in - """ - if user in self.info: - if self.LOGIN_TIMEOUT > 0 and self.info[user]['login']['timestamp'] + self.LOGIN_TIMEOUT < time.time(): - self.log_debug("Reached login timeout for %s" % user) - return self.relogin(user) if relogin else False - else: - return True - else: - return False + def skip_login(self, msg=_("Already signed in")): + return self.skip(msg) diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index bb5cbcf50..91136a2a0 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -6,13 +6,14 @@ import urlparse from module.common.json_layer import json_loads from module.plugins.internal.Account import Account +# from module.plugins.internal.MultiAccount import MultiAccount from module.plugins.internal.Plugin import parse_html_form, set_cookie class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.46" + __version__ = "0.48" __status__ = "testing" __description__ = """XFileSharing account plugin""" @@ -21,8 +22,8 @@ class XFSAccount(Account): ("Walter Purcaro", "vuolter@gmail.com" )] - HOSTER_DOMAIN = None - HOSTER_URL = None + PLUGIN_DOMAIN = None + PLUGIN_URL = None LOGIN_URL = None COOKIES = True @@ -38,21 +39,22 @@ class XFSAccount(Account): LEECH_TRAFFIC_UNIT = "MB" #: Used only if no group was found LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<' + LOGIN_SKIP_PATTERN = r'op=logout' - def grab_info(self, user, password, data, req): + def grab_info(self, user, password, data): validuntil = None trafficleft = None leechtraffic = None premium = None - if not self.HOSTER_URL: #@TODO: Remove in 0.4.10 + if not self.PLUGIN_URL: #@TODO: Remove in 0.4.10 return {'validuntil' : validuntil, 'trafficleft' : trafficleft, 'leechtraffic': leechtraffic, 'premium' : premium} - html = self.load(self.HOSTER_URL, + html = self.load(self.PLUGIN_URL, get={'op': "my_account"}, cookies=self.COOKIES) @@ -139,31 +141,34 @@ class XFSAccount(Account): 'premium' : premium} - def login(self, user, password, data, req): - if self.HOSTER_DOMAIN: - if not self.HOSTER_URL: - self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN + def signin(self, user, password, data): + if self.PLUGIN_DOMAIN: + if not self.PLUGIN_URL: + self.PLUGIN_URL = "http://www.%s/" % self.PLUGIN_DOMAIN if self.COOKIES: - if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: + self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) else: - set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") + set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") - if not self.HOSTER_URL: - self.fail_login(_("Missing HOSTER_URL")) + if not self.PLUGIN_URL: + self.fail_login(_("Missing PLUGIN_URL")) else: - self.HOSTER_URL = self.HOSTER_URL.rstrip('/') + "/" + self.PLUGIN_URL = self.PLUGIN_URL.rstrip('/') + "/" if not self.LOGIN_URL: - self.LOGIN_URL = urlparse.urljoin(self.HOSTER_URL, "login.html") + self.LOGIN_URL = urlparse.urljoin(self.PLUGIN_URL, "login.html") html = self.load(self.LOGIN_URL, cookies=self.COOKIES) + if re.search(self.LOGIN_SKIP_PATTERN, html): + self.skip_login() + action, inputs = parse_html_form('name="FL"', html) if not inputs: inputs = {'op' : "login", - 'redirect': self.HOSTER_URL} + 'redirect': self.PLUGIN_URL} inputs.update({'login' : user, 'password': password}) @@ -171,7 +176,7 @@ class XFSAccount(Account): if action: url = urlparse.urljoin("http://", action) else: - url = self.HOSTER_URL + url = self.PLUGIN_URL html = self.load(url, post=inputs, cookies=self.COOKIES) -- cgit v1.2.3 From 8af39bfa2c409d4fd07d430485117bf60fdcc935 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 29 Sep 2015 21:43:22 +0200 Subject: Update internal plugins --- module/plugins/internal/Addon.py | 1 - module/plugins/internal/Container.py | 1 - module/plugins/internal/Crypter.py | 6 +- module/plugins/internal/Extractor.py | 3 +- module/plugins/internal/Hook.py | 2 - module/plugins/internal/Hoster.py | 244 ++++++++++++++------------ module/plugins/internal/MultiAccount.py | 287 +++++++++++++++++++++++++++++++ module/plugins/internal/MultiCrypter.py | 4 +- module/plugins/internal/MultiHook.py | 260 ---------------------------- module/plugins/internal/MultiHoster.py | 14 +- module/plugins/internal/Plugin.py | 60 ++++--- module/plugins/internal/SevenZip.py | 6 +- module/plugins/internal/SimpleCrypter.py | 6 +- module/plugins/internal/SimpleHoster.py | 51 +++--- module/plugins/internal/UnRar.py | 6 +- module/plugins/internal/UnZip.py | 1 - module/plugins/internal/XFSCrypter.py | 18 +- module/plugins/internal/XFSHoster.py | 26 +-- 18 files changed, 523 insertions(+), 473 deletions(-) create mode 100644 module/plugins/internal/MultiAccount.py delete mode 100644 module/plugins/internal/MultiHook.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 5150e88f6..3a252fdfb 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -26,7 +26,6 @@ class Addon(Plugin): __version__ = "0.06" __status__ = "testing" - __config__ = [] #: [("name", "type", "desc", "default")] __threaded__ = [] #@TODO: Remove in 0.4.10 __description__ = """Base addon plugin""" diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 430590421..524b55839 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -17,7 +17,6 @@ class Container(Crypter): __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [] #: [("name", "type", "desc", "default")] __description__ = """Base container decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 2033b67df..ad5bcc74e 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -7,7 +7,7 @@ from module.utils import save_path as safe_filename class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.08" + __version__ = "0.09" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -83,7 +83,7 @@ class Crypter(Hoster): self.pyload.api.setPackageData(pid, {'password': package_password}) #: Workaround to do not break API addPackage method - set_folder = lambda x: self.pyload.api.setPackageData(pid, {'folder': safe_filename(x) or ""}) + set_folder = lambda x="": self.pyload.api.setPackageData(pid, {'folder': safe_filename(x)}) if use_subfolder: if not subfolder_per_package: @@ -98,4 +98,4 @@ class Crypter(Hoster): self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) elif folder_per_package: - set_folder(None) + set_folder() diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index f21fe473c..3ab5d6a0d 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -5,6 +5,7 @@ import re from module.PyFile import PyFile from module.plugins.internal.Plugin import Plugin +from module.utils import fs_encode class ArchiveError(Exception): @@ -22,7 +23,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.34" + __version__ = "0.35" __status__ = "testing" __description__ = """Base extractor plugin""" diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 1f566f824..8ae731a7f 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -9,8 +9,6 @@ class Hook(Addon): __version__ = "0.13" __status__ = "testing" - __config__ = [] #: [("name", "type", "desc", "default")] - __description__ = """Base hook plugin""" __license__ = "GPLv3" __authors__ = [("mkaay" , "mkaay@mkaay.de" ), diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 5d0a64f1a..bc340e78f 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -11,8 +11,8 @@ import urlparse from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, decode, encode, exists, parse_html_form, - parse_html_tag_attr_value, parse_name, + chunks, decode, encode, exists, fixurl, + parse_html_form, parse_html_tag_attr_value, parse_name, replace_patterns, seconds_to_midnight, set_cookie, set_cookies, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename @@ -41,21 +41,27 @@ def create_getInfo(klass): return get_info +#@NOTE: `check_abort` decorator +def check_abort(fn): + + def wrapper(self, *args, **kwargs): + self.check_abort() + return fn(self, *args, **kwargs) + + return wrapper + + class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.28" + __version__ = "0.31" __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [] #: [("name", "type", "desc", "default")] __description__ = """Base hoster plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), - ("spoob" , "spoob@pyload.org" ), - ("mkaay" , "mkaay@mkaay.de" ), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] def __init__(self, pyfile): @@ -74,7 +80,7 @@ class Hoster(Plugin): #: Account handler instance, see :py:class:`Account` self.account = None - self.req = None #: Browser instance, see `network.Browser` + self.user = None #@TODO: Remove in 0.4.10 #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile @@ -98,7 +104,7 @@ class Hoster(Plugin): #: Dict of the amount of retries already made self.retries = {} - self.retry_free = False #@TODO: Recheck in 0.4.10 + self.force_free = False #@TODO: Recheck in 0.4.10 self._setup() self.init() @@ -116,9 +122,10 @@ class Hoster(Plugin): @classmethod def get_info(cls, url="", html=""): + url = fixurl(url, unquote=True) return {'name' : parse_name(url), 'size' : 0, - 'status': 3 if url.strip() else 8, + 'status': 3 if url else 8, 'url' : url} @@ -142,6 +149,11 @@ class Hoster(Plugin): self.last_download = "" self.pyfile.error = "" + try: + self.req.close() + except Exception: + pass + if self.account: self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) self.chunk_limit = -1 #: -1 for unlimited @@ -162,16 +174,16 @@ class Hoster(Plugin): """ self.thread = thread - if self.retry_free: + if self.force_free: self.account = False else: self.load_account() #@TODO: Move to PluginThread in 0.4.10 - self.retry_free = False + self.force_free = False self._setup() + # self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 self.pyfile.setStatus("starting") - self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 self.check_abort() @@ -185,17 +197,17 @@ class Hoster(Plugin): def load_account(self): - if self.req: - self.req.close() - if not self.account: self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - if self.account: - if not hasattr(self.account, 'user'): #@TODO: Move to `Account` in 0.4.10 - self.account.user = self.account.select()[0] + if not self.account: + self.account = False + self.user = None #@TODO: Remove in 0.4.10 - if not hasattr(self.account, 'logged'): + else: + self.account.choose() + self.user = self.account.user #@TODO: Remove in 0.4.10 + if self.account.user is None: self.account = False @@ -207,12 +219,8 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): - if reconnect: - self.log_info(_("Requesting line reconnection...")) - else: - self.log_debug("Reconnect: %s" % reconnect) - - self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) + self.log_debug("RECONNECT %s required" % ("" if reconnect else "not"), + "Previous wantReconnect: %s" % self.wantReconnect) self.wantReconnect = bool(reconnect) @@ -226,8 +234,8 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_info(_("Waiting %d seconds...") % wait_time) - self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) + self.log_debug("WAIT set to %d seconds" % wait_time, + "Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until @@ -249,13 +257,17 @@ class Hoster(Plugin): self.waiting = True - status = pyfile.status #@NOTE: Remove in 0.4.10 + status = pyfile.status #@NOTE: Recheck in 0.4.10 pyfile.setStatus("waiting") - if not self.wantReconnect or self.account: + self.log_info(_("Waiting %d seconds...") % pyfile.waitUntil - time.time()) + + if self.wantReconnect: + self.log_info(_("Requiring reconnection...")) if self.account: self.log_warning("Ignore reconnection due logged account") + if not self.wantReconnect or self.account: while pyfile.waitUntil > time.time(): self.check_abort() time.sleep(2) @@ -263,24 +275,24 @@ class Hoster(Plugin): else: while pyfile.waitUntil > time.time(): self.check_abort() + self.thread.m.reconnecting.wait(1) if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False raise Reconnect - self.thread.m.reconnecting.wait(2) time.sleep(2) self.waiting = False - pyfile.status = status #@NOTE: Remove in 0.4.10 + pyfile.status = status #@NOTE: Recheck in 0.4.10 def skip(self, msg=""): """ Skip and give msg """ - raise Skip(encode(msg or self.pyfile.error)) #@TODO: Remove `encode` in 0.4.10 + raise Skip(encode(msg or self.pyfile.error or self.pyfile.pluginname)) #@TODO: Remove `encode` in 0.4.10 #@TODO: Remove in 0.4.10 @@ -293,7 +305,7 @@ class Hoster(Plugin): if msg: self.pyfile.error = msg else: - msg = self.pyfile.error + msg = self.pyfile.error or (self.info['error'] if 'error' in self.info else self.pyfile.getStatusName()) raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 @@ -359,30 +371,33 @@ class Hoster(Plugin): if nopremium: if self.premium: - self.retry_free = True + self.force_free = True else: self.fail("%s | %s" % (msg, _("Download was already free"))) raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def fixurl(self, url, baseurl=None): + def fixurl(self, url, baseurl=None, unquote=None): + url = fixurl(url) + if not baseurl: - baseurl = self.pyfile.url + baseurl = fixurl(self.pyfile.url) if not urlparse.urlparse(url).scheme: url_p = urlparse.urlparse(baseurl) baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) url = urlparse.urljoin(baseurl, url) - return url + return fixurl(url, unquote) + @check_abort def load(self, *args, **kwargs): - self.check_abort() return super(Hoster, self).load(*args, **kwargs) + @check_abort def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ Downloads the content at url to download folder @@ -396,16 +411,18 @@ class Hoster(Plugin): the filename will be changed if needed :return: The location where the file was saved """ - self.check_abort() - if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, - *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) + + url = self.fixurl(url, unquote=True) self.pyfile.name = parse_name(self.pyfile.name) #: Safe check self.captcha.correct() - self.check_for_same_files() + + if self.pyload.config.get("download", "skip_existing"): + self.check_filedupe() self.pyfile.setStatus("downloading") @@ -441,7 +458,9 @@ class Hoster(Plugin): if finalname != newname: try: - os.rename(fs_join(location, newname), fs_join(location, finalname)) + oldname_enc = fs_join(download_location, newname) + newname_enc = fs_join(download_location, finalname) + os.rename(oldname_enc, newname_enc) except OSError, e: self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e) @@ -463,16 +482,16 @@ class Hoster(Plugin): if not self.pyfile.abort: return - if self.pyfile.hasStatus("failed"): + if self.pyfile.status is 8: self.fail() - elif self.pyfile.hasStatus("skipped"): + elif self.pyfile.status is 4: self.skip(self.pyfile.statusname) - elif self.pyfile.hasStatus("offline"): + elif self.pyfile.status is 1: self.offline() - elif self.pyfile.hasStatus("temp. offline"): + elif self.pyfile.status is 6: self.temp_offline() else: @@ -489,7 +508,8 @@ class Hoster(Plugin): if not self.last_download: return - download_size = os.stat(fs_encode(self.last_download)).st_size + download_location = fs_encode(self.last_download) + download_size = os.stat(download_location).st_size if download_size < 1: self.fail(_("Empty file")) @@ -505,7 +525,7 @@ class Hoster(Plugin): self.log_warning(_("File size is not equal to expected size")) - def check_download(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): + def check_file(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): """ Checks the content of the last downloaded file, re match is saved to `last_check` @@ -552,7 +572,66 @@ class Hoster(Plugin): else: self.log_info(_("File deleted: ") + self.last_download) - self.last_download = "" + self.last_download = "" #: Recheck in 0.4.10 + + + def check_traffic(self): + if not self.account: + return True + + traffic = self.account.get_data(refresh=True)['trafficleft'] + + if traffic is None: + return False + + elif traffic is -1: + return True + + else: + size = self.pyfile.size / 1024 #@TODO: Remove in 0.4.10 + self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) #@TODO: Rewrite in 0.4.10 + return size <= traffic + + + def check_filedupe(self): + """ + Checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises Skip: + """ + pack = self.pyfile.package() + + for pyfile in self.pyload.files.cache.values(): + if pyfile is self.pyfile: + continue + + if pyfile.name != self.pyfile.name or pyfile.package().folder != pack.folder: + continue + + if pyfile.status in (0, 5, 7, 12): #: (finished, waiting, starting, downloading) + self.skip(pyfile.pluginname) + + download_folder = self.pyload.config.get("general", "download_folder") + package_folder = pack.folder if self.pyload.config.get("general", "folder_per_package") else "" + download_location = fs_join(download_folder, package_folder, self.pyfile.name) + + if not exists(download_location): + return + + pyfile = self.pyload.db.findDuplicates(self.pyfile.id, package_folder, self.pyfile.name) + if pyfile: + self.skip(pyfile[0]) + + size = os.stat(download_location).st_size + if size >= self.pyfile.size: + self.skip(_("File exists")) + + + #: Deprecated method, use `check_filedupe` instead (Remove in 0.4.10) + def checkForSameFiles(self, *args, **kwargs): + if self.pyload.config.get("download", "skip_existing"): + return self.check_filedupe() def direct_link(self, url, follow_location=None): @@ -598,10 +677,10 @@ class Hoster(Plugin): if 'content-disposition' in header: link = url - elif 'location' in header and header['location']: + elif header.get('location'): location = self.fixurl(header['location'], url) - if 'code' in header and header['code'] == 302: + if header.get('code') == 302: link = location if follow_location: @@ -611,7 +690,7 @@ class Hoster(Plugin): else: extension = os.path.splitext(parse_name(url))[-1] - if 'content-type' in header and header['content-type']: + if header.get('content-type'): mimetype = header['content-type'].split(';')[0].strip() elif extension: @@ -641,63 +720,8 @@ class Hoster(Plugin): return parse_html_form(attr_str, self.html, input_names) - def check_traffic_left(self): - if not self.account: - return True - - traffic = self.account.get_data(self.account.user, True)['trafficleft'] - - if traffic is None: - return False - - elif traffic == -1: - return True - - else: - size = self.pyfile.size / 1024 - self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) - return size <= traffic - - def get_password(self): """ Get the password the user provided in the package """ return self.pyfile.package().password or "" - - - #: Deprecated method, use `check_for_same_files` instead (Remove in 0.4.10) - def checkForSameFiles(self, *args, **kwargs): - return self.check_for_same_files(*args, **kwargs) - - - def check_for_same_files(self, starting=False): - """ - Checks if same file was/is downloaded within same package - - :param starting: indicates that the current download is going to start - :raises Skip: - """ - pack = self.pyfile.package() - - for pyfile in self.pyload.files.cache.values(): - if pyfile != self.pyfile and pyfile.name is self.pyfile.name and pyfile.package().folder is pack.folder: - if pyfile.status in (0, 12): #: Finished or downloading - self.skip(pyfile.pluginname) - elif pyfile.status in (5, 7) and starting: #: A download is waiting/starting and was appenrently started before - self.skip(pyfile.pluginname) - - download_folder = self.pyload.config.get("general", "download_folder") - location = fs_join(download_folder, pack.folder, self.pyfile.name) - - if starting and self.pyload.config.get("download", "skip_existing") and exists(location): - size = os.stat(location).st_size - if size >= self.pyfile.size: - self.skip("File exists") - - pyfile = self.pyload.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) - if pyfile: - if exists(location): - self.skip(pyfile[0]) - - self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name) diff --git a/module/plugins/internal/MultiAccount.py b/module/plugins/internal/MultiAccount.py new file mode 100644 index 000000000..b38670ce7 --- /dev/null +++ b/module/plugins/internal/MultiAccount.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- + +import re +import time + +from module.plugins.internal.Account import Account +from module.utils import decode, remove_chars + + +class MultiAccount(Account): + __name__ = "MultiAccount" + __type__ = "account" + __version__ = "0.02" + __status__ = "testing" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Multi hoster account plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team" , "admin@pyload.org" ), + ("Walter Purcaro", "vuolter@gmail.com")] + + + REFRESH_INTERVAL = 1 * 60 * 60 #: 1 hour + + DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"), + (r'bayfiles\.net' , "bayfiles.com" ), + (r'cloudnator\.com' , "shragle.com" ), + (r'dfiles\.eu' , "depositfiles.com" ), + (r'easy-share\.com' , "crocko.com" ), + (r'freakshare\.net' , "freakshare.com" ), + (r'hellshare\.com' , "hellshare.cz" ), + (r'ifile\.it' , "filecloud.io" ), + (r'nowdownload\.\w+', "nowdownload.sx" ), + (r'nowvideo\.\w+' , "nowvideo.sx" ), + (r'putlocker\.com' , "firedrive.com" ), + (r'share-?rapid\.cz', "multishare.cz" ), + (r'ul\.to' , "uploaded.to" ), + (r'uploaded\.net' , "uploaded.to" ), + (r'uploadhero\.co' , "uploadhero.com" ), + (r'zshares\.net' , "zshare.net" ), + (r'^1' , "one" ), + (r'^2' , "two" ), + (r'^3' , "three" ), + (r'^4' , "four" ), + (r'^5' , "five" ), + (r'^6' , "six" ), + (r'^7' , "seven" ), + (r'^8' , "eight" ), + (r'^9' , "nine" ), + (r'^0' , "zero" )] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + def init(self): + self.plugins = [] + self.supported = [] + self.new_supported = [] + + self.account = None + self.pluginclass = None + self.pluginmodule = None + self.pluginname = None + self.plugintype = None + + self.init_plugin() + + + def init_plugin(self): + self.pluginname = self.__name__.rsplit("Hook", 1)[0] + plugin, self.plugintype = self.pyload.pluginManager.findPlugin(self.pluginname) + + if plugin: + self.pluginmodule = self.pyload.pluginManager.loadModule(self.plugintype, self.pluginname) + self.pluginclass = getattr(self.pluginmodule, self.pluginname) + else: + self.log_warning(_("Hook plugin will be deactivated due missing plugin reference")) + self.set_config('activated', False) + + + def load_account(self): + self.account = self.pyload.accountManager.getAccountPlugin(self.pluginname) + + if self.account and not self.account.select()[0]: + self.account = False + + if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: + self.log_warning(_("Hook plugin will be deactivated due missing account reference")) + self.set_config('activated', False) + + + def activate(self): + self.init_periodical(threaded=True) + + + def plugins_cached(self): + if self.plugins: + return self.plugins + + for _i in xrange(5): + try: + pluginset = self._plugin_set(self.grab_hosters()) + break + + except Exception, e: + self.log_warning(e, _("Waiting 1 minute and retry")) + time.sleep(60) + else: + self.log_error(_("No hoster list retrieved")) + self.interval = self.REFRESH_INTERVAL + return list() + + try: + configmode = self.get_config('pluginmode', 'all') + if configmode in ("listed", "unlisted"): + pluginlist = self.get_config('pluginlist', '').replace('|', ',').replace(';', ',').split(',') + configset = self._plugin_set(pluginlist) + + if configmode == "listed": + pluginset &= configset + else: + pluginset -= configset + + except Exception, e: + self.log_error(e) + + self.plugins = list(pluginset) + + return self.plugins + + + def _plugin_set(self, plugins): + regexp = re.compile(r'^[\w\-.^_]{3,63}\.[a-zA-Z]{2,}$', re.U) + plugins = [decode(p.strip()).lower() for p in plugins if regexp.match(p.strip())] + + for r in self.DOMAIN_REPLACEMENTS: + rf, rt = r + repr = re.compile(rf, re.I|re.U) + plugins = [re.sub(rf, rt, p) if repr.match(p) else p for p in plugins] + + return set(plugins) + + + def grab_hosters(self, user, password, data): + """ + Load list of supported hoster + + :return: List of domain names + """ + raise NotImplementedError + + + def periodical(self): + """ + Reload plugin list periodically + """ + self.load_account() + + if self.get_config('reload', True): + self.interval = max(self.get_config('reloadinterval', 12) * 60 * 60, self.REFRESH_INTERVAL) + else: + self.pyload.scheduler.removeJob(self.cb) + self.cb = None + + self.log_info(_("Reloading supported %s list") % self.plugintype) + + old_supported = self.supported + + self.supported = [] + self.new_supported = [] + self.plugins = [] + + self.override_plugins() + + old_supported = [plugin for plugin in old_supported if plugin not in self.supported] + + if old_supported: + self.log_debug("Unload: %s" % ", ".join(old_supported)) + for plugin in old_supported: + self.unload_plugin(plugin) + + + def override_plugins(self): + excludedList = [] + + if self.plugintype == "hoster": + pluginMap = dict((name.lower(), name) for name in self.pyload.pluginManager.hosterPlugins.keys()) + accountList = [account.type.lower() for account in self.pyload.api.getAccounts(False) if account.valid and account.premium] + else: + pluginMap = {} + accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.pyload.pluginManager.crypterPlugins.keys()] + + for plugin in self.plugins_cached(): + name = remove_chars(plugin, "-.") + + if name in accountList: + excludedList.append(plugin) + else: + if name in pluginMap: + self.supported.append(pluginMap[name]) + else: + self.new_supported.append(plugin) + + if not self.supported and not self.new_supported: + self.log_error(_("No %s loaded") % self.plugintype) + return + + #: Inject plugin plugin + self.log_debug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) + + for plugin in self.supported: + hdict = self.pyload.pluginManager.plugins[self.plugintype][plugin] + hdict['new_module'] = self.pluginmodule + hdict['new_name'] = self.pluginname + + if excludedList: + self.log_info(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList)))) + + if self.new_supported: + plugins = sorted(self.new_supported) + + self.log_debug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) + + #: Create new regexp + regexp = r'.*(?P%s).*' % "|".join(x.replace('.', '\.') for x in plugins) + if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and "://" in self.pluginclass.__pattern__: + regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) + + self.log_debug("Regexp: %s" % regexp) + + hdict = self.pyload.pluginManager.plugins[self.plugintype][self.pluginname] + hdict['pattern'] = regexp + hdict['re'] = re.compile(regexp) + + + def unload_plugin(self, plugin): + hdict = self.pyload.pluginManager.plugins[self.plugintype][plugin] + if "module" in hdict: + hdict.pop('module', None) + + if "new_module" in hdict: + hdict.pop('new_module', None) + hdict.pop('new_name', None) + + + def deactivate(self): + """ + Remove override for all plugins. Scheduler job is removed by hookmanager + """ + for plugin in self.supported: + self.unload_plugin(plugin) + + #: Reset pattern + hdict = self.pyload.pluginManager.plugins[self.plugintype][self.pluginname] + + hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') + hdict['re'] = re.compile(hdict['pattern']) diff --git a/module/plugins/internal/MultiCrypter.py b/module/plugins/internal/MultiCrypter.py index ca7b03941..ae8785116 100644 --- a/module/plugins/internal/MultiCrypter.py +++ b/module/plugins/internal/MultiCrypter.py @@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter class MultiCrypter(SimpleCrypter): __name__ = "MultiCrypter" __type__ = "hoster" - __version__ = "0.02" + __version__ = "0.03" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -19,7 +19,7 @@ class MultiCrypter(SimpleCrypter): def init(self): - self.CRYPTER_NAME = self.pyload.pluginManager.crypterPlugins[self.__name__]['name'] + self.PLUGIN_NAME = self.pyload.pluginManager.crypterPlugins[self.__name__]['name'] def _log(self, level, plugintype, pluginname, messages): diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py deleted file mode 100644 index 42a1985b5..000000000 --- a/module/plugins/internal/MultiHook.py +++ /dev/null @@ -1,260 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -import time -import traceback - -from module.plugins.internal.Hook import Hook -from module.utils import decode, remove_chars - - -class MultiHook(Hook): - __name__ = "MultiHook" - __type__ = "hook" - __version__ = "0.54" - __status__ = "testing" - - __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), - ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), - ("reload" , "bool" , "Reload plugin list" , True ), - ("reloadinterval", "int" , "Reload interval in hours" , 12 )] - - __description__ = """Hook plugin for multi hoster/crypter""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team" , "admin@pyload.org" ), - ("Walter Purcaro", "vuolter@gmail.com")] - - - MIN_RELOAD_INTERVAL = 1 * 60 * 60 #: 1 hour - - DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"), - (r'bayfiles\.net' , "bayfiles.com" ), - (r'cloudnator\.com' , "shragle.com" ), - (r'dfiles\.eu' , "depositfiles.com" ), - (r'easy-share\.com' , "crocko.com" ), - (r'freakshare\.net' , "freakshare.com" ), - (r'hellshare\.com' , "hellshare.cz" ), - (r'ifile\.it' , "filecloud.io" ), - (r'nowdownload\.\w+', "nowdownload.sx" ), - (r'nowvideo\.\w+' , "nowvideo.sx" ), - (r'putlocker\.com' , "firedrive.com" ), - (r'share-?rapid\.cz', "multishare.cz" ), - (r'ul\.to' , "uploaded.to" ), - (r'uploaded\.net' , "uploaded.to" ), - (r'uploadhero\.co' , "uploadhero.com" ), - (r'zshares\.net' , "zshare.net" ), - (r'^1' , "one" ), - (r'^2' , "two" ), - (r'^3' , "three" ), - (r'^4' , "four" ), - (r'^5' , "five" ), - (r'^6' , "six" ), - (r'^7' , "seven" ), - (r'^8' , "eight" ), - (r'^9' , "nine" ), - (r'^0' , "zero" )] - - - def init(self): - self.plugins = [] - self.supported = [] - self.new_supported = [] - - self.account = None - self.pluginclass = None - self.pluginmodule = None - self.pluginname = None - self.plugintype = None - - self.init_plugin() - - - def init_plugin(self): - self.pluginname = self.__name__.rsplit("Hook", 1)[0] - plugin, self.plugintype = self.pyload.pluginManager.findPlugin(self.pluginname) - - if plugin: - self.pluginmodule = self.pyload.pluginManager.loadModule(self.plugintype, self.pluginname) - self.pluginclass = getattr(self.pluginmodule, self.pluginname) - else: - self.log_warning(_("Hook plugin will be deactivated due missing plugin reference")) - self.set_config('activated', False) - - - def load_account(self): - self.account = self.pyload.accountManager.getAccountPlugin(self.pluginname) - - if self.account and not self.account.select()[0]: - self.account = False - - if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: - self.log_warning(_("Hook plugin will be deactivated due missing account reference")) - self.set_config('activated', False) - - - def activate(self): - self.init_periodical(threaded=True) - - - def plugins_cached(self): - if self.plugins: - return self.plugins - - for _i in xrange(5): - try: - pluginset = self._plugin_set(self.get_hosters()) - break - - except Exception, e: - self.log_warning(e, _("Waiting 1 minute and retry")) - time.sleep(60) - else: - self.log_error(_("No hoster list retrieved")) - self.interval = self.MIN_RELOAD_INTERVAL - return list() - - try: - configmode = self.get_config('pluginmode', 'all') - if configmode in ("listed", "unlisted"): - pluginlist = self.get_config('pluginlist', '').replace('|', ',').replace(';', ',').split(',') - configset = self._plugin_set(pluginlist) - - if configmode == "listed": - pluginset &= configset - else: - pluginset -= configset - - except Exception, e: - self.log_error(e) - - self.plugins = list(pluginset) - - return self.plugins - - - def _plugin_set(self, plugins): - regexp = re.compile(r'^[\w\-.^_]{3,63}\.[a-zA-Z]{2,}$', re.U) - plugins = [decode(p.strip()).lower() for p in plugins if regexp.match(p.strip())] - - for r in self.DOMAIN_REPLACEMENTS: - rf, rt = r - repr = re.compile(rf, re.I|re.U) - plugins = [re.sub(rf, rt, p) if repr.match(p) else p for p in plugins] - - return set(plugins) - - - def get_hosters(self): - """ - Load list of supported hoster - - :return: List of domain names - """ - raise NotImplementedError - - - def periodical(self): - """ - Reload plugin list periodically - """ - self.load_account() - - if self.get_config('reload', True): - self.interval = max(self.get_config('reloadinterval', 12) * 60 * 60, self.MIN_RELOAD_INTERVAL) - else: - self.pyload.scheduler.removeJob(self.cb) - self.cb = None - - self.log_info(_("Reloading supported %s list") % self.plugintype) - - old_supported = self.supported - - self.supported = [] - self.new_supported = [] - self.plugins = [] - - self.override_plugins() - - old_supported = [plugin for plugin in old_supported if plugin not in self.supported] - - if old_supported: - self.log_debug("Unload: %s" % ", ".join(old_supported)) - for plugin in old_supported: - self.unload_plugin(plugin) - - - def override_plugins(self): - excludedList = [] - - if self.plugintype == "hoster": - pluginMap = dict((name.lower(), name) for name in self.pyload.pluginManager.hosterPlugins.keys()) - accountList = [account.type.lower() for account in self.pyload.api.getAccounts(False) if account.valid and account.premium] - else: - pluginMap = {} - accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.pyload.pluginManager.crypterPlugins.keys()] - - for plugin in self.plugins_cached(): - name = remove_chars(plugin, "-.") - - if name in accountList: - excludedList.append(plugin) - else: - if name in pluginMap: - self.supported.append(pluginMap[name]) - else: - self.new_supported.append(plugin) - - if not self.supported and not self.new_supported: - self.log_error(_("No %s loaded") % self.plugintype) - return - - #: Inject plugin plugin - self.log_debug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) - - for plugin in self.supported: - hdict = self.pyload.pluginManager.plugins[self.plugintype][plugin] - hdict['new_module'] = self.pluginmodule - hdict['new_name'] = self.pluginname - - if excludedList: - self.log_info(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList)))) - - if self.new_supported: - plugins = sorted(self.new_supported) - - self.log_debug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) - - #: Create new regexp - regexp = r'.*(?P%s).*' % "|".join(x.replace('.', '\.') for x in plugins) - if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and "://" in self.pluginclass.__pattern__: - regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) - - self.log_debug("Regexp: %s" % regexp) - - hdict = self.pyload.pluginManager.plugins[self.plugintype][self.pluginname] - hdict['pattern'] = regexp - hdict['re'] = re.compile(regexp) - - - def unload_plugin(self, plugin): - hdict = self.pyload.pluginManager.plugins[self.plugintype][plugin] - if "module" in hdict: - hdict.pop('module', None) - - if "new_module" in hdict: - hdict.pop('new_module', None) - hdict.pop('new_name', None) - - - def deactivate(self): - """ - Remove override for all plugins. Scheduler job is removed by hookmanager - """ - for plugin in self.supported: - self.unload_plugin(plugin) - - #: Reset pattern - hdict = self.pyload.pluginManager.plugins[self.plugintype][self.pluginname] - - hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') - hdict['re'] = re.compile(hdict['pattern']) diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index c0c928a45..fbfab1ade 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.50" + __version__ = "0.51" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -21,21 +21,21 @@ class MultiHoster(SimpleHoster): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - HOSTER_NAME = None + PLUGIN_NAME = None LEECH_HOSTER = False LOGIN_ACCOUNT = True def init(self): - self.HOSTER_NAME = self.pyload.pluginManager.hosterPlugins[self.__name__]['name'] + self.PLUGIN_NAME = self.pyload.pluginManager.hosterPlugins[self.__name__]['name'] def _log(self, level, plugintype, pluginname, messages): return super(MultiHoster, self)._log(level, plugintype, pluginname, - (self.HOSTER_NAME,) + messages) + (self.PLUGIN_NAME,) + messages) def setup(self): @@ -83,11 +83,11 @@ class MultiHoster(SimpleHoster): self.check_errors() self.check_status(getinfo=False) - if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): + if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as premium download...")) self.handle_premium(pyfile) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as free download...")) self.handle_free(pyfile) @@ -95,7 +95,7 @@ class MultiHoster(SimpleHoster): self.log_info(_("Downloading file...")) self.download(self.link, disposition=self.DISPOSITION) - self.check_file() + self.check_download() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 if self.premium: diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 51192d8c9..f1cd74fcb 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -6,7 +6,6 @@ import datetime import inspect import os import re -import sys import traceback import urllib import urlparse @@ -28,6 +27,11 @@ def decode(string, encoding='utf8'): return unicode(string) +#@TODO: Remove in 0.4.10 +def _decode(*args, **kwargs): + return decode(*args, **kwargs) + + #@TODO: Move to utils in 0.4.10 def encode(string, encoding='utf8'): """ Decode string to utf8 """ @@ -49,18 +53,25 @@ def exists(path): return False -#@TODO: Move to utils in 0.4.10 -def parse_name(url): - url = urllib.unquote(url) - url = url.decode('unicode-escape') - url = html_unescape(url) - url = urllib.quote(url) +#@TODO: Recheck in 0.4.10 +def fixurl(url, unquote=None): + newurl = urllib.unquote(url) + + if unquote is None: + unquote = newurl == url + + newurl = html_unescape(newurl.decode('unicode-escape')).strip() - url_p = urlparse.urlparse(url.strip().rstrip('/')) + return newurl if unquote else urllib.quote(newurl) - name = (url_p.path.split('/')[-1] or - url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or - url_p.netloc.split('.', 1)[0]) + +#@TODO: Recheck in 0.4.10 +def parse_name(string): + path = fixurl(decode(string), unquote=False) + url_p = urlparse.urlparse(path.rstrip('/')) + name = (url_p.path.split('/')[-1] or + url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or + url_p.netloc.split('.', 1)[0]) return urllib.unquote(name) @@ -96,7 +107,7 @@ def seconds_to_midnight(utc=None): else: now = datetime.datetime.utcnow() + datetime.timedelta(hours=utc) - midnight = now.replace(hour=0, minute=10, second=0, microsecond=0) + datetime.timedelta(days=1) + midnight = now.replace(hour=0, minute=1, second=0, microsecond=0) + datetime.timedelta(days=1) return (midnight - now).seconds @@ -173,7 +184,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.37" + __version__ = "0.38" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -199,8 +210,8 @@ class Plugin(object): def _init(self, core): self.pyload = core - self.info = {} #: Provide information in dict here - self.req = None + self.info = {} #: Provide information in dict here + self.req = None #: Browser instance, see `network.Browser` def init(self): @@ -231,8 +242,6 @@ class Plugin(object): def log_warning(self, *args): self._log("warning", self.__type__, self.__name__, args) - if self.pyload.debug: - traceback.print_exc() def log_error(self, *args): @@ -278,7 +287,7 @@ class Plugin(object): return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) - def set_config(self, option, value): + def set_config(self, option, value, plugin=None): """ Set config value for current plugin @@ -286,7 +295,7 @@ class Plugin(object): :param value: :return: """ - self.pyload.config.setPlugin(self.__name__, option, value) + self.pyload.config.setPlugin(plugin or self.__name__, option, value) def get_config(self, option, default="", plugin=None): @@ -347,7 +356,9 @@ class Plugin(object): """ if self.pyload.debug: self.log_debug("LOAD URL " + url, - *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) + + url = fixurl(url, unquote=True) #: Recheck in 0.4.10 if req is None: req = self.req or self.pyload.requestFactory.getRequest(self.__name__) @@ -364,7 +375,7 @@ class Plugin(object): #@TODO: Move to network in 0.4.10 if isinstance(decode, basestring): - res = sys.modules[self.__name__].decode(res, decode) #@TODO: See #1787, use utils.decode() in 0.4.10 + res = _decode(res, decode) #@NOTE: Use `utils.decode()` in 0.4.10 if self.pyload.debug: frame = inspect.currentframe() @@ -409,11 +420,12 @@ class Plugin(object): Clean everything and remove references """ try: + self.req.clearCookies() self.req.close() except Exception: pass - for a in ("pyfile", "thread", "html", "req"): - if hasattr(self, a): - setattr(self, a, None) + for attr in ("account", "html", "pyfile", "req", "thread"): + if hasattr(self, attr): + setattr(self, attr, None) diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index b79256536..9d3a71a21 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -5,12 +5,12 @@ import re import subprocess from module.plugins.internal.UnRar import ArchiveError, CRCError, PasswordError, UnRar, renice -from module.utils import fs_encode, save_join as fs_join +from module.utils import save_join as fs_join class SevenZip(UnRar): __name__ = "SevenZip" - __version__ = "0.15" + __version__ = "0.16" __status__ = "testing" __description__ = """7-Zip extractor plugin""" @@ -128,7 +128,7 @@ class SevenZip(UnRar): args.append("-y") #: Set a password - if "password" in kwargs and kwargs['password']: + if kwargs.get('password'): args.append("-p%s" % kwargs['password']) else: args.append("-p-") diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 8c5d3599d..53d69b2a5 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.62" + __version__ = "0.63" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -81,7 +81,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.log_debug("Redirect #%d to: %s" % (i, redirect)) header = self.load(redirect, just_header=True) - if 'location' in header and header['location']: + if header.get('location'): self.link = header['location'] else: break @@ -137,7 +137,7 @@ class SimpleCrypter(Crypter, SimpleHoster): try: pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) - except AttributeError: + except (AttributeError, IndexError, ValueError): pages = 1 for p in xrange(2, pages + 1): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 0f030e000..00b63ac12 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -6,7 +6,6 @@ import os import re import time -from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo @@ -14,14 +13,10 @@ from module.plugins.internal.Plugin import Fail, encode, parse_name, replace_pat from module.utils import fixup, fs_encode, parseFileSize as parse_size -#@TODO: Adapt and move to PyFile in 0.4.10 -statusMap = dict((v, k) for k, v in _statusMap.items()) - - class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.86" + __version__ = "1.87" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -124,7 +119,7 @@ class SimpleHoster(Hoster): try: info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: Pattern groups will be saved here - except AttributeError: + except (AttributeError, IndexError): info['pattern'] = {} if not html and not online: @@ -164,7 +159,7 @@ class SimpleHoster(Hoster): if all(True for k in pdict if k not in info['pattern']): info['pattern'].update(pdict) - except AttributeError: + except (AttributeError, IndexError): continue else: @@ -275,11 +270,11 @@ class SimpleHoster(Hoster): if 'status' not in self.info or self.info['status'] is 3: #@TODO: Recheck in 0.4.10 self.check_info() - if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): + if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as premium download...")) self.handle_premium(pyfile) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as free download...")) self.handle_free(pyfile) @@ -287,7 +282,7 @@ class SimpleHoster(Hoster): self.log_info(_("Downloading file...")) self.download(self.link, disposition=self.DISPOSITION) - self.check_file() + self.check_download() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 if self.get_config('premium_fallback', True) and self.premium: @@ -298,26 +293,26 @@ class SimpleHoster(Hoster): raise Fail(encode(e)) #@TODO: Remove `encode` in 0.4.10 - def check_file(self): - self.log_info(_("Checking file...")) + def check_download(self): + self.log_info(_("Checking downloaded file...")) if self.captcha.task and not self.last_download: self.captcha.invalid() self.retry(10, msg=_("Wrong captcha")) - elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, + elif self.check_file({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, delete=True): self.error(_("Empty file")) else: - if self.get_config('chk_filesize', False) and 'size' in self.info: + if self.get_config('chk_filesize', False) and self.info.get('size'): # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB self.check_filesize(self.info['size'], size_tolerance=10485760) self.log_debug("Using default check rules...") for r, p in self.FILE_ERRORS: - errmsg = self.check_download({r: re.compile(p)}) + errmsg = self.check_file({r: re.compile(p)}) if errmsg is not None: errmsg = errmsg.strip().capitalize() @@ -338,7 +333,6 @@ class SimpleHoster(Hoster): self.check_errors() self.log_info(_("No errors found")) - self.pyfile.error = "" #@TODO: Recheck in 0.4.10 def check_errors(self): @@ -361,7 +355,7 @@ class SimpleHoster(Hoster): try: errmsg = m.group(1).strip() - except AttributeError: + except (AttributeError, IndexError): errmsg = m.group(0).strip() self.info['error'] = re.sub(r'<.*?>', " ", errmsg) @@ -385,7 +379,7 @@ class SimpleHoster(Hoster): try: errmsg = m.group(1).strip() - except AttributeError: + except (AttributeError, IndexError): errmsg = m.group(0).strip() self.info['error'] = re.sub(r'<.*?>', " ", errmsg) @@ -436,7 +430,7 @@ class SimpleHoster(Hoster): try: waitmsg = m.group(1).strip() - except AttributeError: + except (AttributeError, IndexError): waitmsg = m.group(0).strip() wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in @@ -455,22 +449,19 @@ class SimpleHoster(Hoster): self.log_debug("Previous file info: %s" % old_info) try: - status = self.info['status'] or None + status = self.info['status'] or 14 - if status == 1: + if status is 1: self.offline() - elif status == 6: + elif status is 6: self.temp_offline() - elif status == 8: - if 'error' in self.info: - self.fail(self.info['error']) - else: - self.fail(_("File status: " + statusMap[status])) + elif status is 8: + self.fail() finally: - self.log_info(_("File status: ") + (statusMap[status] if status else _("Unknown"))) + self.log_info(_("File status: ") + self.pyfile.getStatusName(status)) def check_name_size(self, getinfo=True): @@ -492,7 +483,7 @@ class SimpleHoster(Hoster): if name and name is not url: self.pyfile.name = name - if 'size' in self.info and self.info['size'] > 0: + if self.info.get('size') > 0: self.pyfile.size = int(self.info['size']) #@TODO: Fix int conversion in 0.4.10 # self.pyfile.sync() diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 88c490750..7efecc5ef 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -8,7 +8,7 @@ from glob import glob from string import digits from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import fs_decode, fs_encode, save_join as fs_join +from module.utils import fs_decode, save_join as fs_join def renice(pid, value): @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.26" + __version__ = "1.27" __status__ = "testing" __description__ = """Rar extractor plugin""" @@ -229,7 +229,7 @@ class UnRar(Extractor): args.append("-y") #: Set a password - if "password" in kwargs and kwargs['password']: + if kwargs.get('password'): args.append("-p%s" % kwargs['password']) else: args.append("-p-") diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index ac197a80d..87cbd568a 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -7,7 +7,6 @@ import sys import zipfile from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import fs_encode class UnZip(Extractor): diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 12a48d4a3..8047fd03e 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -7,7 +7,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.14" + __version__ = "0.15" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -17,7 +17,7 @@ class XFSCrypter(SimpleCrypter): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - HOSTER_DOMAIN = None + PLUGIN_DOMAIN = None URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')] @@ -29,22 +29,22 @@ class XFSCrypter(SimpleCrypter): def prepare(self): - if not self.HOSTER_DOMAIN: + if not self.PLUGIN_DOMAIN: if self.account: account = self.account else: account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") account = self.pyload.accountManager.getAccountPlugin(account_name) - if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: - self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + if account and hasattr(account, "PLUGIN_DOMAIN") and account.PLUGIN_DOMAIN: + self.PLUGIN_DOMAIN = account.PLUGIN_DOMAIN else: - self.fail(_("Missing HOSTER_DOMAIN")) + self.fail(_("Missing PLUGIN_DOMAIN")) if self.COOKIES: - if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: + self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) else: - set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") + set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") return super(XFSCrypter, self).prepare() diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 729c9a0ee..8af6b83ed 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.60" + __version__ = "0.61" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -26,7 +26,7 @@ class XFSHoster(SimpleHoster): ("Walter Purcaro", "vuolter@gmail.com" )] - HOSTER_DOMAIN = None + PLUGIN_DOMAIN = None LEECH_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... @@ -61,26 +61,26 @@ class XFSHoster(SimpleHoster): """ Initialize important variables """ - if not self.HOSTER_DOMAIN: + if not self.PLUGIN_DOMAIN: if self.account: account = self.account else: account = self.pyload.accountManager.getAccountPlugin(self.__name__) - if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: - self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + if account and hasattr(account, "PLUGIN_DOMAIN") and account.PLUGIN_DOMAIN: + self.PLUGIN_DOMAIN = account.PLUGIN_DOMAIN else: - self.fail(_("Missing HOSTER_DOMAIN")) + self.fail(_("Missing PLUGIN_DOMAIN")) if self.COOKIES: - if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: + self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) else: - set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") + set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") if not self.LINK_PATTERN: pattern = r'(?:file: "(.+?)"|(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<])' - self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') + self.LINK_PATTERN = pattern % self.PLUGIN_DOMAIN.replace('.', '\.') super(XFSHoster, self).prepare() @@ -129,7 +129,7 @@ class XFSHoster(SimpleHoster): self.fail(_("Only registered or premium users can use url leech feature")) #: Only tested with easybytez.com - self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) + self.html = self.load("http://www.%s/" % self.PLUGIN_DOMAIN) action, inputs = self.parse_html_form() @@ -244,7 +244,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) - except AttributeError: + except (AttributeError, IndexError): captcha_key = recaptcha.detect_key() else: @@ -258,7 +258,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) - except AttributeError: + except (AttributeError, IndexError): captcha_key = solvemedia.detect_key() else: -- cgit v1.2.3 From 9fdf1ed882fa981efd96179aaee23fb87e597c9b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 29 Sep 2015 21:48:26 +0200 Subject: Update crypter plugins --- module/plugins/internal/Container.py | 3 ++- module/plugins/internal/Plugin.py | 5 +---- module/plugins/internal/SevenZip.py | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 524b55839..2300c4cab 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -20,7 +20,8 @@ class Container(Crypter): __description__ = """Base container decrypter plugin""" __license__ = "GPLv3" - __authors__ = [("mkaay", "mkaay@mkaay.de")] + __authors__ = [("mkaay" , "mkaay@mkaay.de" ), + ("Walter Purcaro", "vuolter@gmail.com")] def process(self, pyfile): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index f1cd74fcb..6f3e52962 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -192,10 +192,7 @@ class Plugin(object): __description__ = """Base plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), - ("spoob" , "spoob@pyload.org" ), - ("mkaay" , "mkaay@mkaay.de" ), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] def __init__(self, core): diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index 9d3a71a21..7ed64ecdf 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -15,8 +15,8 @@ class SevenZip(UnRar): __description__ = """7-Zip extractor plugin""" __license__ = "GPLv3" - __authors__ = [("Michael Nowak" , "" ), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com"), + ("Michael Nowak" , None )] CMD = "7z" -- cgit v1.2.3 From 3353ea228835ffa96cc73d5b5e23f6d92ba84203 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 29 Sep 2015 22:43:17 +0200 Subject: Update hoster plugins --- module/plugins/internal/Account.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 5c76d4ccb..918c32c42 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -12,7 +12,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.50" + __version__ = "0.51" __status__ = "testing" __description__ = """Base account plugin""" @@ -187,14 +187,14 @@ class Account(Plugin): return self.info - def get_login(self, key=None): + def get_login(self, key=None, default=None): d = self.get_info()['login'] - return d.get(key) or d + return d.get(key, default) if key or d - def get_data(self, key=None): + def get_data(self, key=None, default=None): d = self.get_info()['data'] - return d.get(key) or d + return d.get(key, default) if key or d def _grab_info(self): -- cgit v1.2.3 From ee746bbe9c1bdb8fefbb3c8c8ac52d97672e5f19 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 29 Sep 2015 23:04:51 +0200 Subject: [Account] Hotfix --- module/plugins/internal/Account.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 918c32c42..196589e85 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -12,7 +12,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.51" + __version__ = "0.52" __status__ = "testing" __description__ = """Base account plugin""" @@ -189,12 +189,12 @@ class Account(Plugin): def get_login(self, key=None, default=None): d = self.get_info()['login'] - return d.get(key, default) if key or d + return d.get(key, default) if key else d def get_data(self, key=None, default=None): d = self.get_info()['data'] - return d.get(key, default) if key or d + return d.get(key, default) if key else d def _grab_info(self): -- cgit v1.2.3 From 9fdc7e026f763368acea68a9c52d54bef952af54 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 30 Sep 2015 00:12:53 +0200 Subject: [SimpleHoster] Fix https://github.com/pyload/pyload/issues/1890 --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 00b63ac12..3bb3ff211 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -16,7 +16,7 @@ from module.utils import fixup, fs_encode, parseFileSize as parse_size class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.87" + __version__ = "1.88" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -461,7 +461,7 @@ class SimpleHoster(Hoster): self.fail() finally: - self.log_info(_("File status: ") + self.pyfile.getStatusName(status)) + self.log_info(_("File status: ") + self.pyfile.getStatusName()) def check_name_size(self, getinfo=True): -- cgit v1.2.3 From 0c9e92172f69769f72a2a258f717be4bc4d7a397 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 30 Sep 2015 00:29:22 +0200 Subject: [XFSHoster] Revert https://github.com/pyload/pyload/pull/1883 --- module/plugins/internal/XFSHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index ffeab21a5..8af6b83ed 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -45,8 +45,8 @@ class XFSHoster(SimpleHoster): CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' CAPTCHA_BLOCK_PATTERN = r'>Enter code.*?(.+?)' - RECAPTCHA_PATTERN = r'api\.recaptcha\.net/challenge\?k=(.+?)["\']' - SOLVEMEDIA_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']' + RECAPTCHA_PATTERN = None + SOLVEMEDIA_PATTERN = None FORM_PATTERN = None FORM_INPUTS_MAP = None #: Dict passed as input_names to parse_html_form -- cgit v1.2.3 From 3b3a4bd889286777ba6a590a1f74cf3edae36a72 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 30 Sep 2015 00:30:53 +0200 Subject: Add __config__ to MultiAccount (and XFSAccount) based plugins --- module/plugins/internal/Account.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 196589e85..7060959ee 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -303,6 +303,7 @@ class Account(Plugin): @lock def removeAccount(self, user): + self.log_info(_("Removing user `%s`...") % user) self.accounts.pop(user, None) if user is self.user: self.choose() -- cgit v1.2.3 From c3b6253f1ea9b8c4a5aeb2c2c0be292f66a2abee Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 30 Sep 2015 00:44:55 +0200 Subject: [Captcha] _decrypt -> decrypt_image --- module/plugins/internal/Captcha.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 93642b32a..3d105ed12 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -11,7 +11,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.45" + __version__ = "0.46" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -52,11 +52,10 @@ class Captcha(Plugin): def decrypt(self, url, get={}, post={}, ref=False, cookies=True, decode=False, req=None, input_type='jpg', output_type='textual', ocr=True, timeout=120): img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode, req=req or self.plugin.req) - return self._decrypt(img, input_type, output_type, ocr, timeout) + return self.decrypt_image(img, input_type, output_type, ocr, timeout) - #@TODO: Definitely choose a better name for this method! - def _decrypt(self, data, input_type='jpg', output_type='textual', ocr=False, timeout=120): + def decrypt_image(self, data, input_type='jpg', output_type='textual', ocr=False, timeout=120): """ Loads a captcha and decrypts it with ocr, plugin, user input -- cgit v1.2.3 From 71c075a25b43220f793269c122b4fb17aa63f160 Mon Sep 17 00:00:00 2001 From: Manuel Mausz Date: Wed, 30 Sep 2015 21:53:04 +0200 Subject: Strip trailing slashes before case sensitive checking Strip trailing slashes. Otherwise the last pathname component might be empty --- module/plugins/internal/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 6f3e52962..771966389 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -45,7 +45,7 @@ def encode(string, encoding='utf8'): def exists(path): if os.path.exists(path): if os.name == "nt": - dir, name = os.path.split(path) + dir, name = os.path.split(path.rstrip(os.sep)) return name in os.listdir(dir) else: return True -- cgit v1.2.3 From 2937185811f427f3c052bcfa4d498c8d0890fd6a Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Thu, 1 Oct 2015 01:05:48 +0300 Subject: Update XFSHoster.py --- module/plugins/internal/XFSHoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 8af6b83ed..346422ff7 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.61" + __version__ = "0.62" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -244,7 +244,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) - except (AttributeError, IndexError): + except (AttributeError, IndexError, TypeError): captcha_key = recaptcha.detect_key() else: @@ -258,7 +258,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) - except (AttributeError, IndexError): + except (AttributeError, IndexError, TypeError): captcha_key = solvemedia.detect_key() else: -- cgit v1.2.3 From 21be6c5990af016d5ef4298ece52bf434d11f306 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 1 Oct 2015 04:59:34 +0200 Subject: A lot of plugin code cosmetics (2) --- module/plugins/internal/Account.py | 2 +- module/plugins/internal/Captcha.py | 3 +-- module/plugins/internal/XFSAccount.py | 4 ++-- module/plugins/internal/XFSHoster.py | 12 ++++++------ 4 files changed, 10 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 7060959ee..123241be3 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -331,7 +331,7 @@ class Account(Plugin): continue except Exception: - self.log_warning(_("Wrong time format `%s` for account `%s`, use 1:22-3:44") + self.log_warning(_("Invalid time format `%s` for account `%s`, use 1:22-3:44") % (user, time_data)) if data['trafficleft'] == 0: diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 3d105ed12..a8f48b5e4 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -104,8 +104,7 @@ class Captcha(Plugin): self.fail(self.task.error) elif not self.task.result: - self.invalid() - self.plugin.retry(msg=_("No captcha result obtained in appropiate time")) + self.plugin.retry_captcha(msg=_("No captcha result obtained in appropriate time")) result = self.task.result diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 91136a2a0..3b92a191b 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -61,7 +61,7 @@ class XFSAccount(Account): premium = True if re.search(self.PREMIUM_PATTERN, html) else False m = re.search(self.VALID_UNTIL_PATTERN, html) - if m: + if m is not None: expiredate = m.group(1).strip() self.log_debug("Expire date: " + expiredate) @@ -84,7 +84,7 @@ class XFSAccount(Account): self.log_debug("VALID_UNTIL_PATTERN not found") m = re.search(self.TRAFFIC_LEFT_PATTERN, html) - if m: + if m is not None: try: traffic = m.groupdict() size = traffic['S'] diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 8af6b83ed..98ed4e016 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -95,7 +95,7 @@ class XFSHoster(SimpleHoster): self.check_errors() m = re.search(self.LINK_PATTERN, self.html, re.S) - if m: + if m is not None: break data = self.get_post_parameters() @@ -111,7 +111,7 @@ class XFSHoster(SimpleHoster): break m = re.search(self.LINK_PATTERN, self.html, re.S) - if m: + if m is not None: break else: if 'op' in data: @@ -163,7 +163,7 @@ class XFSHoster(SimpleHoster): self.retry(20, 3 * 60, _("Can not leech file")) elif 'today' in stmsg: - self.retry(delay=seconds_to_midnight(), msg=_("You've used all Leech traffic today")) + self.retry(wait=seconds_to_midnight(), msg=_("You've used all Leech traffic today")) else: self.fail(stmsg) @@ -202,7 +202,7 @@ class XFSHoster(SimpleHoster): if not self.premium: m = re.search(self.WAIT_PATTERN, self.html) - if m: + if m is not None: wait_time = int(m.group(1)) self.set_wait(wait_time, False) @@ -223,13 +223,13 @@ class XFSHoster(SimpleHoster): def handle_captcha(self, inputs): m = re.search(self.CAPTCHA_PATTERN, self.html) - if m: + if m is not None: captcha_url = m.group(1) inputs['code'] = self.captcha.decrypt(captcha_url) return m = re.search(self.CAPTCHA_BLOCK_PATTERN, self.html, re.S) - if m: + if m is not None: captcha_div = m.group(1) numerals = re.findall(r'(\d)', html_unescape(captcha_div)) -- cgit v1.2.3 From 8d85a655d8cc408574f471306613bc05115c8c64 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 1 Oct 2015 18:59:55 +0200 Subject: Improve Hoster and Crypter + general fixup --- module/plugins/internal/Base.py | 493 ++++++++++++++++++++++++++++++ module/plugins/internal/Crypter.py | 28 +- module/plugins/internal/Hoster.py | 502 ++++--------------------------- module/plugins/internal/MultiCrypter.py | 5 +- module/plugins/internal/MultiHoster.py | 12 +- module/plugins/internal/Plugin.py | 45 ++- module/plugins/internal/SimpleCrypter.py | 5 +- module/plugins/internal/SimpleHoster.py | 134 ++++----- module/plugins/internal/XFSAccount.py | 18 +- module/plugins/internal/XFSCrypter.py | 20 +- module/plugins/internal/XFSHoster.py | 21 +- 11 files changed, 701 insertions(+), 582 deletions(-) create mode 100644 module/plugins/internal/Base.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Base.py b/module/plugins/internal/Base.py new file mode 100644 index 000000000..4235cf94d --- /dev/null +++ b/module/plugins/internal/Base.py @@ -0,0 +1,493 @@ +# -*- coding: utf-8 -*- + +import inspect +import mimetypes +import os +import time +import urlparse + +from module.plugins.internal.Captcha import Captcha +from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, + decode, encode, fixurl, parse_html_form, + parse_name, replace_patterns) + + +#@TODO: Remove in 0.4.10 +def getInfo(urls): + #: result = [ .. (name, size, status, url) .. ] + pass + + +#@TODO: Remove in 0.4.10 +def create_getInfo(klass): + def get_info(urls): + for url in urls: + if hasattr(klass, "URL_REPLACEMENTS"): + url = replace_patterns(url, klass.URL_REPLACEMENTS) + yield parse_fileInfo(klass, url) + + return get_info + + +#@NOTE: `check_abort` decorator +def check_abort(fn): + + def wrapper(self, *args, **kwargs): + self.check_abort() + return fn(self, *args, **kwargs) + + return wrapper + + +class Base(Plugin): + __name__ = "Base" + __type__ = "base" + __version__ = "0.01" + __status__ = "testing" + + __pattern__ = r'^unmatchable$' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] + + __description__ = """Base plugin for Hoster and Crypter""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def __init__(self, pyfile): + self._init(pyfile.m.core) + + #: Engage wan reconnection + self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10 + + #: Enable simultaneous processing of multiple downloads + self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10 + + #: time.time() + wait in seconds + self.wait_until = 0 + self.waiting = False + + #: Account handler instance, see :py:class:`Account` + self.account = None + self.user = None #@TODO: Remove in 0.4.10 + + #: Associated pyfile instance, see `PyFile` + self.pyfile = pyfile + + self.thread = None #: Holds thread in future + + #: Js engine, see `JsEngine` + self.js = self.pyload.js + + #: Captcha stuff + self.captcha = Captcha(self) + + #: Some plugins store html code here + self.html = None + + #: Dict of the amount of retries already made + self.retries = {} + + + def _log(self, level, plugintype, pluginname, messages): + log = getattr(self.pyload.log, level) + msg = u" | ".join(decode(a).strip() for a in messages if a) + log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" + % {'plugintype': plugintype.upper(), + 'pluginname': pluginname, + 'id' : self.pyfile.id, + 'msg' : msg}) + + + @classmethod + def get_info(cls, url="", html=""): + url = fixurl(url) + info = {'name' : parse_name(url), + 'size' : 0, + 'status': 3 if url else 8, + 'url' : url} + + return info + + + def init(self): + """ + Initialize the plugin (in addition to `__init__`) + """ + pass + + + def setup(self): + """ + Setup for enviroment and other things, called before downloading (possibly more than one time) + """ + pass + + + def _setup(self): + #@TODO: Remove in 0.4.10 + self.html = "" + self.pyfile.error = "" + self.last_html = None + + if self.get_config('use_premium', True): + self.load_account() #@TODO: Move to PluginThread in 0.4.10 + else: + self.account = False + self.user = None #@TODO: Remove in 0.4.10 + + try: + self.req.close() + except Exception: + pass + + if self.account: + self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) + self.chunk_limit = -1 #: -1 for unlimited + self.resume_download = True + self.premium = self.account.premium + else: + self.req = self.pyload.requestFactory.getRequest(self.__name__) + self.chunk_limit = 1 + self.resume_download = False + self.premium = False + + self.setup() + + + def load_account(self): + if not self.account: + self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) + + if not self.account: + self.account = False + self.user = None #@TODO: Remove in 0.4.10 + + else: + self.account.choose() + self.user = self.account.user #@TODO: Remove in 0.4.10 + if self.account.user is None: + self.account = False + + + def _process(self, thread): + """ + Handles important things to do before starting + """ + self.thread = thread + + self._setup() + + # self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 + self.check_abort() + + self.pyfile.setStatus("starting") + + self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) + self.process(self.pyfile) + + + #: Deprecated method, use `_process` instead (Remove in 0.4.10) + def preprocessing(self, *args, **kwargs): + return self._process(*args, **kwargs) + + + def process(self, pyfile): + """ + The "main" method of every hoster plugin, you **have to** overwrite it + """ + raise NotImplementedError + + + def set_reconnect(self, reconnect): + self.log_debug("RECONNECT %s required" % ("" if reconnect else "not"), + "Previous wantReconnect: %s" % self.wantReconnect) + self.wantReconnect = bool(reconnect) + + + def set_wait(self, seconds, reconnect=None): + """ + Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + wait_time = max(int(seconds), 1) + wait_until = time.time() + wait_time + 1 + + self.log_debug("WAIT set to %d seconds" % wait_time, + "Previous waitUntil: %f" % self.pyfile.waitUntil) + + self.pyfile.waitUntil = wait_until + + if reconnect is not None: + self.set_reconnect(reconnect) + + + def wait(self, seconds=None, reconnect=None): + """ + Waits the time previously set + """ + pyfile = self.pyfile + + if seconds is not None: + self.set_wait(seconds) + + if reconnect is not None: + self.set_reconnect(reconnect) + + self.waiting = True + + status = pyfile.status #@NOTE: Recheck in 0.4.10 + pyfile.setStatus("waiting") + + self.log_info(_("Waiting %d seconds...") % (pyfile.waitUntil - time.time())) + + if self.wantReconnect: + self.log_info(_("Requiring reconnection...")) + if self.account: + self.log_warning("Ignore reconnection due logged account") + + if not self.wantReconnect or self.account: + while pyfile.waitUntil > time.time(): + self.check_abort() + time.sleep(2) + + else: + while pyfile.waitUntil > time.time(): + self.check_abort() + self.thread.m.reconnecting.wait(1) + + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + time.sleep(2) + + self.waiting = False + pyfile.status = status #@NOTE: Recheck in 0.4.10 + + + def skip(self, msg=""): + """ + Skip and give msg + """ + raise Skip(encode(msg or self.pyfile.error or self.pyfile.pluginname)) #@TODO: Remove `encode` in 0.4.10 + + + #@TODO: Remove in 0.4.10 + def fail(self, msg): + """ + Fail and give msg + """ + msg = msg.strip() + + if msg: + self.pyfile.error = msg + else: + msg = self.pyfile.error or (self.info['error'] if 'error' in self.info else self.pyfile.getStatusName()) + + raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 + + + def error(self, msg="", type=_("Parse")): + type = _("%s error") % type.strip().capitalize() if type else _("Unknown") + msg = _("%(type)s: %(msg)s | Plugin may be out of date" + % {'type': type, 'msg': msg or self.pyfile.error}) + + self.fail(msg) + + + def abort(self, msg=""): + """ + Abort and give msg + """ + if msg: #@TODO: Remove in 0.4.10 + self.pyfile.error = encode(msg) + + raise Abort + + + #@TODO: Recheck in 0.4.10 + def offline(self, msg=""): + """ + Fail and indicate file is offline + """ + self.fail("offline") + + + #@TODO: Recheck in 0.4.10 + def temp_offline(self, msg=""): + """ + Fail and indicates file ist temporary offline, the core may take consequences + """ + self.fail("temp. offline") + + + def retry(self, attemps=5, wait=1, msg=""): + """ + Retries and begin again from the beginning + + :param attemps: number of maximum retries + :param wait: time to wait in seconds before retry + :param msg: message passed to fail if attemps value was reached + """ + id = inspect.currentframe().f_back.f_lineno + if id not in self.retries: + self.retries[id] = 0 + + if 0 < attemps <= self.retries[id]: + self.fail(msg or _("Max retries reached")) + + self.wait(wait, False) + + self.retries[id] += 1 + raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 + + + def retry_captcha(self, attemps=10, wait=1, msg=_("Wrong captcha")): + self.captcha.invalid() + self.retry(attemps, wait, msg) + + + def fixurl(self, url, baseurl=None, unquote=True): + url = fixurl(url) + + if not baseurl: + baseurl = fixurl(self.pyfile.url) + + if not urlparse.urlparse(url).scheme: + url_p = urlparse.urlparse(baseurl) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + url = urlparse.urljoin(baseurl, url) + + return fixurl(url, unquote) + + + @check_abort + def load(self, *args, **kwargs): + return super(Hoster, self).load(*args, **kwargs) + + + def check_abort(self): + if not self.pyfile.abort: + return + + if self.pyfile.status is 8: + self.fail() + + elif self.pyfile.status is 4: + self.skip(self.pyfile.statusname) + + elif self.pyfile.status is 1: + self.offline() + + elif self.pyfile.status is 6: + self.temp_offline() + + else: + self.abort() + + + def direct_link(self, url, follow_location=None): + link = "" + + if follow_location is None: + redirect = 1 + + elif type(follow_location) is int: + redirect = max(follow_location, 1) + + else: + redirect = self.get_config("maxredirs", 10, "UserAgentSwitcher") + + for i in xrange(redirect): + try: + self.log_debug("Redirect #%d to: %s" % (i, url)) + header = self.load(url, just_header=True) + + except Exception: #: Bad bad bad... rewrite this part in 0.4.10 + res = self.load(url, + just_header=True, + req=self.pyload.requestFactory.getRequest(self.__name__)) + + header = {'code': req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: + continue + + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() + + if key in header: + if type(header[key]) is list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value + + if 'content-disposition' in header: + link = url + + elif header.get('location'): + location = self.fixurl(header['location'], url) + + if header.get('code') == 302: + link = location + + if follow_location: + url = location + continue + + else: + extension = os.path.splitext(parse_name(url))[-1] + + if header.get('content-type'): + mimetype = header['content-type'].split(';')[0].strip() + + elif extension: + mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" + + else: + mimetype = "" + + if mimetype and (link or 'html' not in mimetype): + link = url + else: + link = "" + + break + + else: + try: + self.log_error(_("Too many redirects")) + + except Exception: + pass + + return link + + + def parse_html_form(self, attr_str="", input_names={}): + return parse_html_form(attr_str, self.html, input_names) + + + def get_password(self): + """ + Get the password the user provided in the package + """ + return self.pyfile.package().password or "" + + + def clean(self): + """ + Clean everything and remove references + """ + super(Base, self).clean() + + for attr in ("account", "html", "pyfile", "thread"): + if hasattr(self, attr): + setattr(self, attr, None) diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index ad5bcc74e..069ffb589 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -1,17 +1,18 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.Hoster import Hoster, parse_name +from module.plugins.internal.Base import Base, parse_name from module.utils import save_path as safe_filename -class Crypter(Hoster): +class Crypter(Base): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.09" + __version__ = "0.10" __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides pyload.config.get("general", "folder_per_package") + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides pyload.config.get("general", "folder_per_package") ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] __description__ = """Base decrypter plugin""" @@ -19,11 +20,8 @@ class Crypter(Hoster): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - html = None #: Last html loaded #@TODO: Move to Hoster - - def __init__(self, pyfile): - super(Crypter, self).__init__(pyfile) + super(Base, self).__init__(pyfile) #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) self.packages = [] @@ -31,6 +29,16 @@ class Crypter(Hoster): #: List of urls, pyLoad will generate packagenames self.urls = [] + self._setup() + self.init() + + + def _setup(self): + super(Base, self)._setup() + + self.packages = [] + self.urls = [] + def process(self, pyfile): """ @@ -48,6 +56,9 @@ class Crypter(Hoster): def decrypt(self, pyfile): + """ + The "main" method of every crypter plugin, you **have to** overwrite it + """ raise NotImplementedError @@ -77,6 +88,7 @@ class Crypter(Hoster): "Saved to folder: %s" % folder if folder else "Saved to download folder") links = map(self.fixurl, links) + pid = self.pyload.api.addPackage(name, links, package_queue) if package_password: diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index bc340e78f..d1b894c6f 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -2,19 +2,10 @@ from __future__ import with_statement -import inspect -import mimetypes import os -import random -import time -import urlparse - -from module.plugins.internal.Captcha import Captcha -from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, decode, encode, exists, fixurl, - parse_html_form, parse_html_tag_attr_value, parse_name, - replace_patterns, seconds_to_midnight, - set_cookie, set_cookies, timestamp) + +from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo +from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename @@ -24,40 +15,17 @@ def parse_fileInfo(klass, url="", html=""): return encode(info['name']), info['size'], info['status'], info['url'] -#@TODO: Remove in 0.4.10 -def getInfo(urls): - #: result = [ .. (name, size, status, url) .. ] - pass - - -#@TODO: Remove in 0.4.10 -def create_getInfo(klass): - def get_info(urls): - for url in urls: - if hasattr(klass, "URL_REPLACEMENTS"): - url = replace_patterns(url, klass.URL_REPLACEMENTS) - yield parse_fileInfo(klass, url) - - return get_info - - -#@NOTE: `check_abort` decorator -def check_abort(fn): - - def wrapper(self, *args, **kwargs): - self.check_abort() - return fn(self, *args, **kwargs) - - return wrapper - -class Hoster(Plugin): +class Hoster(Base): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.31" + __version__ = "0.32" __status__ = "testing" __pattern__ = r'^unmatchable$' + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("fallback_premium", "bool", "Fallback to free download if premium fails", True), + ("chk_filesize" , "bool", "Check file size" , True)] __description__ = """Base hoster plugin""" __license__ = "GPLv3" @@ -65,107 +33,39 @@ class Hoster(Plugin): def __init__(self, pyfile): - self._init(pyfile.m.core) - - #: Engage wan reconnection - self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10 + super(Base, self).__init__(pyfile) #: Enable simultaneous processing of multiple downloads - self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10 self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 - #: time.time() + wait in seconds - self.wait_until = 0 - self.waiting = False - - #: Account handler instance, see :py:class:`Account` - self.account = None - self.user = None #@TODO: Remove in 0.4.10 - - #: Associated pyfile instance, see `PyFile` - self.pyfile = pyfile - - self.thread = None #: Holds thread in future - #: Location where the last call to download was saved - self.last_download = "" + self.last_download = None #: Re match of the last call to `checkDownload` self.last_check = None - #: Js engine, see `JsEngine` - self.js = self.pyload.js - - #: Captcha stuff - self.captcha = Captcha(self) - - #: Some plugins store html code here - self.html = None - - #: Dict of the amount of retries already made - self.retries = {} - self.force_free = False #@TODO: Recheck in 0.4.10 + #: Restart flag + self.rst_free = False #@TODO: Recheck in 0.4.10 self._setup() self.init() - def _log(self, level, plugintype, pluginname, messages): - log = getattr(self.pyload.log, level) - msg = u" | ".join(decode(a).strip() for a in messages if a) - log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" - % {'plugintype': plugintype.upper(), - 'pluginname': pluginname, - 'id' : self.pyfile.id, - 'msg' : msg}) - - - @classmethod - def get_info(cls, url="", html=""): - url = fixurl(url, unquote=True) - return {'name' : parse_name(url), - 'size' : 0, - 'status': 3 if url else 8, - 'url' : url} - - - def init(self): - """ - Initialize the plugin (in addition to `__init__`) - """ - pass - - - def setup(self): - """ - Setup for enviroment and other things, called before downloading (possibly more than one time) - """ - pass + def _setup(self): + super(Base, self)._setup() + self.last_download = None + self.last_check = None + self.rst_free = False - def _setup(self): - #@TODO: Remove in 0.4.10 - self.html = "" - self.last_download = "" - self.pyfile.error = "" - try: - self.req.close() - except Exception: - pass - - if self.account: - self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) - self.chunk_limit = -1 #: -1 for unlimited - self.resume_download = True - self.premium = self.account.premium + def load_account(self): + if self.rst_free: + self.account = False + self.user = None #@TODO: Remove in 0.4.10 else: - self.req = self.pyload.requestFactory.getRequest(self.__name__) - self.chunk_limit = 1 - self.resume_download = False - self.premium = False - - return self.setup() + super(Base, self).load_account() + # self.rst_free = False def _process(self, thread): @@ -174,229 +74,44 @@ class Hoster(Plugin): """ self.thread = thread - if self.force_free: - self.account = False - else: - self.load_account() #@TODO: Move to PluginThread in 0.4.10 - self.force_free = False - self._setup() # self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 - self.pyfile.setStatus("starting") - self.check_abort() - self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) - return self.process(self.pyfile) - - - #: Deprecated method, use `_process` instead (Remove in 0.4.10) - def preprocessing(self, *args, **kwargs): - return self._process(*args, **kwargs) - - - def load_account(self): - if not self.account: - self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - - if not self.account: - self.account = False - self.user = None #@TODO: Remove in 0.4.10 - - else: - self.account.choose() - self.user = self.account.user #@TODO: Remove in 0.4.10 - if self.account.user is None: - self.account = False - - - def process(self, pyfile): - """ - The 'main' method of every plugin, you **have to** overwrite it - """ - raise NotImplementedError - - - def set_reconnect(self, reconnect): - self.log_debug("RECONNECT %s required" % ("" if reconnect else "not"), - "Previous wantReconnect: %s" % self.wantReconnect) - self.wantReconnect = bool(reconnect) - - - def set_wait(self, seconds, reconnect=None): - """ - Set a specific wait time later used with `wait` - - :param seconds: wait time in seconds - :param reconnect: True if a reconnect would avoid wait time - """ - wait_time = max(int(seconds), 1) - wait_until = time.time() + wait_time + 1 - - self.log_debug("WAIT set to %d seconds" % wait_time, - "Previous waitUntil: %f" % self.pyfile.waitUntil) - - self.pyfile.waitUntil = wait_until - - if reconnect is not None: - self.set_reconnect(reconnect) - - - def wait(self, seconds=None, reconnect=None): - """ - Waits the time previously set - """ - pyfile = self.pyfile - - if seconds is not None: - self.set_wait(seconds) - - if reconnect is not None: - self.set_reconnect(reconnect) - - self.waiting = True - - status = pyfile.status #@NOTE: Recheck in 0.4.10 - pyfile.setStatus("waiting") - - self.log_info(_("Waiting %d seconds...") % pyfile.waitUntil - time.time()) - - if self.wantReconnect: - self.log_info(_("Requiring reconnection...")) - if self.account: - self.log_warning("Ignore reconnection due logged account") - - if not self.wantReconnect or self.account: - while pyfile.waitUntil > time.time(): - self.check_abort() - time.sleep(2) - - else: - while pyfile.waitUntil > time.time(): - self.check_abort() - self.thread.m.reconnecting.wait(1) - - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - time.sleep(2) - - self.waiting = False - pyfile.status = status #@NOTE: Recheck in 0.4.10 - - - def skip(self, msg=""): - """ - Skip and give msg - """ - raise Skip(encode(msg or self.pyfile.error or self.pyfile.pluginname)) #@TODO: Remove `encode` in 0.4.10 - - - #@TODO: Remove in 0.4.10 - def fail(self, msg): - """ - Fail and give msg - """ - msg = msg.strip() - - if msg: - self.pyfile.error = msg - else: - msg = self.pyfile.error or (self.info['error'] if 'error' in self.info else self.pyfile.getStatusName()) - - raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - - - def error(self, msg="", type=_("Parse")): - type = _("%s error") % type.strip().capitalize() if type else _("Unknown") - msg = _("%(type)s: %(msg)s | Plugin may be out of date" - % {'type': type, 'msg': msg or self.pyfile.error}) - - self.fail(msg) - - - def abort(self, msg=""): - """ - Abort and give msg - """ - if msg: #@TODO: Remove in 0.4.10 - self.pyfile.error = encode(msg) - - raise Abort - - - #@TODO: Recheck in 0.4.10 - def offline(self, msg=""): - """ - Fail and indicate file is offline - """ - self.fail("offline") - - - #@TODO: Recheck in 0.4.10 - def temp_offline(self, msg=""): - """ - Fail and indicates file ist temporary offline, the core may take consequences - """ - self.fail("temp. offline") - + self.pyfile.setStatus("starting") - def retry(self, attemps=5, delay=1, msg=""): - """ - Retries and begin again from the beginning + try: + self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) #@TODO: Remove in 0.4.10 + self.process(self.pyfile) - :param attemps: number of maximum retries - :param delay: time to wait in seconds - :param msg: msg for retrying, will be passed to fail if attemps value was reached - """ - id = inspect.currentframe().f_back.f_lineno - if id not in self.retries: - self.retries[id] = 0 + self.check_abort() - if 0 < attemps <= self.retries[id]: - self.fail(msg or _("Max retries reached")) + self.log_debug("CHECK DOWNLOAD") #@TODO: Recheck in 0.4.10 + self._check_download() - self.wait(delay, False) + except Fail, e: #@TODO: Move to PluginThread in 0.4.10 + if self.get_config('fallback_premium', True) and self.premium: + self.log_warning(_("Premium download failed"), e) + self.restart() - self.retries[id] += 1 - raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 + else: + raise Fail(e) - def restart(self, msg=None, nopremium=False): + def restart(self, msg="", premium=False): if not msg: - msg = _("Fallback to free download") if nopremium else _("Restart") + msg = _("Simple restart") if premium else _("Fallback to free download") - if nopremium: + if not premium: if self.premium: - self.force_free = True + self.rst_free = True else: self.fail("%s | %s" % (msg, _("Download was already free"))) raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def fixurl(self, url, baseurl=None, unquote=None): - url = fixurl(url) - - if not baseurl: - baseurl = fixurl(self.pyfile.url) - - if not urlparse.urlparse(url).scheme: - url_p = urlparse.urlparse(baseurl) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - url = urlparse.urljoin(baseurl, url) - - return fixurl(url, unquote) - - - @check_abort - def load(self, *args, **kwargs): - return super(Hoster, self).load(*args, **kwargs) - - @check_abort def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ @@ -415,7 +130,7 @@ class Hoster(Plugin): self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) - url = self.fixurl(url, unquote=True) + url = self.fixurl(url) self.pyfile.name = parse_name(self.pyfile.name) #: Safe check @@ -475,27 +190,7 @@ class Hoster(Plugin): self.last_download = filename - return self.last_download - - - def check_abort(self): - if not self.pyfile.abort: - return - - if self.pyfile.status is 8: - self.fail() - - elif self.pyfile.status is 4: - self.skip(self.pyfile.statusname) - - elif self.pyfile.status is 1: - self.offline() - - elif self.pyfile.status is 6: - self.temp_offline() - - else: - self.abort() + return filename def check_filesize(self, file_size, size_tolerance=1024): @@ -537,7 +232,7 @@ class Hoster(Plugin): :return: dictionary key of the first rule that matched """ do_delete = False - last_download = fs_encode(self.last_download) + last_download = fs_encode(self.last_download) #@TODO: Recheck in 0.4.10 if not self.last_download or not exists(last_download): self.fail(self.pyfile.error or _("No file downloaded")) @@ -558,7 +253,7 @@ class Hoster(Plugin): elif hasattr(rule, "search"): m = rule.search(content) - if m: + if m is not None: do_delete = True self.last_check = m return name @@ -575,11 +270,25 @@ class Hoster(Plugin): self.last_download = "" #: Recheck in 0.4.10 + def _check_download(self): + if self.captcha.task and not self.last_download: + self.retry_captcha() + + elif self.check_file({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, + delete=True): + self.error(_("Empty file")) + + elif self.get_config('chk_filesize', False) and self.info.get('size'): + # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size + # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB + self.check_filesize(self.info['size'], size_tolerance=10485760) + + def check_traffic(self): if not self.account: return True - traffic = self.account.get_data(refresh=True)['trafficleft'] + traffic = self.account.get_data('trafficleft') if traffic is None: return False @@ -632,96 +341,3 @@ class Hoster(Plugin): def checkForSameFiles(self, *args, **kwargs): if self.pyload.config.get("download", "skip_existing"): return self.check_filedupe() - - - def direct_link(self, url, follow_location=None): - link = "" - - if follow_location is None: - redirect = 1 - - elif type(follow_location) is int: - redirect = max(follow_location, 1) - - else: - redirect = self.get_config("maxredirs", 10, "UserAgentSwitcher") - - for i in xrange(redirect): - try: - self.log_debug("Redirect #%d to: %s" % (i, url)) - header = self.load(url, just_header=True) - - except Exception: #: Bad bad bad... rewrite this part in 0.4.10 - res = self.load(url, - just_header=True, - req=self.pyload.requestFactory.getRequest(self.__name__)) - - header = {'code': req.code} - for line in res.splitlines(): - line = line.strip() - if not line or ":" not in line: - continue - - key, none, value = line.partition(":") - key = key.lower().strip() - value = value.strip() - - if key in header: - if type(header[key]) is list: - header[key].append(value) - else: - header[key] = [header[key], value] - else: - header[key] = value - - if 'content-disposition' in header: - link = url - - elif header.get('location'): - location = self.fixurl(header['location'], url) - - if header.get('code') == 302: - link = location - - if follow_location: - url = location - continue - - else: - extension = os.path.splitext(parse_name(url))[-1] - - if header.get('content-type'): - mimetype = header['content-type'].split(';')[0].strip() - - elif extension: - mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" - - else: - mimetype = "" - - if mimetype and (link or 'html' not in mimetype): - link = url - else: - link = "" - - break - - else: - try: - self.log_error(_("Too many redirects")) - - except Exception: - pass - - return link - - - def parse_html_form(self, attr_str="", input_names={}): - return parse_html_form(attr_str, self.html, input_names) - - - def get_password(self): - """ - Get the password the user provided in the package - """ - return self.pyfile.package().password or "" diff --git a/module/plugins/internal/MultiCrypter.py b/module/plugins/internal/MultiCrypter.py index ae8785116..9d4ac3ab9 100644 --- a/module/plugins/internal/MultiCrypter.py +++ b/module/plugins/internal/MultiCrypter.py @@ -10,8 +10,9 @@ class MultiCrypter(SimpleCrypter): __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), - ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] __description__ = """Multi decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index fbfab1ade..d7d3c5ccd 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,12 +9,14 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.51" + __version__ = "0.52" __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("use_premium" , "bool", "Use premium account if available" , True), - ("revertfailed", "bool", "Revert to standard download if fails", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("fallback_premium", "bool", "Fallback to free download if premium fails", True), + ("chk_filesize" , "bool", "Check file size" , True), + ("revertfailed" , "bool", "Revert to standard download if fails" , True)] __description__ = """Multi hoster plugin""" __license__ = "GPLv3" @@ -100,7 +102,7 @@ class MultiHoster(SimpleHoster): except Fail, e: #@TODO: Move to PluginThread in 0.4.10 if self.premium: self.log_warning(_("Premium download failed")) - self.restart(nopremium=True) + self.restart() elif self.get_config("revertfailed", True) \ and "new_module" in self.pyload.pluginManager.hosterPlugins[self.__name__]: @@ -116,7 +118,7 @@ class MultiHoster(SimpleHoster): hdict['new_module'] = tmp_module hdict['new_name'] = tmp_name - self.restart(_("Revert to original hoster plugin")) + self.restart(_("Revert to original hoster plugin"), premium=True) else: raise Fail(encode(e)) #@TODO: Remove `encode` in 0.4.10 diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 6f3e52962..55574aae0 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -76,11 +76,6 @@ def parse_name(string): return urllib.unquote(name) -#@TODO: Move to utils in 0.4.10 -def timestamp(): - return int(time.time() * 1000) - - #@TODO: Move to utils in 0.4.10 def which(program): """ @@ -184,7 +179,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.38" + __version__ = "0.40" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -206,9 +201,10 @@ class Plugin(object): def _init(self, core): - self.pyload = core - self.info = {} #: Provide information in dict here - self.req = None #: Browser instance, see `network.Browser` + self.pyload = core + self.info = {} #: Provide information in dict here + self.req = None #: Browser instance, see `network.Browser` + self.last_html = None def init(self): @@ -355,7 +351,7 @@ class Plugin(object): self.log_debug("LOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) - url = fixurl(url, unquote=True) #: Recheck in 0.4.10 + url = fixurl(url) #: Recheck in 0.4.10 if req is None: req = self.req or self.pyload.requestFactory.getRequest(self.__name__) @@ -364,15 +360,17 @@ class Plugin(object): if isinstance(cookies, list): set_cookies(req.cj, cookies) - res = req.load(url, get, post, ref, bool(cookies), just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 + html = req.load(url, get, post, ref, bool(cookies), just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 #@TODO: Move to network in 0.4.10 if decode: - res = html_unescape(res) + html = html_unescape(html) #@TODO: Move to network in 0.4.10 if isinstance(decode, basestring): - res = _decode(res, decode) #@NOTE: Use `utils.decode()` in 0.4.10 + html = _decode(html, decode) #@NOTE: Use `utils.decode()` in 0.4.10 + + self.last_html = html if self.pyload.debug: frame = inspect.currentframe() @@ -383,15 +381,18 @@ class Plugin(object): with open(framefile, "wb") as f: del frame #: Delete the frame or it wont be cleaned - f.write(encode(res)) + f.write(encode(html)) except IOError, e: self.log_error(e) - if just_header: - #: Parse header + if not just_header: + return html + + else: + #@TODO: Move to network in 0.4.10 header = {'code': req.code} - for line in res.splitlines(): + for line in html.splitlines(): line = line.strip() if not line or ":" not in line: continue @@ -407,14 +408,13 @@ class Plugin(object): header[key] = [header[key], value] else: header[key] = value - res = header - return res + return header def clean(self): """ - Clean everything and remove references + Remove references """ try: self.req.clearCookies() @@ -423,6 +423,5 @@ class Plugin(object): except Exception: pass - for attr in ("account", "html", "pyfile", "req", "thread"): - if hasattr(self, attr): - setattr(self, attr, None) + else: + self.req = None diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 53d69b2a5..d4ae85ebe 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -13,8 +13,9 @@ class SimpleCrypter(Crypter, SimpleHoster): __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), - ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] __description__ = """Simple decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3bb3ff211..8ba227c92 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -16,12 +16,12 @@ from module.utils import fixup, fs_encode, parseFileSize as parse_size class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.88" + __version__ = "1.89" __status__ = "testing" __pattern__ = r'^unmatchable$' __config__ = [("use_premium" , "bool", "Use premium account if available" , True), - ("premium_fallback", "bool", "Fallback to free download if premium fails", True), + ("fallback_premium", "bool", "Fallback to free download if premium fails", True), ("chk_filesize" , "bool", "Check file size" , True)] __description__ = """Simple hoster plugin""" @@ -200,9 +200,6 @@ class SimpleHoster(Hoster): self.direct_dl = False self.leech_dl = False - if not self.get_config('use_premium', True) and self.premium: - self.restart(nopremium=True) - if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) @@ -243,94 +240,70 @@ class SimpleHoster(Hoster): def process(self, pyfile): - try: - self.prepare() - self.check_info() #@TODO: Remove in 0.4.10 - - if self.leech_dl: - self.log_info(_("Processing as debrid download...")) - self.handle_multi(pyfile) + self.prepare() + self.check_info() #@TODO: Remove in 0.4.10 - if not self.link and not was_downloaded(): - self.log_info(_("Failed to leech url")) - - else: - if not self.link and self.direct_dl and not self.last_download: - self.log_info(_("Looking for direct download link...")) - self.handle_direct(pyfile) + if self.leech_dl: + self.log_info(_("Processing as debrid download...")) + self.handle_multi(pyfile) - if self.link or self.last_download: - self.log_info(_("Direct download link detected")) - else: - self.log_info(_("Direct download link not found")) + if not self.link and not was_downloaded(): + self.log_info(_("Failed to leech url")) - if not self.link and not self.last_download: - self.preload() + else: + if not self.link and self.direct_dl and not self.last_download: + self.log_info(_("Looking for direct download link...")) + self.handle_direct(pyfile) - if 'status' not in self.info or self.info['status'] is 3: #@TODO: Recheck in 0.4.10 - self.check_info() + if self.link or self.last_download: + self.log_info(_("Direct download link detected")) + else: + self.log_info(_("Direct download link not found")) - if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): - self.log_info(_("Processing as premium download...")) - self.handle_premium(pyfile) + if not self.link and not self.last_download: + self.preload() - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()): - self.log_info(_("Processing as free download...")) - self.handle_free(pyfile) + if 'status' not in self.info or self.info['status'] is 3: #@TODO: Recheck in 0.4.10 + self.check_info() - if not self.last_download: - self.log_info(_("Downloading file...")) - self.download(self.link, disposition=self.DISPOSITION) + if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): + self.log_info(_("Processing as premium download...")) + self.handle_premium(pyfile) - self.check_download() + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()): + self.log_info(_("Processing as free download...")) + self.handle_free(pyfile) - except Fail, e: #@TODO: Move to PluginThread in 0.4.10 - if self.get_config('premium_fallback', True) and self.premium: - self.log_warning(_("Premium download failed"), e) - self.restart(nopremium=True) + if not self.last_download: + self.log_info(_("Downloading file...")) + self.download(self.link, disposition=self.DISPOSITION) - else: - raise Fail(encode(e)) #@TODO: Remove `encode` in 0.4.10 + self.check_download() def check_download(self): self.log_info(_("Checking downloaded file...")) + self.log_debug("Using default check rules...") + for r, p in self.FILE_ERRORS: + errmsg = self.check_file({r: re.compile(p)}) + if errmsg is not None: + errmsg = errmsg.strip().capitalize() - if self.captcha.task and not self.last_download: - self.captcha.invalid() - self.retry(10, msg=_("Wrong captcha")) + try: + errmsg += " | " + self.last_check.group(1).strip() - elif self.check_file({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, - delete=True): - self.error(_("Empty file")) + except Exception: + pass + self.log_warning(_("Check result: ") + errmsg, _("Waiting 1 minute and retry")) + self.wantReconnect = True + self.retry(wait=60, msg=errmsg) else: - if self.get_config('chk_filesize', False) and self.info.get('size'): - # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size - # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB - self.check_filesize(self.info['size'], size_tolerance=10485760) - - self.log_debug("Using default check rules...") - for r, p in self.FILE_ERRORS: - errmsg = self.check_file({r: re.compile(p)}) - if errmsg is not None: - errmsg = errmsg.strip().capitalize() - - try: - errmsg += " | " + self.last_check.group(1).strip() - - except Exception: - pass - - self.log_warning(_("Check result: ") + errmsg, _("Waiting 1 minute and retry")) - self.wantReconnect = True - self.retry(delay=60, msg=errmsg) - else: - if self.CHECK_FILE: - self.log_debug("Using custom check rules...") - with open(fs_encode(self.last_download), "rb") as f: - self.html = f.read(1048576) #@TODO: Recheck in 0.4.10 - self.check_errors() + if self.CHECK_FILE: + self.log_debug("Using custom check rules...") + with open(fs_encode(self.last_download), "rb") as f: + self.html = f.read(1048576) #@TODO: Recheck in 0.4.10 + self.check_errors() self.log_info(_("No errors found")) @@ -375,7 +348,7 @@ class SimpleHoster(Hoster): if hasattr(self, 'ERROR_PATTERN'): m = re.search(self.ERROR_PATTERN, self.html) - if m: + if m is not None: try: errmsg = m.group(1).strip() @@ -399,8 +372,7 @@ class SimpleHoster(Hoster): self.fail(_("Connection from your current IP address is not allowed")) elif re.search('captcha|code', errmsg, re.I): - self.captcha.invalid() - self.retry(10, msg=_("Wrong captcha")) + self.retry_captcha() elif re.search('countdown|expired', errmsg, re.I): self.retry(10, 60, _("Link expired")) @@ -415,18 +387,18 @@ class SimpleHoster(Hoster): self.offline() elif re.search('filename', errmsg, re.I): - self.fail(_("Wrong url")) + self.fail(_("Invalid url")) elif re.search('premium', errmsg, re.I): self.fail(_("File can be downloaded by premium users only")) else: self.wantReconnect = True - self.retry(delay=60, msg=errmsg) + self.retry(wait=60, msg=errmsg) elif hasattr(self, 'WAIT_PATTERN'): m = re.search(self.WAIT_PATTERN, self.html) - if m: + if m is not None: try: waitmsg = m.group(1).strip() diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 3b92a191b..7b9410222 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -13,7 +13,7 @@ from module.plugins.internal.Plugin import parse_html_form, set_cookie class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.48" + __version__ = "0.49" __status__ = "testing" __description__ = """XFileSharing account plugin""" @@ -42,6 +42,16 @@ class XFSAccount(Account): LOGIN_SKIP_PATTERN = r'op=logout' + def set_xfs_cookie(self): + if not self.COOKIES: + return + + if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: + self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) + else: + set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") + + def grab_info(self, user, password, data): validuntil = None trafficleft = None @@ -146,11 +156,7 @@ class XFSAccount(Account): if not self.PLUGIN_URL: self.PLUGIN_URL = "http://www.%s/" % self.PLUGIN_DOMAIN - if self.COOKIES: - if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: - self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) - else: - set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") + self.set_xfs_cookie() if not self.PLUGIN_URL: self.fail_login(_("Missing PLUGIN_URL")) diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 8047fd03e..69565e22c 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -7,10 +7,13 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.15" + __version__ = "0.16" __status__ = "testing" __pattern__ = r'^unmatchable$' + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] __description__ = """XFileSharing decrypter plugin""" __license__ = "GPLv3" @@ -27,6 +30,15 @@ class XFSCrypter(SimpleCrypter): OFFLINE_PATTERN = r'>\s*(No such user|\w+ (Not Found|file (was|has been) removed|no longer available)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' + def set_xfs_cookie(self): + if not self.COOKIES: + return + + if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: + self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) + else: + set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") + def prepare(self): if not self.PLUGIN_DOMAIN: @@ -41,10 +53,6 @@ class XFSCrypter(SimpleCrypter): else: self.fail(_("Missing PLUGIN_DOMAIN")) - if self.COOKIES: - if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: - self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) - else: - set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") + self.set_xfs_cookie() return super(XFSCrypter, self).prepare() diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 98ed4e016..ce87642f0 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,10 +14,13 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.61" + __version__ = "0.62" __status__ = "testing" __pattern__ = r'^unmatchable$' + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("fallback_premium", "bool", "Fallback to free download if premium fails", True), + ("chk_filesize" , "bool", "Check file size" , True)] __description__ = """XFileSharing hoster plugin""" __license__ = "GPLv3" @@ -57,6 +60,16 @@ class XFSHoster(SimpleHoster): self.resume_download = self.multiDL = self.premium + def set_xfs_cookie(self): + if not self.COOKIES: + return + + if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: + self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) + else: + set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") + + def prepare(self): """ Initialize important variables @@ -72,11 +85,7 @@ class XFSHoster(SimpleHoster): else: self.fail(_("Missing PLUGIN_DOMAIN")) - if self.COOKIES: - if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: - self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) - else: - set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") + self.set_xfs_cookie() if not self.LINK_PATTERN: pattern = r'(?:file: "(.+?)"|(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<])' -- cgit v1.2.3 From ab3c90a7b0d18f47d3efc8259906895c1f50765d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 1 Oct 2015 23:21:14 +0200 Subject: Fixpack (1) --- module/plugins/internal/Base.py | 16 ++++-- module/plugins/internal/Crypter.py | 6 +-- module/plugins/internal/Hoster.py | 17 ++---- module/plugins/internal/Plugin.py | 19 ++++--- module/plugins/internal/SimpleCrypter.py | 41 ++++++++++++--- module/plugins/internal/SimpleHoster.py | 89 +++++++++++++++++++------------- module/plugins/internal/XFSCrypter.py | 1 + module/plugins/internal/XFSHoster.py | 6 +-- 8 files changed, 125 insertions(+), 70 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Base.py b/module/plugins/internal/Base.py index 4235cf94d..843d2ad11 100644 --- a/module/plugins/internal/Base.py +++ b/module/plugins/internal/Base.py @@ -18,12 +18,22 @@ def getInfo(urls): pass +#@TODO: Remove in 0.4.10 +def parse_fileInfo(klass, url="", html=""): + info = klass.get_info(url, html) + return encode(info['name']), info['size'], info['status'], info['url'] + + #@TODO: Remove in 0.4.10 def create_getInfo(klass): def get_info(urls): for url in urls: - if hasattr(klass, "URL_REPLACEMENTS"): + try: url = replace_patterns(url, klass.URL_REPLACEMENTS) + + except Exception: + pass + yield parse_fileInfo(klass, url) return get_info @@ -42,7 +52,7 @@ def check_abort(fn): class Base(Plugin): __name__ = "Base" __type__ = "base" - __version__ = "0.01" + __version__ = "0.02" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -366,7 +376,7 @@ class Base(Plugin): @check_abort def load(self, *args, **kwargs): - return super(Hoster, self).load(*args, **kwargs) + return super(Base, self).load(*args, **kwargs) def check_abort(self): diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 069ffb589..a5c88aed9 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -7,7 +7,7 @@ from module.utils import save_path as safe_filename class Crypter(Base): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.10" + __version__ = "0.11" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -21,7 +21,7 @@ class Crypter(Base): def __init__(self, pyfile): - super(Base, self).__init__(pyfile) + super(Crypter, self).__init__(pyfile) #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) self.packages = [] @@ -34,7 +34,7 @@ class Crypter(Base): def _setup(self): - super(Base, self)._setup() + super(Crypter, self)._setup() self.packages = [] self.urls = [] diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index d1b894c6f..cb7b215ba 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -4,22 +4,15 @@ from __future__ import with_statement import os -from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo +from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo, parse_fileInfo from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename -#@TODO: Remove in 0.4.10 -def parse_fileInfo(klass, url="", html=""): - info = klass.get_info(url, html) - return encode(info['name']), info['size'], info['status'], info['url'] - - - class Hoster(Base): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.32" + __version__ = "0.33" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -33,7 +26,7 @@ class Hoster(Base): def __init__(self, pyfile): - super(Base, self).__init__(pyfile) + super(Hoster, self).__init__(pyfile) #: Enable simultaneous processing of multiple downloads self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 @@ -52,7 +45,7 @@ class Hoster(Base): def _setup(self): - super(Base, self)._setup() + super(Hoster, self)._setup() self.last_download = None self.last_check = None @@ -64,7 +57,7 @@ class Hoster(Base): self.account = False self.user = None #@TODO: Remove in 0.4.10 else: - super(Base, self).load_account() + super(Hoster, self).load_account() # self.rst_free = False diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index dbf6775cb..5725bfa58 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -6,6 +6,7 @@ import datetime import inspect import os import re +import time import traceback import urllib import urlparse @@ -76,6 +77,11 @@ def parse_name(string): return urllib.unquote(name) +#@TODO: Move to utils in 0.4.10 +def timestamp(): + return int(time.time() * 1000) + + #@TODO: Move to utils in 0.4.10 def which(program): """ @@ -145,7 +151,10 @@ def parse_html_form(attr_str, html, input_names={}): else: inputs[name] = value - if input_names: + if not input_names: + #: No attribute check + return action, inputs + else: #: Check input attributes for key, val in input_names.items(): if key in inputs: @@ -155,14 +164,12 @@ def parse_html_form(attr_str, html, input_names={}): continue elif hasattr(val, "search") and re.match(val, inputs[key]): continue - break #: Attibute value does not match + else: + break #: Attibute value does not match else: break #: Attibute name does not match else: return action, inputs #: Passed attribute check - else: - #: No attribute check - return action, inputs return {}, None #: No matching form found @@ -179,7 +186,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.40" + __version__ = "0.41" __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index d4ae85ebe..20e91ac1e 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -3,13 +3,14 @@ import re from module.plugins.internal.Crypter import Crypter -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookie, set_cookies +from module.plugins.internal.Plugin import replace_patterns, set_cookie, set_cookies +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.63" + __version__ = "0.64" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -47,13 +48,35 @@ class SimpleCrypter(Crypter, SimpleHoster): and its loadPage method: - def load_page(self, page_n): return the html of the page number page_n """ - DIRECT_LINK = True - LEECH_HOSTER = False + NAME_REPLACEMENTS = [] + URL_REPLACEMENTS = [] + + COOKIES = True #: or False or list of tuples [(domain, name, value)] + DIRECT_LINK = True #: Set to True to looking for direct link (as defined in handle_direct method), set to None to do it if self.account is True else False + LOGIN_ACCOUNT = False #: Set to True to require account login + LOGIN_PREMIUM = False #: Set to True to require premium account login + # LEECH_HOSTER = False #: Set to True to leech other hoster link (as defined in handle_multi method) + TEXT_ENCODING = True #: Set to encoding name if encoding value in http header is not correct + PAGES_PATTERN = None + + LINK_PATTERN = None + + NAME_PATTERN = None + HASHSUM_PATTERN = None + OFFLINE_PATTERN = None + TEMP_OFFLINE_PATTERN = None + + WAIT_PATTERN = None + PREMIUM_ONLY_PATTERN = None + HAPPY_HOUR_PATTERN = None + IP_BLOCKED_PATTERN = None + DL_LIMIT_PATTERN = None + SIZE_LIMIT_PATTERN = None + ERROR_PATTERN = None #@TODO: Remove in 0.4.10 @@ -113,7 +136,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.links = self.get_links() or list() - if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): + if self.PAGES_PATTERN: self.handle_pages(pyfile) self.log_debug("Package has %d links" % len(self.links)) @@ -134,11 +157,15 @@ class SimpleCrypter(Crypter, SimpleHoster): return re.findall(self.LINK_PATTERN, self.html) + def load_page(self, number) + raise NotImplementedError + + def handle_pages(self, pyfile): try: pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) - except (AttributeError, IndexError, ValueError): + except Exception: pages = 1 for p in xrange(2, pages + 1): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 8ba227c92..5d2307d51 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -16,7 +16,7 @@ from module.utils import fixup, fs_encode, parseFileSize as parse_size class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.89" + __version__ = "1.90" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -85,25 +85,43 @@ class SimpleHoster(Hoster): LINK_PREMIUM_PATTERN: (optional) group(1) should be the direct link for premium download example: LINK_PREMIUM_PATTERN = r'
)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)'), - ('Request error', r'([Aa]n error occured while processing your request)' ), - ('Html file' , r'\A\s*)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)'), + ('Request error', r'([Aa]n error occured while processing your request)' ), + ('Html file' , r'\A\s* 300 self.retry(1, wait_time, _("Download limit exceeded")) - if hasattr(self, 'HAPPY_HOUR_PATTERN') and re.search(self.HAPPY_HOUR_PATTERN, self.html): + if self.HAPPY_HOUR_PATTERN and re.search(self.HAPPY_HOUR_PATTERN, self.html): self.multiDL = True - if hasattr(self, 'ERROR_PATTERN'): + if self.ERROR_PATTERN: m = re.search(self.ERROR_PATTERN, self.html) if m is not None: try: @@ -396,7 +414,7 @@ class SimpleHoster(Hoster): self.wantReconnect = True self.retry(wait=60, msg=errmsg) - elif hasattr(self, 'WAIT_PATTERN'): + elif self.WAIT_PATTERN: m = re.search(self.WAIT_PATTERN, self.html) if m is not None: try: @@ -496,7 +514,7 @@ class SimpleHoster(Hoster): def handle_free(self, pyfile): - if not hasattr(self, 'LINK_FREE_PATTERN'): + if not self.LINK_FREE_PATTERN: self.log_error(_("Free download not implemented")) m = re.search(self.LINK_FREE_PATTERN, self.html) @@ -507,10 +525,9 @@ class SimpleHoster(Hoster): def handle_premium(self, pyfile): - if not hasattr(self, 'LINK_PREMIUM_PATTERN'): + if not self.LINK_PREMIUM_PATTERN: self.log_error(_("Premium download not implemented")) - self.log_info(_("Processing as free download...")) - self.handle_free(pyfile) + self.restart() m = re.search(self.LINK_PREMIUM_PATTERN, self.html) if m is None: diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 69565e22c..876575874 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -30,6 +30,7 @@ class XFSCrypter(SimpleCrypter): OFFLINE_PATTERN = r'>\s*(No such user|\w+ (Not Found|file (was|has been) removed|no longer available)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' + def set_xfs_cookie(self): if not self.COOKIES: return diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index dcb85d23b..b59f34122 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.62" + __version__ = "0.63" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -253,7 +253,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) - except (AttributeError, IndexError, TypeError): + except Exception: captcha_key = recaptcha.detect_key() else: @@ -267,7 +267,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) - except (AttributeError, IndexError, TypeError): + except Exception: captcha_key = solvemedia.detect_key() else: -- cgit v1.2.3 From 4e6234d8530c513ae37307f57d65c8c2b56efb1f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 2 Oct 2015 00:26:01 +0200 Subject: [Plugin] Improve fixurl function https://github.com/pyload/pyload/issues/1873 --- module/plugins/internal/Plugin.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 5725bfa58..4546186d4 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -61,9 +61,13 @@ def fixurl(url, unquote=None): if unquote is None: unquote = newurl == url - newurl = html_unescape(newurl.decode('unicode-escape')).strip() + newurl = html_unescape(newurl.decode('unicode-escape')) + newurl = re.sub(r'/{2,}', '/', newurl).strip().lstrip('.') - return newurl if unquote else urllib.quote(newurl) + if not unquote: + newurl = urllib.quote(newurl) + + return newurl #@TODO: Recheck in 0.4.10 @@ -186,7 +190,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.41" + __version__ = "0.42" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 7adfcbe536339bb8c51024573cad346cac7e0eda Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 2 Oct 2015 01:37:12 +0200 Subject: [Hoster] Fix https://github.com/pyload/pyload/issues/1918 --- module/plugins/internal/Hoster.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index cb7b215ba..ea5a3cb29 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -2,6 +2,7 @@ from __future__ import with_statement +import re import os from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo, parse_fileInfo @@ -12,7 +13,7 @@ from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path a class Hoster(Base): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.33" + __version__ = "0.34" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 67a21410eeb8a2bd7bc4d71f7fe5cd68312b702b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 2 Oct 2015 02:34:47 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1917 + new function: parse_time --- module/plugins/internal/Hoster.py | 2 +- module/plugins/internal/Plugin.py | 46 ++++++++++++++++++++++++++++++--- module/plugins/internal/SimpleHoster.py | 21 ++++----------- 3 files changed, 48 insertions(+), 21 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index ea5a3cb29..26da436a5 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -2,8 +2,8 @@ from __future__ import with_statement -import re import os +import re from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo, parse_fileInfo from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 4546186d4..24cff70c3 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -54,7 +54,6 @@ def exists(path): return False -#@TODO: Recheck in 0.4.10 def fixurl(url, unquote=None): newurl = urllib.unquote(url) @@ -62,7 +61,7 @@ def fixurl(url, unquote=None): unquote = newurl == url newurl = html_unescape(newurl.decode('unicode-escape')) - newurl = re.sub(r'/{2,}', '/', newurl).strip().lstrip('.') + newurl = re.sub(r'[^:]/{2,}', '/', newurl).strip().lstrip('.') if not unquote: newurl = urllib.quote(newurl) @@ -70,7 +69,6 @@ def fixurl(url, unquote=None): return newurl -#@TODO: Recheck in 0.4.10 def parse_name(string): path = fixurl(decode(string), unquote=False) url_p = urlparse.urlparse(path.rstrip('/')) @@ -81,6 +79,46 @@ def parse_name(string): return urllib.unquote(name) +#@TODO: Move to utils in 0.4.10 +def str2int(string): + try: + return int(string) + except: + pass + + ones = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", + "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", + "sixteen", "seventeen", "eighteen", "nineteen"] + tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", + "eighty", "ninety"] + + o_tuple = [(w, i) for i, w in enumerate(ones)] + t_tuple = [(w, i * 10) for i, w in enumerate(tens)] + + numwords = dict(o_tuple + t_tuple) + tokens = re.split(r"[\s-]+", string.lower()) + + try: + return sum(numwords[word] for word in tokens) + except: + return 0 + + +def parse_time(string): + if re.search("da(il)?y|today", string): + time = seconds_to_midnight() + + else: + this = re.compile("this", re.I) + regex = re.compile(r'(\d+|\w+)\s*(hr|hour|min|sec|)', re.I) + + time = sum(1 if this.match(v) else str2int(v) * + {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, '': 1}[u.lower()] + for v, u in regex.findall(string)) + + return time + + #@TODO: Move to utils in 0.4.10 def timestamp(): return int(time.time() * 1000) @@ -190,7 +228,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.42" + __version__ = "0.43" __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 5d2307d51..3a3e44619 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -9,14 +9,14 @@ import time from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, encode, parse_name, replace_patterns, seconds_to_midnight, set_cookie, set_cookies +from module.plugins.internal.Plugin import Fail, encode, parse_name, parse_time, replace_patterns, seconds_to_midnight, set_cookie, set_cookies from module.utils import fixup, fs_encode, parseFileSize as parse_size class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.90" + __version__ = "1.91" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -352,12 +352,7 @@ class SimpleHoster(Hoster): self.info['error'] = re.sub(r'<.*?>', " ", errmsg) self.log_warning(self.info['error']) - if re.search('da(il)?y|today', errmsg, re.I): - wait_time = seconds_to_midnight() - else: - wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) - + wait_time = parse_time(errmsg) self.wantReconnect = wait_time > 300 self.retry(1, wait_time, _("Download limit exceeded")) @@ -377,12 +372,7 @@ class SimpleHoster(Hoster): self.log_warning(self.info['error']) if re.search('limit|wait|slot', errmsg, re.I): - if re.search("da(il)?y|today", errmsg): - wait_time = seconds_to_midnight() - else: - wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) - + wait_time = parse_time(errmsg) self.wantReconnect = wait_time > 300 self.retry(1, wait_time, _("Download limit exceeded")) @@ -423,8 +413,7 @@ class SimpleHoster(Hoster): except (AttributeError, IndexError): waitmsg = m.group(0).strip() - wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec|)', waitmsg, re.I)) + wait_time = parse_time(waitmsg) self.wait(wait_time, wait_time > 300) self.info.pop('error', None) -- cgit v1.2.3 From 2105c775f01a31079c82f80c06f87beaa0354de6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 2 Oct 2015 03:40:35 +0200 Subject: [UptoboxCom] Fix https://github.com/pyload/pyload/issues/1530 and https://github.com/pyload/pyload/issues/1904 --- module/plugins/internal/Base.py | 2 +- module/plugins/internal/SimpleHoster.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Base.py b/module/plugins/internal/Base.py index 843d2ad11..bc9ef9158 100644 --- a/module/plugins/internal/Base.py +++ b/module/plugins/internal/Base.py @@ -355,7 +355,7 @@ class Base(Plugin): raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def retry_captcha(self, attemps=10, wait=1, msg=_("Wrong captcha")): + def retry_captcha(self, attemps=10, wait=1, msg=_("Max captcha retries reached")): self.captcha.invalid() self.retry(attemps, wait, msg) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3a3e44619..5d6892ec5 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -16,7 +16,7 @@ from module.utils import fixup, fs_encode, parseFileSize as parse_size class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.91" + __version__ = "1.92" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -353,8 +353,8 @@ class SimpleHoster(Hoster): self.log_warning(self.info['error']) wait_time = parse_time(errmsg) - self.wantReconnect = wait_time > 300 - self.retry(1, wait_time, _("Download limit exceeded")) + self.wait(wait_time, reconnect=wait_time > 300) + self.restart(_("Download limit exceeded"), premium=True) if self.HAPPY_HOUR_PATTERN and re.search(self.HAPPY_HOUR_PATTERN, self.html): self.multiDL = True @@ -373,8 +373,8 @@ class SimpleHoster(Hoster): if re.search('limit|wait|slot', errmsg, re.I): wait_time = parse_time(errmsg) - self.wantReconnect = wait_time > 300 - self.retry(1, wait_time, _("Download limit exceeded")) + self.wait(wait_time, reconnect=wait_time > 300) + self.restart(_("Download limit exceeded"), premium=True) elif re.search('country|ip|region|nation', errmsg, re.I): self.fail(_("Connection from your current IP address is not allowed")) @@ -414,7 +414,7 @@ class SimpleHoster(Hoster): waitmsg = m.group(0).strip() wait_time = parse_time(waitmsg) - self.wait(wait_time, wait_time > 300) + self.wait(wait_time, econnect=wait_time > 300) self.info.pop('error', None) -- cgit v1.2.3 From ab695d17397e101447102877cdd282d825051035 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 2 Oct 2015 04:22:19 +0200 Subject: [Account] Improve parse_traffic method + code cosmetics --- module/plugins/internal/Account.py | 14 ++++++-------- module/plugins/internal/OCR.py | 2 +- module/plugins/internal/Plugin.py | 10 +++++----- module/plugins/internal/SevenZip.py | 2 +- module/plugins/internal/SimpleHoster.py | 10 +++++----- module/plugins/internal/UnRar.py | 6 +++--- 6 files changed, 21 insertions(+), 23 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 123241be3..ad78403de 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -5,14 +5,14 @@ import time import threading from module.plugins.Plugin import SkipDownload as Skip -from module.plugins.internal.Plugin import Plugin -from module.utils import compare_time, lock, parseFileSize as parse_size +from module.plugins.internal.Plugin import Plugin, parse_size +from module.utils import compare_time, lock class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.52" + __version__ = "0.53" __status__ = "testing" __description__ = """Base account plugin""" @@ -391,11 +391,9 @@ class Account(Plugin): ########################################################################### - def parse_traffic(self, value, unit=None): #@NOTE: Returns kilobytes in 0.4.9 - if not isinstance(unit, basestring): - unit = "KB" - - return parse_size(value, unit) / 1024 #@TODO: Remove `/ 1024` in 0.4.10 + def parse_traffic(self, size, unit="KB"): #@NOTE: Returns kilobytes in 0.4.9 + size = re.search(r'(\d*[\.,]?\d+)', size).group(1) #@TODO: Recjeck in 0.4.10 + return parse_size(size, unit) / 1024 #@TODO: Remove `/ 1024` in 0.4.10 def fail_login(self, msg=_("Login handshake has failed")): diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 3e5afae69..884639b6b 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -94,7 +94,7 @@ class OCR(Plugin): self.pyload.log_debug("Saving tiff...") self.image.save(tmpTif.name, 'TIFF') - if os.name == "nt": + if os.name is "nt": tessparams = [os.path.join(pypath, "tesseract", "tesseract.exe")] else: tessparams = ["tesseract"] diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 24cff70c3..6d94ca1e4 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -11,12 +11,12 @@ import traceback import urllib import urlparse -if os.name != "nt": +if os.name is not "nt": import grp import pwd from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 -from module.utils import fs_encode, fs_decode, html_unescape, save_join as fs_join +from module.utils import fs_encode, fs_decode, html_unescape, parseFileSize as parse_size, save_join as fs_join #@TODO: Move to utils in 0.4.10 @@ -45,7 +45,7 @@ def encode(string, encoding='utf8'): #@TODO: Move to utils in 0.4.10 def exists(path): if os.path.exists(path): - if os.name == "nt": + if os.name is "nt": dir, name = os.path.split(path.rstrip(os.sep)) return name in os.listdir(dir) else: @@ -228,7 +228,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.43" + __version__ = "0.44" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -314,7 +314,7 @@ class Plugin(object): self.log_warning(_("Setting path mode failed"), e) try: - if os.name != "nt" and self.pyload.config.get("permission", "change_dl"): + if os.name is not "nt" and self.pyload.config.get("permission", "change_dl"): uid = pwd.getpwnam(self.pyload.config.get("permission", "user"))[2] gid = grp.getgrnam(self.pyload.config.get("permission", "group"))[2] os.chown(path, uid, gid) diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index 7ed64ecdf..f73e935e8 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -38,7 +38,7 @@ class SevenZip(UnRar): @classmethod def find(cls): try: - if os.name == "nt": + if os.name is "nt": cls.CMD = os.path.join(pypath, "7z.exe") p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 5d6892ec5..e5526a7bd 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -16,7 +16,7 @@ from module.utils import fixup, fs_encode, parseFileSize as parse_size class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.92" + __version__ = "1.93" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -314,8 +314,8 @@ class SimpleHoster(Hoster): pass self.log_warning(_("Check result: ") + errmsg, _("Waiting 1 minute and retry")) - self.wantReconnect = True - self.retry(wait=60, msg=errmsg) + self.wait(60, reconnect=True) + self.restart(errmsg, premium=True) else: if self.CHECK_FILE: self.log_debug("Using custom check rules...") @@ -401,8 +401,8 @@ class SimpleHoster(Hoster): self.fail(_("File can be downloaded by premium users only")) else: - self.wantReconnect = True - self.retry(wait=60, msg=errmsg) + self.wait(60, reconnect=True) + self.restart(errmsg, premium=True) elif self.WAIT_PATTERN: m = re.search(self.WAIT_PATTERN, self.html) diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 7efecc5ef..908689109 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -12,7 +12,7 @@ from module.utils import fs_decode, save_join as fs_join def renice(pid, value): - if value and os.name != "nt": + if value and os.name is not "nt": try: subprocess.Popen(["renice", str(value), str(pid)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=-1) @@ -49,7 +49,7 @@ class UnRar(Extractor): @classmethod def find(cls): try: - if os.name == "nt": + if os.name is "nt": cls.CMD = os.path.join(pypath, "RAR.exe") else: cls.CMD = "rar" @@ -61,7 +61,7 @@ class UnRar(Extractor): except OSError: try: - if os.name == "nt": + if os.name is "nt": cls.CMD = os.path.join(pypath, "UnRAR.exe") else: cls.CMD = "unrar" -- cgit v1.2.3