From fcdc1f730118050e390ad2d9fa337ce62809d9a6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 30 May 2015 15:54:38 +0200 Subject: [SimpleCrypter] Merge with SimpleDereferer --- module/plugins/internal/SimpleCrypter.py | 34 ++++++--- module/plugins/internal/SimpleDereferer.py | 107 ----------------------------- module/plugins/internal/SimpleHoster.py | 2 +- 3 files changed, 26 insertions(+), 117 deletions(-) delete mode 100644 module/plugins/internal/SimpleDereferer.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index b843a28f0..2d4d52f3c 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -11,7 +11,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.46" + __version__ = "0.47" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] @@ -59,7 +59,7 @@ class SimpleCrypter(Crypter, SimpleHoster): URL_REPLACEMENTS = [] TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] + COOKIES = True #: or False or list of tuples [(domain, name, value)] LOGIN_ACCOUNT = False LOGIN_PREMIUM = False @@ -99,18 +99,34 @@ class SimpleCrypter(Crypter, SimpleHoster): self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + def handleDirect(self, pyfile): + while True: + header = self.load(self.link or pyfile.url, just_header=True, decode=True) + if 'location' in header and header['location']: + self.link = header['location'] + else: + break + + def decrypt(self, pyfile): self.prepare() - self.preload() - self.checkInfo() + self.logDebug("Looking for link redirect...") + self.handleDirect(pyfile) + + if self.link: + self.urls = [self.link] + + else: + self.preload() + self.checkInfo() - self.links = self.getLinks() + self.links = self.getLinks() - if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): - self.handlePages(pyfile) + if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): + self.handlePages(pyfile) - self.logDebug("Package has %d links" % len(self.links)) + self.logDebug("Package has %d links" % len(self.links)) if self.links: self.packages = [(self.info['name'], self.links, self.info['folder'])] @@ -155,7 +171,7 @@ class SimpleCrypter(Crypter, SimpleHoster): links = [urlparse.urljoin(baseurl, link) if not urlparse.urlparse(link).scheme else link \ for link in re.findall(self.LINK_PATTERN, self.html)] - return [html_unescape(l.strip().decode('unicode-escape')) for l in links] + return [html_unescape(l.decode('unicode-escape').strip()) for l in links] def handlePages(self, pyfile): diff --git a/module/plugins/internal/SimpleDereferer.py b/module/plugins/internal/SimpleDereferer.py deleted file mode 100644 index 2e7e08321..000000000 --- a/module/plugins/internal/SimpleDereferer.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.Crypter import Crypter -from module.plugins.internal.SimpleHoster import create_getInfo, set_cookies -from module.utils import html_unescape - - -class SimpleDereferer(Crypter): - __name__ = "SimpleDereferer" - __type__ = "crypter" - __version__ = "0.14" - - __pattern__ = r'^unmatchable$' - __config__ = [] #@TODO: Remove in 0.4.10 - - __description__ = """Simple dereferer plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - """ - Following patterns should be defined by each crypter: - - LINK_PATTERN: Regex to catch the redirect url in group(1) - example: LINK_PATTERN = r'
Date: Mon, 1 Jun 2015 19:46:11 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1445 --- module/plugins/internal/SimpleHoster.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c333f34b0..222ba4b05 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.50" + __version__ = "1.51" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -474,7 +474,7 @@ class SimpleHoster(Hoster): if not self.link and not self.lastDownload: self.MULTI_HOSTER = False - self.retry(1, reason="Multi hoster fails") + self.retry(1, reason=_("Multi hoster fails")) if not self.link and not self.lastDownload: self.preload() @@ -635,9 +635,14 @@ class SimpleHoster(Hoster): elif re.search('up to', errmsg, re.I): self.fail(_("File too large for free download")) - elif re.search('offline|delet|remov|not (found|available)', errmsg, re.I): + elif re.search('offline|delet|remov|not? (found|(longer)? available)', errmsg, re.I): self.offline() + elif re.search('filename', errmsg, re.I): + url_p = urlparse.urlparse(self.pyfile.url) + self.pyfile.url = "%s://%s/%s" % (url_p.scheme, url_p.netloc, url_p.path.strip('/').split('/')[0]) + self.retry(1, reason=_("Wrong url")) + elif re.search('premium', errmsg, re.I): self.fail(_("File can be downloaded by premium users only")) -- cgit v1.2.3 From 2d2e8914fa17d4de87bd18f60d416461ba9fa4f7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 1 Jun 2015 19:47:18 +0200 Subject: [XFSHoster] Improve OFFLINE_PATTERN --- module/plugins/internal/XFSHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 34e319500..b0a1ce4ee 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.52" + __version__ = "0.53" __pattern__ = r'^unmatchable$' @@ -34,7 +34,7 @@ class XFSHoster(SimpleHoster): NAME_PATTERN = r'(Filename[ ]*:[ ]*()?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P.+?)(\s*<|")' SIZE_PATTERN = r'(Size[ ]*:[ ]*()?|File:.*>|\s*\(|<[\w^_]+ class="size">)\s*(?P[\d.,]+)\s*(?P[\w^_]+)' - OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed|no longer available)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' WAIT_PATTERN = r'(\d+)|id="countdown" value=".*?(\d+).*?"' -- cgit v1.2.3 From 109b33f5c6648563943e8461be64111d06196f94 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 2 Jun 2015 00:00:21 +0300 Subject: Update SimpleCrypter.py --- module/plugins/internal/SimpleCrypter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 2d4d52f3c..8a37839b5 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -11,7 +11,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.47" + __version__ = "0.48" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] @@ -101,7 +101,7 @@ class SimpleCrypter(Crypter, SimpleHoster): def handleDirect(self, pyfile): while True: - header = self.load(self.link or pyfile.url, just_header=True, decode=True) + header = self.load(self.link if hasattr(self, 'link') and self.link else pyfile.url, just_header=True, decode=True) if 'location' in header and header['location']: self.link = header['location'] else: @@ -114,7 +114,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.logDebug("Looking for link redirect...") self.handleDirect(pyfile) - if self.link: + if hasattr(self, 'link') and self.link: self.urls = [self.link] else: -- cgit v1.2.3 From 6637b832d74e0cf1fb96a1731daeb5832ea31822 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 2 Jun 2015 01:10:08 +0200 Subject: [SimpleCrypter] Update --- module/plugins/internal/Extractor.py | 2 +- module/plugins/internal/ReCaptcha.py | 2 +- module/plugins/internal/SimpleCrypter.py | 16 ++++------------ module/plugins/internal/SimpleHoster.py | 14 +++++++++----- module/plugins/internal/UnZip.py | 2 +- module/plugins/internal/XFSCrypter.py | 2 +- module/plugins/internal/XFSHoster.py | 4 +--- 7 files changed, 18 insertions(+), 24 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 159b65ffe..1a98060d9 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -29,8 +29,8 @@ class Extractor: EXTENSIONS = [] - VERSION = "" REPAIR = False + VERSION = "" @classmethod diff --git a/module/plugins/internal/ReCaptcha.py b/module/plugins/internal/ReCaptcha.py index dea714a62..a9d0f3752 100644 --- a/module/plugins/internal/ReCaptcha.py +++ b/module/plugins/internal/ReCaptcha.py @@ -22,8 +22,8 @@ class ReCaptcha(Captcha): ("zapp-brannigan", "fuerst.reinje@web.de")] - KEY_V2_PATTERN = r'(?:data-sitekey=["\']|["\']sitekey["\']:\s*["\'])([\w-]+)' KEY_V1_PATTERN = r'(?:recaptcha(?:/api|\.net)/(?:challenge|noscript)\?k=|Recaptcha\.create\s*\(\s*["\'])([\w-]+)' + KEY_V2_PATTERN = r'(?:data-sitekey=["\']|["\']sitekey["\']:\s*["\'])([\w-]+)' def detect_key(self, html=None): diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 8a37839b5..81f977e57 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -11,7 +11,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.48" + __version__ = "0.49" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] @@ -55,15 +55,6 @@ class SimpleCrypter(Crypter, SimpleHoster): LINK_PATTERN = None - NAME_REPLACEMENTS = [("&#?\w+;", fixup)] - URL_REPLACEMENTS = [] - - TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] - - LOGIN_ACCOUNT = False - LOGIN_PREMIUM = False - #@TODO: Remove in 0.4.10 def init(self): @@ -83,6 +74,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.info = {} self.html = "" + self.link = "" #@TODO: Move to hoster class in 0.4.10 self.links = [] #@TODO: Move to hoster class in 0.4.10 if self.LOGIN_PREMIUM and not self.premium: @@ -101,7 +93,7 @@ class SimpleCrypter(Crypter, SimpleHoster): def handleDirect(self, pyfile): while True: - header = self.load(self.link if hasattr(self, 'link') and self.link else pyfile.url, just_header=True, decode=True) + header = self.load(self.link or pyfile.url, just_header=True, decode=True) if 'location' in header and header['location']: self.link = header['location'] else: @@ -114,7 +106,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.logDebug("Looking for link redirect...") self.handleDirect(pyfile) - if hasattr(self, 'link') and self.link: + if self.link: self.urls = [self.link] else: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 222ba4b05..52ec25a5a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.51" + __version__ = "1.52" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -308,13 +308,14 @@ class SimpleHoster(Hoster): SIZE_REPLACEMENTS = [] URL_REPLACEMENTS = [] - TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + COOKIES = True #: or False or list of tuples [(domain, name, value)] DIRECT_LINK = None #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False - MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) - LOGIN_ACCOUNT = False #: Set to True to require account login DISPOSITION = True #: Set to True to use any content-disposition value in http header as file name + LOGIN_ACCOUNT = False #: Set to True to require account login + LOGIN_PREMIUM = False #: Set to True to require premium account login + MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) + TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct directLink = getFileURL #@TODO: Remove in 0.4.10 @@ -430,6 +431,9 @@ class SimpleHoster(Hoster): if not self.getConfig('use_premium', True): self.retryFree() + if self.LOGIN_PREMIUM and not self.premium: + self.fail(_("Required premium account not found")) + if self.LOGIN_ACCOUNT and not self.account: self.fail(_("Required account not found")) diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 8d3fec370..4c18a0e35 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -19,8 +19,8 @@ class UnZip(Extractor): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - EXTENSIONS = [".zip", ".zip64"] VERSION ="(python %s.%s.%s)" % (sys.version_info[0], sys.version_info[1], sys.version_info[2]) + EXTENSIONS = [".zip", ".zip64"] @classmethod diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 80eff53ea..8b333b45c 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -19,8 +19,8 @@ class XFSCrypter(SimpleCrypter): URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')] - LINK_PATTERN = r'<(?:td|TD).*?>\s*(?:<.+>\s*)?.+?(?:)?\s*(?:<.+>\s*)?' NAME_PATTERN = r'<[Tt]itle>.*?\: (?P.+) folder' + LINK_PATTERN = r'<(?:td|TD).*?>\s*(?:<.+>\s*)?.+?(?:)?\s*(?:<.+>\s*)?' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index b0a1ce4ee..e0fd8fa8d 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -27,9 +27,7 @@ class XFSHoster(SimpleHoster): HOSTER_DOMAIN = None - TEXT_ENCODING = False - DIRECT_LINK = None - MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... + MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... NAME_PATTERN = r'(Filename[ ]*:[ ]*()?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P.+?)(\s*<|")' SIZE_PATTERN = r'(Size[ ]*:[ ]*()?|File:.*>|\s*\(|<[\w^_]+ class="size">)\s*(?P[\d.,]+)\s*(?P[\w^_]+)' -- cgit v1.2.3 From f8ee62fa313beddc7af46dc1c361adb8577d0fa4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 2 Jun 2015 03:26:41 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1446 --- module/plugins/internal/DeadCrypter.py | 8 ++++---- module/plugins/internal/DeadHoster.py | 8 ++++---- module/plugins/internal/SimpleHoster.py | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index c93447164..a12399501 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- from module.plugins.internal.SimpleCrypter import create_getInfo -from module.plugins.Crypter import Crypter as _Crypter +from module.plugins.Crypter import Crypter -class DeadCrypter(_Crypter): +class DeadCrypter(Crypter): __name__ = "DeadCrypter" __type__ = "crypter" __version__ = "0.05" @@ -17,8 +17,8 @@ class DeadCrypter(_Crypter): @classmethod - def apiInfo(cls, url): - api = super(DeadCrypter, cls).apiInfo(url) + def apiInfo(cls, *args, **kwargs): + api = super(DeadCrypter, cls).apiInfo(*args, **kwargs) api['status'] = 1 return api diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index f159ae5fa..22825e87b 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- from module.plugins.internal.SimpleHoster import create_getInfo -from module.plugins.Hoster import Hoster as _Hoster +from module.plugins.Hoster import Hoster -class DeadHoster(_Hoster): +class DeadHoster(Hoster): __name__ = "DeadHoster" __type__ = "hoster" __version__ = "0.15" @@ -17,8 +17,8 @@ class DeadHoster(_Hoster): @classmethod - def apiInfo(cls, url): - api = super(DeadHoster, cls).apiInfo(url) + def apiInfo(cls, *args, **kwargs): + api = super(DeadHoster, cls).apiInfo(*args, **kwargs) api['status'] = 1 return api diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 52ec25a5a..16e9c7f2e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -317,6 +317,7 @@ class SimpleHoster(Hoster): MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct + directLink = getFileURL #@TODO: Remove in 0.4.10 -- cgit v1.2.3 From 3abd6df2b3df9cecdfd9aca298476e258b98a183 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 2 Jun 2015 12:37:59 +0200 Subject: [NitroflareCom] Premium support --- module/plugins/internal/SimpleHoster.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 16e9c7f2e..33e7d3674 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.52" + __version__ = "1.53" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -630,6 +630,7 @@ class SimpleHoster(Hoster): elif re.search('captcha|code', errmsg, re.I): self.invalidCaptcha() + self.retry(10, reason=_("Wrong captcha")) elif re.search('countdown|expired', errmsg, re.I): self.retry(wait_time=60, reason=_("Link expired")) -- cgit v1.2.3 From 08d629c98d7c0acb24dd03c785f3d64fb98bbdeb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 5 Jun 2015 16:23:42 +0200 Subject: Fix http://forum.pyload.org/viewtopic.php?f=10&t=4301 --- module/plugins/internal/SimpleHoster.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 33e7d3674..4c1441e4b 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.53" + __version__ = "1.54" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -615,7 +615,7 @@ class SimpleHoster(Hoster): self.info['error'] = re.sub(r'<.*?>', " ", errmsg) self.logWarning(self.info['error']) - if re.search('limit|wait', errmsg, re.I): + if re.search('limit|wait|slot', errmsg, re.I): if re.search("da(il)?y|today", errmsg): wait_time = secondsToMidnight(gmt=2) else: @@ -633,12 +633,12 @@ class SimpleHoster(Hoster): self.retry(10, reason=_("Wrong captcha")) elif re.search('countdown|expired', errmsg, re.I): - self.retry(wait_time=60, reason=_("Link expired")) + self.retry(10, wait_time=60, reason=_("Link expired")) elif re.search('maintenance|maintainance|temp', errmsg, re.I): self.tempOffline() - elif re.search('up to', errmsg, re.I): + elif re.search('up to|size', errmsg, re.I): self.fail(_("File too large for free download")) elif re.search('offline|delet|remov|not? (found|(longer)? available)', errmsg, re.I): -- cgit v1.2.3 From afa6e88d46d61de5523a5d00b657d4a7f742a5cc Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 6 Jun 2015 07:34:41 +0200 Subject: [UploadedTo] Fixup (2) --- module/plugins/internal/SimpleHoster.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 4c1441e4b..a369559ca 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.54" + __version__ = "1.55" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -285,8 +285,8 @@ class SimpleHoster(Hoster): IP_BLOCKED_PATTERN: (optional) example: IP_BLOCKED_PATTERN = r'in your country' - DOWNLOAD_LIMIT_PATTERN: (optional) - example: DOWNLOAD_LIMIT_PATTERN = r'download limit' + DL_LIMIT_PATTERN: (optional) + example: DL_LIMIT_PATTERN = r'download limit' SIZE_LIMIT_PATTERN: (optional) example: SIZE_LIMIT_PATTERN = r'up to' @@ -545,9 +545,12 @@ class SimpleHoster(Hoster): if r not in rules: rules[r] = p - for r, a in [('Error' , "ERROR_PATTERN" ), - ('Premium only', "PREMIUM_ONLY_PATTERN"), - ('Wait error' , "WAIT_PATTERN" )]: + for r, a in [("IP blocked" , "IP_BLOCKED_PATTERN" ), + ("Download limit", "DL_LIMIT_PATTERN" ), + ("Size limit" , "SIZE_LIMIT_PATTERN" ), + ("Error" , "ERROR_PATTERN" ), + ("Premium only" , "PREMIUM_ONLY_PATTERN"), + ("Wait error" , "WAIT_PATTERN" )]: if r not in rules and hasattr(self, a): rules[r] = getattr(self, a) @@ -582,8 +585,8 @@ class SimpleHoster(Hoster): elif hasattr(self, 'SIZE_LIMIT_PATTERN') and re.search(self.SIZE_LIMIT_PATTERN, self.html): self.fail(_("File too large for free download")) - elif hasattr(self, 'DOWNLOAD_LIMIT_PATTERN') and re.search(self.DOWNLOAD_LIMIT_PATTERN, self.html): - m = re.search(self.DOWNLOAD_LIMIT_PATTERN, self.html) + elif hasattr(self, 'DL_LIMIT_PATTERN') and re.search(self.DL_LIMIT_PATTERN, self.html): + m = re.search(self.DL_LIMIT_PATTERN, self.html) try: errmsg = m.group(1).strip() except Exception: -- cgit v1.2.3 From 681deb7ff09956be0bee04453e33ac6e9d6dab4a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 05:56:10 +0200 Subject: Move base plugins to internal folder --- module/plugins/internal/Account.py | 292 +++++++++++++++++ module/plugins/internal/Container.py | 75 +++++ module/plugins/internal/Crypter.py | 72 ++++ module/plugins/internal/Hook.py | 161 +++++++++ module/plugins/internal/Hoster.py | 33 ++ module/plugins/internal/Plugin.py | 617 +++++++++++++++++++++++++++++++++++ 6 files changed, 1250 insertions(+) create mode 100644 module/plugins/internal/Account.py create mode 100644 module/plugins/internal/Container.py create mode 100644 module/plugins/internal/Crypter.py create mode 100644 module/plugins/internal/Hook.py create mode 100644 module/plugins/internal/Hoster.py create mode 100644 module/plugins/internal/Plugin.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py new file mode 100644 index 000000000..c147404e0 --- /dev/null +++ b/module/plugins/internal/Account.py @@ -0,0 +1,292 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from random import choice +from time import time +from traceback import print_exc +from threading import RLock + +from Plugin import Base +from module.utils import compare_time, parseFileSize, lock + +class WrongPassword(Exception): + pass + + +class Account(Base): + """ + Base class for every Account plugin. + Just overwrite `login` and cookies will be stored and account becomes accessible in\ + associated hoster plugin. Plugin should also provide `loadAccountInfo` + """ + __name__ = "Account" + __version__ = "0.2" + __type__ = "account" + __description__ = """Account Plugin""" + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") + + #: after that time [in minutes] pyload will relogin the account + login_timeout = 600 + #: account data will be reloaded after this time + info_threshold = 600 + + + def __init__(self, manager, accounts): + Base.__init__(self, manager.core) + + self.manager = manager + self.accounts = {} + self.infos = {} # cache for account information + self.lock = RLock() + + self.timestamps = {} + self.setAccounts(accounts) + self.init() + + def init(self): + pass + + def login(self, user, data, req): + """login into account, the cookies will be saved so user can be recognized + + :param user: loginname + :param data: data dictionary + :param req: `Request` instance + """ + pass + + @lock + def _login(self, user, data): + # set timestamp for login + self.timestamps[user] = time() + + req = self.getAccountRequest(user) + try: + self.login(user, data, req) + except WrongPassword: + self.logWarning( + _("Could not login with account %(user)s | %(msg)s") % {"user": user + , "msg": _("Wrong Password")}) + data["valid"] = False + + except Exception, e: + self.logWarning( + _("Could not login with account %(user)s | %(msg)s") % {"user": user + , "msg": e}) + data["valid"] = False + if self.core.debug: + print_exc() + finally: + if req: req.close() + + def relogin(self, user): + req = self.getAccountRequest(user) + if req: + req.cj.clear() + req.close() + if user in self.infos: + del self.infos[user] #delete old information + + self._login(user, self.accounts[user]) + + def setAccounts(self, accounts): + self.accounts = accounts + for user, data in self.accounts.iteritems(): + self._login(user, data) + self.infos[user] = {} + + def updateAccounts(self, user, password=None, options={}): + """ updates account and return true if anything changed """ + + if user in self.accounts: + self.accounts[user]["valid"] = True #do not remove or accounts will not login + if password: + self.accounts[user]["password"] = password + self.relogin(user) + return True + if options: + before = self.accounts[user]["options"] + self.accounts[user]["options"].update(options) + return self.accounts[user]["options"] != before + else: + self.accounts[user] = {"password": password, "options": options, "valid": True} + self._login(user, self.accounts[user]) + return True + + def removeAccount(self, user): + if user in self.accounts: + del self.accounts[user] + if user in self.infos: + del self.infos[user] + if user in self.timestamps: + del self.timestamps[user] + + @lock + def getAccountInfo(self, name, force=False): + """retrieve account infos for an user, do **not** overwrite this method!\\ + just use it to retrieve infos in hoster plugins. see `loadAccountInfo` + + :param name: username + :param force: reloads cached account information + :return: dictionary with information + """ + data = Account.loadAccountInfo(self, name) + + if force or name not in self.infos: + self.logDebug("Get Account Info for %s" % name) + req = self.getAccountRequest(name) + + try: + infos = self.loadAccountInfo(name, req) + if not type(infos) == dict: + raise Exception("Wrong return format") + except Exception, e: + infos = {"error": str(e)} + + if req: req.close() + + self.logDebug("Account Info: %s" % str(infos)) + + infos["timestamp"] = time() + self.infos[name] = infos + elif "timestamp" in self.infos[name] and self.infos[name][ + "timestamp"] + self.info_threshold * 60 < time(): + self.logDebug("Reached timeout for account data") + self.scheduleRefresh(name) + + data.update(self.infos[name]) + return data + + def isPremium(self, user): + info = self.getAccountInfo(user) + return info["premium"] + + def loadAccountInfo(self, name, req=None): + """this should be overwritten in account plugin,\ + and retrieving account information for user + + :param name: + :param req: `Request` instance + :return: + """ + return { + "validuntil": None, # -1 for unlimited + "login": name, + #"password": self.accounts[name]["password"], #@XXX: security + "options": self.accounts[name]["options"], + "valid": self.accounts[name]["valid"], + "trafficleft": None, # in kb, -1 for unlimited + "maxtraffic": None, + "premium": True, #useful for free accounts + "timestamp": 0, #time this info was retrieved + "type": self.__name__, + } + + def getAllAccounts(self, force=False): + return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] + + def getAccountRequest(self, user=None): + if not user: + user, data = self.selectAccount() + if not user: + return None + + req = self.core.requestFactory.getRequest(self.__name__, user) + return req + + def getAccountCookies(self, user=None): + if not user: + user, data = self.selectAccount() + if not user: + return None + + cj = self.core.requestFactory.getCookieJar(self.__name__, user) + return cj + + def getAccountData(self, user): + return self.accounts[user] + + def selectAccount(self): + """ returns an valid account name and data""" + usable = [] + for user, data in self.accounts.iteritems(): + if not data["valid"]: continue + + if "time" in data["options"] and data["options"]["time"]: + time_data = "" + try: + time_data = data["options"]["time"][0] + start, end = time_data.split("-") + if not compare_time(start.split(":"), end.split(":")): + continue + except: + self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + + if user in self.infos: + if "validuntil" in self.infos[user]: + if self.infos[user]["validuntil"] > 0 and time() > self.infos[user]["validuntil"]: + continue + if "trafficleft" in self.infos[user]: + if self.infos[user]["trafficleft"] == 0: + continue + + usable.append((user, data)) + + if not usable: return None, None + return choice(usable) + + def canUse(self): + return False if self.selectAccount() == (None, None) else True + + def parseTraffic(self, string): #returns kbyte + return parseFileSize(string) / 1024 + + def wrongPassword(self): + raise WrongPassword + + def empty(self, user): + if user in self.infos: + self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) + + self.infos[user].update({"trafficleft": 0}) + self.scheduleRefresh(user, 30 * 60) + + def expired(self, user): + if user in self.infos: + self.logWarning(_("Account %s is expired, checking again in 1h") % user) + + self.infos[user].update({"validuntil": time() - 1}) + self.scheduleRefresh(user, 60 * 60) + + def scheduleRefresh(self, user, time=0, force=True): + """ add task to refresh account info to sheduler """ + self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) + self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) + + @lock + def checkLogin(self, user): + """ checks if user is still logged in """ + if user in self.timestamps: + if self.timestamps[user] + self.login_timeout * 60 < time(): + self.logDebug("Reached login timeout for %s" % user) + self.relogin(user) + return False + + return True diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py new file mode 100644 index 000000000..c233d3710 --- /dev/null +++ b/module/plugins/internal/Container.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from module.plugins.Crypter import Crypter + +from os.path import join, exists, basename +from os import remove +import re + +class Container(Crypter): + __name__ = "Container" + __version__ = "0.1" + __pattern__ = None + __type__ = "container" + __description__ = """Base container plugin""" + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") + + + def preprocessing(self, thread): + """prepare""" + + self.setup() + self.thread = thread + + self.loadToDisk() + + self.decrypt(self.pyfile) + self.deleteTmp() + + self.createPackages() + + + def loadToDisk(self): + """loads container to disk if its stored remotely and overwrite url, + or check existent on several places at disk""" + + if self.pyfile.url.startswith("http"): + self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] + content = self.load(self.pyfile.url) + self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name) + f = open(self.pyfile.url, "wb" ) + f.write(content) + f.close() + + else: + self.pyfile.name = basename(self.pyfile.url) + if not exists(self.pyfile.url): + if exists(join(pypath, self.pyfile.url)): + self.pyfile.url = join(pypath, self.pyfile.url) + else: + self.fail(_("File not exists.")) + + + def deleteTmp(self): + if self.pyfile.name.startswith("tmp_"): + remove(self.pyfile.url) + + diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py new file mode 100644 index 000000000..d1549fe80 --- /dev/null +++ b/module/plugins/internal/Crypter.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from module.plugins.Plugin import Plugin + +class Crypter(Plugin): + __name__ = "Crypter" + __version__ = "0.1" + __pattern__ = None + __type__ = "container" + __description__ = """Base crypter plugin""" + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") + + def __init__(self, pyfile): + Plugin.__init__(self, pyfile) + + #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) + self.packages = [] + + #: List of urls, pyLoad will generate packagenames + self.urls = [] + + self.multiDL = True + self.limitDL = 0 + + + def preprocessing(self, thread): + """prepare""" + self.setup() + self.thread = thread + + self.decrypt(self.pyfile) + + self.createPackages() + + + def decrypt(self, pyfile): + raise NotImplementedError + + def createPackages(self): + """ create new packages from self.packages """ + for pack in self.packages: + + self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } ) + + links = [x.decode("utf-8") for x in pack[1]] + + pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue) + + if self.pyfile.package().password: + self.core.api.setPackageData(pid, {"password": self.pyfile.package().password}) + + if self.urls: + self.core.api.generateAndAddPackages(self.urls) + diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py new file mode 100644 index 000000000..5efd08bae --- /dev/null +++ b/module/plugins/internal/Hook.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay + @interface-version: 0.2 +""" + +from traceback import print_exc + +from Plugin import Base + +class Expose(object): + """ used for decoration to declare rpc services """ + + def __new__(cls, f, *args, **kwargs): + hookManager.addRPC(f.__module__, f.func_name, f.func_doc) + return f + +def threaded(f): + def run(*args,**kwargs): + hookManager.startThread(f, *args, **kwargs) + return run + +class Hook(Base): + """ + Base class for hook plugins. + """ + __name__ = "Hook" + __version__ = "0.2" + __type__ = "hook" + __threaded__ = [] + __config__ = [ ("name", "type", "desc" , "default") ] + __description__ = """interface for hook""" + __author_name__ = ("mkaay", "RaNaN") + __author_mail__ = ("mkaay@mkaay.de", "RaNaN@pyload.org") + + #: automatically register event listeners for functions, attribute will be deleted dont use it yourself + event_map = None + + # Alternative to event_map + #: List of events the plugin can handle, name the functions exactly like eventname. + event_list = None # dont make duplicate entries in event_map + + + #: periodic call interval in secondc + interval = 60 + + def __init__(self, core, manager): + Base.__init__(self, core) + + #: Provide information in dict here, usable by API `getInfo` + self.info = None + + #: Callback of periodical job task, used by hookmanager + self.cb = None + + #: `HookManager` + self.manager = manager + + #register events + if self.event_map: + for event, funcs in self.event_map.iteritems(): + if type(funcs) in (list, tuple): + for f in funcs: + self.manager.addEvent(event, getattr(self,f)) + else: + self.manager.addEvent(event, getattr(self,funcs)) + + #delete for various reasons + self.event_map = None + + if self.event_list: + for f in self.event_list: + self.manager.addEvent(f, getattr(self,f)) + + self.event_list = None + + self.initPeriodical() + self.setup() + + def initPeriodical(self): + if self.interval >=1: + self.cb = self.core.scheduler.addJob(0, self._periodical, threaded=False) + + def _periodical(self): + try: + if self.isActivated(): self.periodical() + except Exception, e: + self.core.log.error(_("Error executing hooks: %s") % str(e)) + if self.core.debug: + print_exc() + + self.cb = self.core.scheduler.addJob(self.interval, self._periodical, threaded=False) + + + def __repr__(self): + return "" % self.__name__ + + def setup(self): + """ more init stuff if needed """ + pass + + def unload(self): + """ called when hook was deactivated """ + pass + + def isActivated(self): + """ checks if hook is activated""" + return self.config.getPlugin(self.__name__, "activated") + + + #event methods - overwrite these if needed + def coreReady(self): + pass + + def coreExiting(self): + pass + + def downloadPreparing(self, pyfile): + pass + + def downloadFinished(self, pyfile): + pass + + def downloadFailed(self, pyfile): + pass + + def packageFinished(self, pypack): + pass + + def beforeReconnecting(self, ip): + pass + + def afterReconnecting(self, ip): + pass + + def periodical(self): + pass + + def newCaptchaTask(self, task): + """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """ + pass + + def captchaCorrect(self, task): + pass + + def captchaInvalid(self, task): + pass \ No newline at end of file diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py new file mode 100644 index 000000000..814a70949 --- /dev/null +++ b/module/plugins/internal/Hoster.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from module.plugins.Plugin import Plugin + +def getInfo(self): + #result = [ .. (name, size, status, url) .. ] + return + +class Hoster(Plugin): + __name__ = "Hoster" + __version__ = "0.1" + __pattern__ = None + __type__ = "hoster" + __description__ = """Base hoster plugin""" + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py new file mode 100644 index 000000000..15bf3971f --- /dev/null +++ b/module/plugins/internal/Plugin.py @@ -0,0 +1,617 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN, spoob, mkaay +""" + +from time import time, sleep +from random import randint + +import os +from os import remove, makedirs, chmod, stat +from os.path import exists, join + +if os.name != "nt": + from os import chown + from pwd import getpwnam + from grp import getgrnam + +from itertools import islice + +from module.utils import save_join, save_path, fs_encode, fs_decode + +def chunks(iterable, size): + it = iter(iterable) + item = list(islice(it, size)) + while item: + yield item + item = list(islice(it, size)) + + +class Abort(Exception): + """ raised when aborted """ + + +class Fail(Exception): + """ raised when failed """ + + +class Reconnect(Exception): + """ raised when reconnected """ + + +class Retry(Exception): + """ raised when start again from beginning """ + + +class SkipDownload(Exception): + """ raised when download should be skipped """ + + +class Base(object): + """ + A Base class with log/config/db methods *all* plugin types can use + """ + + def __init__(self, core): + #: Core instance + self.core = core + #: logging instance + self.log = core.log + #: core config + self.config = core.config + + #log functions + def logInfo(self, *args): + self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logWarning(self, *args): + self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logError(self, *args): + self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logDebug(self, *args): + self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + + def setConf(self, option, value): + """ see `setConfig` """ + self.core.config.setPlugin(self.__name__, option, value) + + def setConfig(self, option, value): + """ Set config value for current plugin + + :param option: + :param value: + :return: + """ + self.setConf(option, value) + + def getConf(self, option): + """ see `getConfig` """ + return self.core.config.getPlugin(self.__name__, option) + + def getConfig(self, option): + """ Returns config value for current plugin + + :param option: + :return: + """ + return self.getConf(option) + + def setStorage(self, key, value): + """ Saves a value persistently to the database """ + self.core.db.setStorage(self.__name__, key, value) + + def store(self, key, value): + """ same as `setStorage` """ + self.core.db.setStorage(self.__name__, key, value) + + def getStorage(self, key=None, default=None): + """ Retrieves saved value or dict of all saved entries if key is None """ + if key is not None: + return self.core.db.getStorage(self.__name__, key) or default + return self.core.db.getStorage(self.__name__, key) + + def retrieve(self, *args, **kwargs): + """ same as `getStorage` """ + return self.getStorage(*args, **kwargs) + + def delStorage(self, key): + """ Delete entry in db """ + self.core.db.delStorage(self.__name__, key) + + +class Plugin(Base): + """ + Base plugin for hoster/crypter. + Overwrite `process` / `decrypt` in your subclassed plugin. + """ + __name__ = "Plugin" + __version__ = "0.4" + __pattern__ = None + __type__ = "hoster" + __config__ = [("name", "type", "desc", "default")] + __description__ = """Base Plugin""" + __author_name__ = ("RaNaN", "spoob", "mkaay") + __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") + + def __init__(self, pyfile): + Base.__init__(self, pyfile.m.core) + + self.wantReconnect = False + #: enables simultaneous processing of multiple downloads + self.multiDL = True + self.limitDL = 0 + #: chunk limit + self.chunkLimit = 1 + self.resumeDownload = False + + #: time() + wait in seconds + self.waitUntil = 0 + self.waiting = False + + self.ocr = None #captcha reader instance + #: account handler instance, see :py:class:`Account` + self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + + #: premium status + self.premium = False + #: username/login + self.user = None + + if self.account and not self.account.canUse(): self.account = None + if self.account: + self.user, data = self.account.selectAccount() + #: Browser instance, see `network.Browser` + self.req = self.account.getAccountRequest(self.user) + self.chunkLimit = -1 # chunk limit, -1 for unlimited + #: enables resume (will be ignored if server dont accept chunks) + self.resumeDownload = True + self.multiDL = True #every hoster with account should provide multiple downloads + #: premium status + self.premium = self.account.isPremium(self.user) + else: + self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) + + #: associated pyfile instance, see `PyFile` + self.pyfile = pyfile + self.thread = None # holds thread in future + + #: location where the last call to download was saved + self.lastDownload = "" + #: re match of the last call to `checkDownload` + self.lastCheck = None + #: js engine, see `JsEngine` + self.js = self.core.js + self.cTask = None #captcha task + + self.retries = 0 # amount of retries already made + self.html = None # some plugins store html code here + + self.init() + + def getChunkCount(self): + if self.chunkLimit <= 0: + return self.config["download"]["chunks"] + return min(self.config["download"]["chunks"], self.chunkLimit) + + def __call__(self): + return self.__name__ + + def init(self): + """initialize the plugin (in addition to `__init__`)""" + pass + + def setup(self): + """ setup for enviroment and other things, called before downloading (possibly more than one time)""" + pass + + def preprocessing(self, thread): + """ handles important things to do before starting """ + self.thread = thread + + if self.account: + self.account.checkLogin(self.user) + else: + self.req.clearCookies() + + self.setup() + + self.pyfile.setStatus("starting") + + return self.process(self.pyfile) + + + def process(self, pyfile): + """the 'main' method of every plugin, you **have to** overwrite it""" + raise NotImplementedError + + def resetAccount(self): + """ dont use account and retry download """ + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retry() + + def checksum(self, local_file=None): + """ + return codes: + 0 - checksum ok + 1 - checksum wrong + 5 - can't get checksum + 10 - not implemented + 20 - unknown error + """ + #@TODO checksum check hook + + return True, 10 + + + def setWait(self, seconds, reconnect=False): + """Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + if reconnect: + self.wantReconnect = True + self.pyfile.waitUntil = time() + int(seconds) + + def wait(self): + """ waits the time previously set """ + self.waiting = True + self.pyfile.setStatus("waiting") + + while self.pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) + + if self.pyfile.abort: raise Abort + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + self.waiting = False + self.pyfile.setStatus("starting") + + def fail(self, reason): + """ fail and give reason """ + raise Fail(reason) + + def offline(self): + """ fail and indicate file is offline """ + raise Fail("offline") + + def tempOffline(self): + """ fail and indicates file ist temporary offline, the core may take consequences """ + raise Fail("temp. offline") + + def retry(self, max_tries=3, wait_time=1, reason=""): + """Retries and begin again from the beginning + + :param max_tries: number of maximum retries + :param wait_time: time to wait in seconds + :param reason: reason for retrying, will be passed to fail if max_tries reached + """ + if 0 < max_tries <= self.retries: + if not reason: reason = "Max retries reached" + raise Fail(reason) + + self.wantReconnect = False + self.setWait(wait_time) + self.wait() + + self.retries += 1 + raise Retry(reason) + + def invalidCaptcha(self): + if self.cTask: + self.cTask.invalid() + + def correctCaptcha(self): + if self.cTask: + self.cTask.correct() + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', + result_type='textual'): + """ Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time())[-6:].replace(".", "") + temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp_file.write(img) + temp_file.close() + + has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins + + if self.core.captcha: + Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) + else: + Ocr = None + + if Ocr and not forceUser: + sleep(randint(3000, 5000) / 1000.0) + if self.pyfile.abort: raise Abort + + ocr = Ocr() + result = ocr.get_captcha(temp_file.name) + else: + captchaManager = self.core.captchaManager + task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) + self.cTask = task + captchaManager.handleCaptcha(task) + + while task.isWaiting(): + if self.pyfile.abort: + captchaManager.removeTask(task) + raise Abort + sleep(1) + + captchaManager.removeTask(task) + + if task.error and has_plugin: #ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) + + result = task.result + self.log.debug("Received captcha result: %s" % str(result)) + + if not self.core.debug: + try: + remove(temp_file.name) + except: + pass + + return result + + + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): + """Load content at url and returns it + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param just_header: if True only the header will be retrieved and returned as dict + :param decode: Wether to decode the output according to http header, should be True in most cases + :return: Loaded content + """ + if self.pyfile.abort: raise Abort + #utf8 vs decode -> please use decode attribute in all future plugins + if type(url) == unicode: url = str(url) + + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) + + if self.core.debug: + from inspect import currentframe + + frame = currentframe() + if not exists(join("tmp", self.__name__)): + makedirs(join("tmp", self.__name__)) + + f = open( + join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + , "wb") + del frame # delete the frame or it wont be cleaned + + try: + tmp = res.encode("utf8") + except: + tmp = res + + f.write(tmp) + f.close() + + if just_header: + #parse header + header = {"code": self.req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: continue + + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() + + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value + res = header + + return res + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + """Downloads the content at url to download folder + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param disposition: if True and server provides content-disposition header\ + the filename will be changed if needed + :return: The location where the file was saved + """ + + self.checkForSameFiles() + + self.pyfile.setStatus("downloading") + + download_folder = self.config['general']['download_folder'] + + location = save_join(download_folder, self.pyfile.package().folder) + + if not exists(location): + makedirs(location, int(self.core.config["permission"]["folder"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(location, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + # convert back to unicode + location = fs_decode(location) + name = save_path(self.pyfile.name) + + filename = join(location, name) + + self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) + + try: + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.setProgress, disposition=disposition) + finally: + self.pyfile.size = self.req.size + + if disposition and newname and newname != name: #triple check, just to be sure + self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) + + fs_filename = fs_encode(filename) + + if self.core.config["permission"]["change_file"]: + chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(fs_filename, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + self.lastDownload = filename + return self.lastDownload + + def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): + """ checks the content of the last downloaded file, re match is saved to `lastCheck` + + :param rules: dict with names and rules to match (compiled regexp or strings) + :param api_size: expected file size + :param max_size: if the file is larger then it wont be checked + :param delete: delete if matched + :param read_size: amount of bytes to read from files larger then max_size + :return: dictionary key of the first rule that matched + """ + lastDownload = fs_encode(self.lastDownload) + if not exists(lastDownload): return None + + size = stat(lastDownload) + size = size.st_size + + if api_size and api_size <= size: return None + elif size > max_size and not read_size: return None + self.log.debug("Download Check triggered") + f = open(lastDownload, "rb") + content = f.read(read_size if read_size else -1) + f.close() + #produces encoding errors, better log to other file in the future? + #self.log.debug("Content: %s" % content) + for name, rule in rules.iteritems(): + if type(rule) in (str, unicode): + if rule in content: + if delete: + remove(lastDownload) + return name + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + if delete: + remove(lastDownload) + self.lastCheck = m + return name + + + def getPassword(self): + """ get the password the user provided in the package""" + password = self.pyfile.package().password + if not password: return "" + return password + + + def checkForSameFiles(self, starting=False): + """ checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises SkipDownload: + """ + + pack = self.pyfile.package() + + for pyfile in self.core.files.cache.values(): + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #finished or downloading + raise SkipDownload(pyfile.pluginname) + elif pyfile.status in ( + 5, 7) and starting: #a download is waiting/starting and was appenrently started before + raise SkipDownload(pyfile.pluginname) + + download_folder = self.config['general']['download_folder'] + location = save_join(download_folder, pack.folder, self.pyfile.name) + + if starting and self.core.config['download']['skip_existing'] and exists(location): + size = os.stat(location).st_size + if size >= self.pyfile.size: + raise SkipDownload("File exists.") + + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + if pyfile: + if exists(location): + raise SkipDownload(pyfile[0]) + + self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) + + def clean(self): + """ clean everything and remove references """ + if hasattr(self, "pyfile"): + del self.pyfile + if hasattr(self, "req"): + self.req.close() + del self.req + if hasattr(self, "thread"): + del self.thread + if hasattr(self, "html"): + del self.html -- cgit v1.2.3 From 0e1ef9bc01579328e17e79416fa3c1c7b77adcc8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 06:08:01 +0200 Subject: Update everything --- module/plugins/internal/Account.py | 6 +++--- module/plugins/internal/Captcha.py | 4 ++-- module/plugins/internal/Container.py | 22 +++++++++++----------- module/plugins/internal/Crypter.py | 24 ++++++++++++------------ module/plugins/internal/DeadCrypter.py | 4 ++-- module/plugins/internal/DeadHoster.py | 4 ++-- module/plugins/internal/Hook.py | 24 ++++++++++++------------ module/plugins/internal/Hoster.py | 6 +++--- module/plugins/internal/MultiHook.py | 4 ++-- module/plugins/internal/MultiHoster.py | 4 ++-- module/plugins/internal/Plugin.py | 8 ++++---- module/plugins/internal/SimpleCrypter.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 6 +++--- module/plugins/internal/SolveMedia.py | 4 ++-- module/plugins/internal/XFSAccount.py | 4 ++-- 15 files changed, 64 insertions(+), 64 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index c147404e0..7632d15c2 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - + @author: mkaay """ @@ -36,7 +36,7 @@ class Account(Base): associated hoster plugin. Plugin should also provide `loadAccountInfo` """ __name__ = "Account" - __version__ = "0.2" + __version__ = "0.02" __type__ = "account" __description__ = """Account Plugin""" __author_name__ = ("mkaay") @@ -76,7 +76,7 @@ class Account(Base): def _login(self, user, data): # set timestamp for login self.timestamps[user] = time() - + req = self.getAccountRequest(user) try: self.login(user, data, req) diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index b4af46493..e5a44d750 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- -from module.plugins.Plugin import Base +from module.plugins.internal.Plugin import Base #@TODO: Extend (new) Plugin class; remove all `html` args class Captcha(Base): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.29" + __version__ = "0.30" __description__ = """Base captcha service plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index c233d3710..43ca95e77 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -13,11 +13,11 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - + @author: mkaay """ -from module.plugins.Crypter import Crypter +from module.plugins.internal.Crypter import Crypter from os.path import join, exists, basename from os import remove @@ -25,7 +25,7 @@ import re class Container(Crypter): __name__ = "Container" - __version__ = "0.1" + __version__ = "0.02" __pattern__ = None __type__ = "container" __description__ = """Base container plugin""" @@ -38,19 +38,19 @@ class Container(Crypter): self.setup() self.thread = thread - + self.loadToDisk() self.decrypt(self.pyfile) self.deleteTmp() - + self.createPackages() - + def loadToDisk(self): - """loads container to disk if its stored remotely and overwrite url, + """loads container to disk if its stored remotely and overwrite url, or check existent on several places at disk""" - + if self.pyfile.url.startswith("http"): self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] content = self.load(self.pyfile.url) @@ -58,7 +58,7 @@ class Container(Crypter): f = open(self.pyfile.url, "wb" ) f.write(content) f.close() - + else: self.pyfile.name = basename(self.pyfile.url) if not exists(self.pyfile.url): @@ -66,10 +66,10 @@ class Container(Crypter): self.pyfile.url = join(pypath, self.pyfile.url) else: self.fail(_("File not exists.")) - + def deleteTmp(self): if self.pyfile.name.startswith("tmp_"): remove(self.pyfile.url) - + diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index d1549fe80..b87ed1882 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -13,33 +13,33 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - + @author: mkaay """ -from module.plugins.Plugin import Plugin +from module.plugins.internal.Plugin import Plugin class Crypter(Plugin): __name__ = "Crypter" - __version__ = "0.1" + __version__ = "0.02" __pattern__ = None __type__ = "container" __description__ = """Base crypter plugin""" __author_name__ = ("mkaay") __author_mail__ = ("mkaay@mkaay.de") - + def __init__(self, pyfile): Plugin.__init__(self, pyfile) - + #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) self.packages = [] #: List of urls, pyLoad will generate packagenames self.urls = [] - + self.multiDL = True self.limitDL = 0 - + def preprocessing(self, thread): """prepare""" @@ -47,9 +47,9 @@ class Crypter(Plugin): self.thread = thread self.decrypt(self.pyfile) - + self.createPackages() - + def decrypt(self, pyfile): raise NotImplementedError @@ -59,9 +59,9 @@ class Crypter(Plugin): for pack in self.packages: self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } ) - + links = [x.decode("utf-8") for x in pack[1]] - + pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue) if self.pyfile.package().password: @@ -69,4 +69,4 @@ class Crypter(Plugin): if self.urls: self.core.api.generateAndAddPackages(self.urls) - + diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index a12399501..28ead4c23 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- +from module.plugins.internal.Crypter import Crypter from module.plugins.internal.SimpleCrypter import create_getInfo -from module.plugins.Crypter import Crypter class DeadCrypter(Crypter): __name__ = "DeadCrypter" __type__ = "crypter" - __version__ = "0.05" + __version__ = "0.06" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 22825e87b..6dc860eea 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- +from module.plugins.internal.Hoster import Hoster from module.plugins.internal.SimpleHoster import create_getInfo -from module.plugins.Hoster import Hoster class DeadHoster(Hoster): __name__ = "DeadHoster" __type__ = "hoster" - __version__ = "0.15" + __version__ = "0.16" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 5efd08bae..5f2639dce 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - + @author: mkaay @interface-version: 0.2 """ @@ -39,7 +39,7 @@ class Hook(Base): Base class for hook plugins. """ __name__ = "Hook" - __version__ = "0.2" + __version__ = "0.02" __type__ = "hook" __threaded__ = [] __config__ = [ ("name", "type", "desc" , "default") ] @@ -108,7 +108,7 @@ class Hook(Base): def __repr__(self): return "" % self.__name__ - + def setup(self): """ more init stuff if needed """ pass @@ -116,37 +116,37 @@ class Hook(Base): def unload(self): """ called when hook was deactivated """ pass - + def isActivated(self): """ checks if hook is activated""" return self.config.getPlugin(self.__name__, "activated") - - #event methods - overwrite these if needed + + #event methods - overwrite these if needed def coreReady(self): pass def coreExiting(self): pass - + def downloadPreparing(self, pyfile): pass - + def downloadFinished(self, pyfile): pass - + def downloadFailed(self, pyfile): pass - + def packageFinished(self, pypack): pass def beforeReconnecting(self, ip): pass - + def afterReconnecting(self, ip): pass - + def periodical(self): pass diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 814a70949..3fed8a7c6 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -13,11 +13,11 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - + @author: mkaay """ -from module.plugins.Plugin import Plugin +from module.plugins.internal.Plugin import Plugin def getInfo(self): #result = [ .. (name, size, status, url) .. ] @@ -25,7 +25,7 @@ def getInfo(self): class Hoster(Plugin): __name__ = "Hoster" - __version__ = "0.1" + __version__ = "0.02" __pattern__ = None __type__ = "hoster" __description__ = """Base hoster plugin""" diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 01ff4b07d..4a2c6654b 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -4,14 +4,14 @@ import re import time import traceback -from module.plugins.Hook import Hook +from module.plugins.internal.Hook import Hook from module.utils import decode, remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.45" + __version__ = "0.46" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index ff4414034..2b4ceb9b0 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -2,14 +2,14 @@ import re -from module.plugins.Plugin import Fail, Retry +from module.plugins.internal.Plugin import Fail, Retry from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.40" + __version__ = "0.41" __pattern__ = r'^unmatchable$' __config__ = [("use_premium" , "bool", "Use premium account if available" , True), diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 15bf3971f..e1de48f72 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -142,7 +142,7 @@ class Plugin(Base): Overwrite `process` / `decrypt` in your subclassed plugin. """ __name__ = "Plugin" - __version__ = "0.4" + __version__ = "0.04" __pattern__ = None __type__ = "hoster" __config__ = [("name", "type", "desc", "default")] @@ -263,7 +263,7 @@ class Plugin(Base): def setWait(self, seconds, reconnect=False): """Set a specific wait time later used with `wait` - + :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ @@ -339,7 +339,7 @@ class Plugin(Base): :param result_type: 'textual' if text is written on the captcha\ or 'positional' for captcha where the user have to click\ on a specific region on the captcha - + :return: result of decrypting """ @@ -528,7 +528,7 @@ class Plugin(Base): def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): """ checks the content of the last downloaded file, re match is saved to `lastCheck` - + :param rules: dict with names and rules to match (compiled regexp or strings) :param api_size: expected file size :param max_size: if the file is larger then it wont be checked diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 81f977e57..6279be1bd 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -3,7 +3,7 @@ import re import urlparse -from module.plugins.Crypter import Crypter +from module.plugins.internal.Crypter import Crypter from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies from module.utils import fixup, html_unescape @@ -11,7 +11,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.49" + __version__ = "0.50" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index a369559ca..da477ad79 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -12,8 +12,8 @@ from module.PyFile import statusMap as _statusMap from module.network.CookieJar import CookieJar from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL -from module.plugins.Hoster import Hoster -from module.plugins.Plugin import Fail, Retry +from module.plugins.internal.Hoster import Hoster +from module.plugins.internal.Plugin import Fail, Retry from module.utils import fixup, fs_encode, html_unescape, parseFileSize @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.55" + __version__ = "1.56" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), diff --git a/module/plugins/internal/SolveMedia.py b/module/plugins/internal/SolveMedia.py index 7f5de51e1..d600ef03c 100644 --- a/module/plugins/internal/SolveMedia.py +++ b/module/plugins/internal/SolveMedia.py @@ -2,14 +2,14 @@ import re -from module.plugins.Plugin import Fail +from module.plugins.internal.Plugin import Fail from module.plugins.internal.Captcha import Captcha class SolveMedia(Captcha): __name__ = "SolveMedia" __type__ = "captcha" - __version__ = "0.13" + __version__ = "0.14" __description__ = """SolveMedia captcha service plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index e619cb038..f230cedf3 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -4,14 +4,14 @@ import re import time import urlparse -from module.plugins.Account import Account +from module.plugins.internal.Account import Account from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.37" + __version__ = "0.38" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" -- cgit v1.2.3 From 3af8bfcf685f82de097ef6b110dffa77fd529eaf Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 10:01:24 +0200 Subject: [SimpleHoster] Improve logging and file checking --- module/plugins/internal/MultiHoster.py | 8 +- module/plugins/internal/SimpleCrypter.py | 13 ++-- module/plugins/internal/SimpleHoster.py | 124 ++++++++++++++++++++----------- 3 files changed, 92 insertions(+), 53 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 2b4ceb9b0..2b3eb8941 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.41" + __version__ = "0.42" __pattern__ = r'^unmatchable$' __config__ = [("use_premium" , "bool", "Use premium account if available" , True), @@ -32,8 +32,8 @@ class MultiHoster(SimpleHoster): def prepare(self): self.info = {} self.html = "" - self.link = "" #@TODO: Move to hoster class in 0.4.10 - self.directDL = False #@TODO: Move to hoster class in 0.4.10 + self.link = "" #@TODO: Move to Hoster in 0.4.10 + self.directDL = False #@TODO: Move to Hoster in 0.4.10 if not self.getConfig('use_premium', True): self.retryFree() @@ -77,7 +77,7 @@ class MultiHoster(SimpleHoster): self.logDebug("Handled as free download") self.handleFree(pyfile) - self.downloadLink(self.link, True) + self.download(self.link, ref=False, disposition=True) self.checkFile() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 6279be1bd..00b872b69 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -11,7 +11,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.50" + __version__ = "0.51" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] @@ -74,8 +74,8 @@ class SimpleCrypter(Crypter, SimpleHoster): self.info = {} self.html = "" - self.link = "" #@TODO: Move to hoster class in 0.4.10 - self.links = [] #@TODO: Move to hoster class in 0.4.10 + self.link = "" #@TODO: Move to Hoster in 0.4.10 + self.links = [] #@TODO: Move to Hoster in 0.4.10 if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) @@ -121,6 +121,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.logDebug("Package has %d links" % len(self.links)) if self.links: + self.links = [html_unescape(l.decode('unicode-escape').strip()) for l in links] #@TODO: Move to Crypter in 0.4.10 self.packages = [(self.info['name'], self.links, self.info['folder'])] elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 @@ -160,10 +161,8 @@ class SimpleCrypter(Crypter, SimpleHoster): url_p = urlparse.urlparse(self.pyfile.url) baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - links = [urlparse.urljoin(baseurl, link) if not urlparse.urlparse(link).scheme else link \ - for link in re.findall(self.LINK_PATTERN, self.html)] - - return [html_unescape(l.decode('unicode-escape').strip()) for l in links] + return [urlparse.urljoin(baseurl, link) if not urlparse.urlparse(link).scheme else link \ + for link in re.findall(self.LINK_PATTERN, self.html)] def handlePages(self, pyfile): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index da477ad79..a1a1a4a67 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -133,7 +133,7 @@ def timestamp(): return int(time.time() * 1000) -#@TODO: Move to hoster class in 0.4.10 +#@TODO: Move to Hoster in 0.4.10 def getFileURL(self, url, follow_location=None): link = "" redirect = 1 @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.56" + __version__ = "1.57" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -308,6 +308,11 @@ class SimpleHoster(Hoster): SIZE_REPLACEMENTS = [] URL_REPLACEMENTS = [] + FILE_ERRORS = [('Html error' , r'\A(?:\s*<.+>)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)'), + ('Request error', r'([Aa]n error occured while processing your request)' ), + ('Html file' , r'\A\s*)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)')}) - - if not errmsg: - for r, p in [('Html file' , re.compile(r'\A\s* Date: Mon, 8 Jun 2015 11:14:56 +0200 Subject: [SimpleHoster] Fixup --- module/plugins/internal/SimpleHoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index a1a1a4a67..e04d9d7f5 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.57" + __version__ = "1.58" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -425,8 +425,8 @@ class SimpleHoster(Hoster): def _log(self, type, args): msg = " | ".join([(a.encode("utf-8", "replace") if type(a) is unicode else str(a)).strip() for a in args if a]) logger = getattr(self.core.log, type) - logger("%(plugin)s[%(id)s]: %(msg)s" % {'plugin' : self.__name, - 'id' : self.pyfile.id + logger("%(plugin)s[%(id)s]: %(msg)s" % {'plugin' : self.__name__, + 'id' : self.pyfile.id, 'msg' : msg or _("%s MARK" % type.upper())}) -- cgit v1.2.3 From 2f114a7359d1702c4a22e16ea2a99b132d151d12 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 11:18:48 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1460 --- module/plugins/internal/SimpleHoster.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e04d9d7f5..828367c97 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.58" + __version__ = "1.59" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -587,12 +587,12 @@ class SimpleHoster(Hoster): self.logDebug(_("Checking last downloaded file with custom rules")) if self.CHECK_FILE: - rules = [r, getattr(self, a) for r, a in (("IP blocked" , "IP_BLOCKED_PATTERN" ), - ("Download limit", "DL_LIMIT_PATTERN" ), - ("Size limit" , "SIZE_LIMIT_PATTERN" ), - ("Error" , "ERROR_PATTERN" ), - ("Premium only" , "PREMIUM_ONLY_PATTERN"), - ("Wait error" , "WAIT_PATTERN" ))] + rules = [(r, getattr(self, a)) for r, a in (("IP blocked" , "IP_BLOCKED_PATTERN" ), + ("Download limit", "DL_LIMIT_PATTERN" ), + ("Size limit" , "SIZE_LIMIT_PATTERN" ), + ("Error" , "ERROR_PATTERN" ), + ("Premium only" , "PREMIUM_ONLY_PATTERN"), + ("Wait error" , "WAIT_PATTERN" ))] self.FILE_ERRORS.extend(rules) for r, p in self.FILE_ERRORS: -- cgit v1.2.3 From 8d29df1feb4494c98b6aa150b3cb2d628c47a463 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 13:41:16 +0200 Subject: [SimpleHoster] Fixup --- module/plugins/internal/SimpleHoster.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 828367c97..f69669341 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -14,7 +14,7 @@ from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL from module.plugins.internal.Hoster import Hoster from module.plugins.internal.Plugin import Fail, Retry -from module.utils import fixup, fs_encode, html_unescape, parseFileSize +from module.utils import decode, fixup, fs_encode, html_unescape, parseFileSize #@TODO: Adapt and move to PyFile in 0.4.10 @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.59" + __version__ = "1.60" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -423,11 +423,13 @@ class SimpleHoster(Hoster): #@TODO: Move to Hoster in 0.4.10 def _log(self, type, args): - msg = " | ".join([(a.encode("utf-8", "replace") if type(a) is unicode else str(a)).strip() for a in args if a]) + msg = " | ".join((fs_encode(a) if type(a) is unicode else #@NOTE: `fs_encode` -> `encode` in 0.4.10 + decode(a) if type(a) is str else + str(a)).strip() for a in args if a) logger = getattr(self.core.log, type) logger("%(plugin)s[%(id)s]: %(msg)s" % {'plugin' : self.__name__, 'id' : self.pyfile.id, - 'msg' : msg or _("%s MARK" % type.upper())}) + 'msg' : msg or _(type.upper() + " MARK")}) #@TODO: Move to Hoster in 0.4.10 @@ -456,12 +458,10 @@ class SimpleHoster(Hoster): return self._log("critical", args) - #@TODO: Move to Hoster in 0.4.10 def setup(self): self.resumeDownload = self.multiDL = self.premium - #@TODO: Move to Hoster in 0.4.10 def prepare(self): self.pyfile.error = "" #@TODO: Remove in 0.4.10 -- cgit v1.2.3 From 71ccba71175238b62845b88fc707ff3d5432c56c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 15:14:24 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1464 --- module/plugins/internal/SimpleHoster.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index f69669341..38e2641c9 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.60" + __version__ = "1.61" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -423,13 +423,13 @@ class SimpleHoster(Hoster): #@TODO: Move to Hoster in 0.4.10 def _log(self, type, args): - msg = " | ".join((fs_encode(a) if type(a) is unicode else #@NOTE: `fs_encode` -> `encode` in 0.4.10 - decode(a) if type(a) is str else - str(a)).strip() for a in args if a) + msg = " | ".join((fs_encode(a) if isinstance(a, unicode) else + decode(a) if isinstance(a, str) else + str(a)).strip() for a in args if a) #@NOTE: `fs_encode` -> `encode` in 0.4.10 logger = getattr(self.core.log, type) - logger("%(plugin)s[%(id)s]: %(msg)s" % {'plugin' : self.__name__, - 'id' : self.pyfile.id, - 'msg' : msg or _(type.upper() + " MARK")}) + logger("%(plugin)s[%(id)s]: %(msg)s" % {'plugin': self.__name__, + 'id' : self.pyfile.id, + 'msg' : msg or _(type.upper() + " MARK")}) #@TODO: Move to Hoster in 0.4.10 -- cgit v1.2.3 From a8c8ace5ee4868f58067c8af30ea5702a2434999 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 21:26:17 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1467 --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 38e2641c9..78d30be91 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.61" + __version__ = "1.62" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -592,7 +592,7 @@ class SimpleHoster(Hoster): ("Size limit" , "SIZE_LIMIT_PATTERN" ), ("Error" , "ERROR_PATTERN" ), ("Premium only" , "PREMIUM_ONLY_PATTERN"), - ("Wait error" , "WAIT_PATTERN" ))] + ("Wait error" , "WAIT_PATTERN" )) if hasattr(self, a)] self.FILE_ERRORS.extend(rules) for r, p in self.FILE_ERRORS: -- cgit v1.2.3 From 4745af09d1095ae8cb5f40e135b44ce430193139 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 22:46:33 +0200 Subject: [SimpleCrypter] Fix handleDirect loop --- module/plugins/internal/SimpleCrypter.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 00b872b69..923775e32 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -11,7 +11,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.51" + __version__ = "0.52" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] @@ -21,7 +21,6 @@ class SimpleCrypter(Crypter, SimpleHoster): __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - """ Following patterns should be defined by each crypter: @@ -53,9 +52,6 @@ class SimpleCrypter(Crypter, SimpleHoster): return the html of the page number page_n """ - LINK_PATTERN = None - - #@TODO: Remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") @@ -92,12 +88,17 @@ class SimpleCrypter(Crypter, SimpleHoster): def handleDirect(self, pyfile): - while True: - header = self.load(self.link or pyfile.url, just_header=True, decode=True) + for i in xrange(10): #@TODO: Use `pycurl.MAXREDIRS` value in 0.4.10 + redirect = self.link or pyfile.url + self.logDebug("Redirect #%d to: %s" % (i, redirect)) + + header = self.load(redirect, just_header=True, decode=True) if 'location' in header and header['location']: self.link = header['location'] else: break + else: + self.logError(_("Too many redirects")) def decrypt(self, pyfile): -- cgit v1.2.3 From 9ac5219308e243711afc0ef99158ace05fd76428 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 22:47:00 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1468 --- module/plugins/internal/SimpleHoster.py | 28 +++++++++++++++++++++------- module/plugins/internal/XFSHoster.py | 3 +-- 2 files changed, 22 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 78d30be91..6f2dc8a6c 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.62" + __version__ = "1.63" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -249,7 +249,6 @@ class SimpleHoster(Hoster): __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - """ Info patterns: @@ -258,6 +257,7 @@ class SimpleHoster(Hoster): or NAME_PATTERN: (mandatory) Name that will be set for the file example: NAME_PATTERN = r'(?Pfile_name)' + SIZE_PATTERN: (mandatory) Size that will be checked for the file example: SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' @@ -297,11 +297,14 @@ class SimpleHoster(Hoster): Instead overriding handleFree and handlePremium methods you may define the following patterns for basic link handling: - LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download - example: LINK_FREE_PATTERN = r'
|<[Cc]enter>|>Error|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))' LINK_LEECH_PATTERN = r'

Download Link

\s*]*>([^<]+)' - LINK_PATTERN = None #: final download url pattern CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' CAPTCHA_BLOCK_PATTERN = r'>Enter code.*?(.+?)
' @@ -112,7 +111,7 @@ class XFSHoster(SimpleHoster): self.logError(data['op'] if 'op' in data else _("UNKNOWN")) return "" - self.link = m.group(1).strip() #@TODO: Remove .strip() in 0.4.10 + self.link = m.group(1).strip() #@TODO: Remove `.strip()` in 0.4.10 def handlePremium(self, pyfile): -- cgit v1.2.3 From da1180bc2d428ad52fbbecda8473fc1fb7267438 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 9 Jun 2015 01:52:24 +0200 Subject: [SimpleHoster] Improve checkFile --- module/plugins/internal/DeadCrypter.py | 2 +- module/plugins/internal/DeadHoster.py | 2 +- module/plugins/internal/SimpleHoster.py | 52 +++++++++++++++------------------ 3 files changed, 26 insertions(+), 30 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 28ead4c23..ef0d12b91 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -11,7 +11,7 @@ class DeadCrypter(Crypter): __pattern__ = r'^unmatchable$' - __description__ = """ Crypter is no longer available """ + __description__ = """Crypter is no longer available""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 6dc860eea..accb15a78 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -11,7 +11,7 @@ class DeadHoster(Hoster): __pattern__ = r'^unmatchable$' - __description__ = """ Hoster is no longer available """ + __description__ = """Hoster is no longer available""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6f2dc8a6c..2454a5072 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.63" + __version__ = "1.64" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -581,47 +581,43 @@ class SimpleHoster(Hoster): def checkFile(self): + lastDownload = fs_encode(self.lastDownload) + if self.cTask and not self.lastDownload: self.invalidCaptcha() self.retry(10, reason=_("Wrong captcha")) - elif not self.lastDownload or not os.path.exists(fs_encode(self.lastDownload)): + elif not self.lastDownload or not os.path.exists(lastDownload): self.lastDownload = "" self.error(self.pyfile.error or _("No file downloaded")) else: - self.logDebug(_("Checking last downloaded file with built-in rules")) - errmsg = self.checkDownload({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}) - - if errmsg: - return - - self.logDebug(_("Checking last downloaded file with custom rules")) - - if self.CHECK_FILE: - rules = [(r, getattr(self, a)) for r, a in (("IP blocked" , "IP_BLOCKED_PATTERN" ), - ("Download limit", "DL_LIMIT_PATTERN" ), - ("Size limit" , "SIZE_LIMIT_PATTERN" ), - ("Error" , "ERROR_PATTERN" ), - ("Premium only" , "PREMIUM_ONLY_PATTERN"), - ("Wait error" , "WAIT_PATTERN" )) if hasattr(self, a)] - self.FILE_ERRORS.extend(rules) + #@TODO: Move to Hoster in 0.4.10 + if os.stat(lastDownload).st_size < 1 or self.checkDownload({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}): + self.error(_("Empty file")) + self.logDebug("Checking last downloaded file with built-in rules") for r, p in self.FILE_ERRORS: errmsg = self.checkDownload({r: re.compile(p)}) + if errmsg is not None: + errmsg = errmsg.strip().capitalize() - if not errmsg: - continue - - errmsg = errmsg.strip().capitalize() + try: + errmsg += " | " + self.lastCheck.group(1).strip() + except Exception: + pass - try: - errmsg += " | " + self.lastCheck.group(1).strip() - except Exception: - pass + self.logWarning("Check result: " + errmsg, "Waiting 1 minute and retry") + self.wantReconnect = True + self.retry(wait_time=60, reason=errmsg) + else: + if self.CHECK_FILE: + self.logDebug("Checking last downloaded file with custom rules") + with open(lastDownload, "rb") as f: + self.html = f.read(50000) #@TODO: Recheck in 0.4.10 + self.checkErrors() - self.logWarning("Check result: " + errmsg, "Waiting 1 minute and retry") - self.retry(3, 60, errmsg) + self.logDebug("No file errors found") def checkErrors(self): -- cgit v1.2.3 From c9bae62b1cd55217d2cb7b690fc5c3c67fbabaec Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 9 Jun 2015 04:19:32 +0200 Subject: [SimpleHoster] Fixup (3) --- module/plugins/internal/SimpleHoster.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2454a5072..3d4733790 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import with_statement + import datetime import mimetypes import os @@ -239,7 +241,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.64" + __version__ = "1.65" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), -- cgit v1.2.3 From d2e46f208737760c65a58bfc21ad8fd0809c0b18 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 9 Jun 2015 13:58:27 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1472 --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3d4733790..49b29e280 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -430,9 +430,9 @@ class SimpleHoster(Hoster): #@TODO: Move to Hoster in 0.4.10 def _log(self, type, args): - msg = " | ".join((fs_encode(a) if isinstance(a, unicode) else + msg = " | ".join((fs_encode(a) if isinstance(a, unicode) else #@NOTE: `fs_encode` -> `encode` in 0.4.10 decode(a) if isinstance(a, str) else - str(a)).strip() for a in args if a) #@NOTE: `fs_encode` -> `encode` in 0.4.10 + str(a)).strip() for a in args if a) logger = getattr(self.core.log, type) logger("%(plugin)s[%(id)s]: %(msg)s" % {'plugin': self.__name__, 'id' : self.pyfile.id, -- cgit v1.2.3 From 6faa98072d3bae3d3f2cbf5f7dfa2c9de61da87e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 9 Jun 2015 14:27:17 +0200 Subject: [SimpleHoster] preload referer to False --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 49b29e280..25a35d9d8 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -241,7 +241,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.65" + __version__ = "1.66" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -514,7 +514,7 @@ class SimpleHoster(Hoster): def preload(self): - self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING) + self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), ref=False, decode=not self.TEXT_ENCODING) if isinstance(self.TEXT_ENCODING, basestring): self.html = unicode(self.html, self.TEXT_ENCODING) -- cgit v1.2.3 From 85e7ad6634399aa54fcbb896ca992a7d773ebd77 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 9 Jun 2015 22:47:21 +0300 Subject: Update SimpleCrypter.py --- module/plugins/internal/SimpleCrypter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 923775e32..be0d5e9d5 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -11,7 +11,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.52" + __version__ = "0.53" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] @@ -122,7 +122,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.logDebug("Package has %d links" % len(self.links)) if self.links: - self.links = [html_unescape(l.decode('unicode-escape').strip()) for l in links] #@TODO: Move to Crypter in 0.4.10 + self.links = [html_unescape(l.decode('unicode-escape').strip()) for l in self.links] #@TODO: Move to Crypter in 0.4.10 self.packages = [(self.info['name'], self.links, self.info['folder'])] elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 -- cgit v1.2.3 From d462f5224bf4ea895e813b97a3ab993c83ea0ac3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 10 Jun 2015 01:45:05 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1479 --- module/plugins/internal/SimpleHoster.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 25a35d9d8..4a3b7dcf7 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -429,14 +429,14 @@ class SimpleHoster(Hoster): #@TODO: Move to Hoster in 0.4.10 - def _log(self, type, args): + def _log(self, level, args): + log = getattr(self.core.log, level) msg = " | ".join((fs_encode(a) if isinstance(a, unicode) else #@NOTE: `fs_encode` -> `encode` in 0.4.10 decode(a) if isinstance(a, str) else str(a)).strip() for a in args if a) - logger = getattr(self.core.log, type) - logger("%(plugin)s[%(id)s]: %(msg)s" % {'plugin': self.__name__, - 'id' : self.pyfile.id, - 'msg' : msg or _(type.upper() + " MARK")}) + log("%(plugin)s[%(id)s]: %(msg)s" % {'plugin': self.__name__, + 'id' : self.pyfile.id, + 'msg' : msg or _(level.upper() + " MARK")}) #@TODO: Move to Hoster in 0.4.10 -- cgit v1.2.3 From aa2db98aab37450a20b3640cf5df0e12236acaec Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 10 Jun 2015 10:46:04 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1473 --- module/plugins/internal/Plugin.py | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index e1de48f72..228685ee5 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -31,6 +31,7 @@ if os.name != "nt": from itertools import islice +from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload from module.utils import save_join, save_path, fs_encode, fs_decode def chunks(iterable, size): @@ -41,26 +42,6 @@ def chunks(iterable, size): item = list(islice(it, size)) -class Abort(Exception): - """ raised when aborted """ - - -class Fail(Exception): - """ raised when failed """ - - -class Reconnect(Exception): - """ raised when reconnected """ - - -class Retry(Exception): - """ raised when start again from beginning """ - - -class SkipDownload(Exception): - """ raised when download should be skipped """ - - class Base(object): """ A Base class with log/config/db methods *all* plugin types can use @@ -142,7 +123,7 @@ class Plugin(Base): Overwrite `process` / `decrypt` in your subclassed plugin. """ __name__ = "Plugin" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = None __type__ = "hoster" __config__ = [("name", "type", "desc", "default")] -- cgit v1.2.3 From ea27af7361bb75d665c29120d9cdc254a209afa8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 14 Jun 2015 17:43:31 +0200 Subject: Move new log functions to Plugin --- module/plugins/internal/Plugin.py | 35 ++++++++++++++++++++-------- module/plugins/internal/SimpleHoster.py | 41 ++------------------------------- 2 files changed, 28 insertions(+), 48 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 228685ee5..aef8fd450 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -31,8 +31,8 @@ if os.name != "nt": from itertools import islice -from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload -from module.utils import save_join, save_path, fs_encode, fs_decode +from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload #@TODO: Remove in 0.4.10 +from module.utils import decode, save_join, save_path, fs_encode, fs_decode def chunks(iterable, size): it = iter(iterable) @@ -55,18 +55,35 @@ class Base(object): #: core config self.config = core.config - #log functions + # Log functions + def _log(self, level, args): + log = getattr(self.core.log, level) + msg = " | ".join((fs_encode(a) if isinstance(a, unicode) else #@NOTE: `fs_encode` -> `encode` in 0.4.10 + str(a)).strip() for a in args if a) + log("%(plugin)s[%(id)s]: %(msg)s" % {'plugin': self.__name__, + 'id' : self.pyfile.id, + 'msg' : msg or _(level.upper() + " MARK")}) + + + def logDebug(self, *args): + if self.core.debug: + return self._log("debug", args) + + def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + return self._log("info", args) + def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + return self._log("warning", args) + def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + return self._log("error", args) - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logCritical(self, *args): + return self._log("critical", args) def setConf(self, option, value): @@ -123,7 +140,7 @@ class Plugin(Base): Overwrite `process` / `decrypt` in your subclassed plugin. """ __name__ = "Plugin" - __version__ = "0.05" + __version__ = "0.06" __pattern__ = None __type__ = "hoster" __config__ = [("name", "type", "desc", "default")] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 4a3b7dcf7..d1563ac89 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -16,7 +16,7 @@ from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL from module.plugins.internal.Hoster import Hoster from module.plugins.internal.Plugin import Fail, Retry -from module.utils import decode, fixup, fs_encode, html_unescape, parseFileSize +from module.utils import fixup, fs_encode, html_unescape, parseFileSize #@TODO: Adapt and move to PyFile in 0.4.10 @@ -241,7 +241,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.66" + __version__ = "1.67" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -428,43 +428,6 @@ class SimpleHoster(Hoster): return info - #@TODO: Move to Hoster in 0.4.10 - def _log(self, level, args): - log = getattr(self.core.log, level) - msg = " | ".join((fs_encode(a) if isinstance(a, unicode) else #@NOTE: `fs_encode` -> `encode` in 0.4.10 - decode(a) if isinstance(a, str) else - str(a)).strip() for a in args if a) - log("%(plugin)s[%(id)s]: %(msg)s" % {'plugin': self.__name__, - 'id' : self.pyfile.id, - 'msg' : msg or _(level.upper() + " MARK")}) - - - #@TODO: Move to Hoster in 0.4.10 - def logDebug(self, *args): - if self.core.debug: - return self._log("debug", args) - - - #@TODO: Move to Hoster in 0.4.10 - def logInfo(self, *args): - return self._log("info", args) - - - #@TODO: Move to Hoster in 0.4.10 - def logWarning(self, *args): - return self._log("warning", args) - - - #@TODO: Move to Hoster in 0.4.10 - def logError(self, *args): - return self._log("error", args) - - - #@TODO: Move to Hoster in 0.4.10 - def logCritical(self, *args): - return self._log("critical", args) - - def setup(self): self.resumeDownload = self.multiDL = self.premium -- cgit v1.2.3 From 117e670d59db7f8d6047562f69cb152a1e2dee53 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 14 Jun 2015 18:35:38 +0200 Subject: Update Hook --- module/plugins/internal/Hook.py | 175 +++++++++++++++++++++++----------------- 1 file changed, 102 insertions(+), 73 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 5f2639dce..5f7f3db55 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -1,161 +1,190 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. +import traceback - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. +from module.plugins.internal.Plugin import Plugin - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay - @interface-version: 0.2 -""" - -from traceback import print_exc - -from Plugin import Base class Expose(object): - """ used for decoration to declare rpc services """ + """Used for decoration to declare rpc services""" def __new__(cls, f, *args, **kwargs): hookManager.addRPC(f.__module__, f.func_name, f.func_doc) return f -def threaded(f): - def run(*args,**kwargs): - hookManager.startThread(f, *args, **kwargs) + +def threaded(fn): + + def run(*args, **kwargs): + hookManager.startThread(fn, *args, **kwargs) + return run -class Hook(Base): - """ - Base class for hook plugins. - """ - __name__ = "Hook" - __version__ = "0.02" - __type__ = "hook" - __threaded__ = [] - __config__ = [ ("name", "type", "desc" , "default") ] - __description__ = """interface for hook""" - __author_name__ = ("mkaay", "RaNaN") - __author_mail__ = ("mkaay@mkaay.de", "RaNaN@pyload.org") - #: automatically register event listeners for functions, attribute will be deleted dont use it yourself - event_map = None +class Hook(Plugin): + __name__ = "Hook" + __type__ = "hook" + __version__ = "0.03" - # Alternative to event_map - #: List of events the plugin can handle, name the functions exactly like eventname. - event_list = None # dont make duplicate entries in event_map + __config__ = [] #: [("name", "type", "desc", "default")] + __description__ = """Base hook plugin""" + __license__ = "GPLv3" + __authors__ = [("mkaay" , "mkaay@mkaay.de" ), + ("RaNaN" , "RaNaN@pyload.org" ), + ("Walter Purcaro", "vuolter@gmail.com")] - #: periodic call interval in secondc - interval = 60 def __init__(self, core, manager): - Base.__init__(self, core) - #: Provide information in dict here, usable by API `getInfo` - self.info = None + self.info = {} - #: Callback of periodical job task, used by hookmanager - self.cb = None + #: Callback of periodical job task, used by HookManager + self.cb = None + self.interval = 60 #: `HookManager` self.manager = manager - #register events + self.setup() + + #: automatically register event listeners for functions, attribute will be deleted dont use it yourself + self.event_map = {} + + # Deprecated alternative to event_map + #: List of events the plugin can handle, name the functions exactly like eventname. + self.event_list = [] #@NOTE: dont make duplicate entries in event_map + + # register events if self.event_map: for event, funcs in self.event_map.iteritems(): if type(funcs) in (list, tuple): for f in funcs: - self.manager.addEvent(event, getattr(self,f)) + self.manager.addEvent(event, getattr(self, f)) else: - self.manager.addEvent(event, getattr(self,funcs)) + self.manager.addEvent(event, getattr(self, funcs)) - #delete for various reasons + # delete for various reasons self.event_map = None if self.event_list: + self.logWarning(_("Deprecated method `event_list`, use `event_map` instead")) + for f in self.event_list: - self.manager.addEvent(f, getattr(self,f)) + self.manager.addEvent(f, getattr(self, f)) self.event_list = None - self.initPeriodical() - self.setup() - def initPeriodical(self): - if self.interval >=1: - self.cb = self.core.scheduler.addJob(0, self._periodical, threaded=False) + def initPeriodical(self, delay=0, threaded=False): + self.cb = self.core.scheduler.addJob(max(0, delay), self._periodical, [threaded], threaded=threaded) + + + def _periodical(self, threaded): + if self.interval < 0: + self.cb = None + return - def _periodical(self): try: - if self.isActivated(): self.periodical() + self.periodical() + except Exception, e: - self.core.log.error(_("Error executing hooks: %s") % str(e)) + self.logError(_("Error executing hook: %s") % e) if self.core.debug: - print_exc() + traceback.print_exc() - self.cb = self.core.scheduler.addJob(self.interval, self._periodical, threaded=False) + self.cb = self.core.scheduler.addJob(self.interval, self._periodical, [threaded], threaded=threaded) def __repr__(self): return "" % self.__name__ + def setup(self): - """ more init stuff if needed """ + """More init stuff if needed""" pass - def unload(self): - """ called when hook was deactivated """ - pass def isActivated(self): - """ checks if hook is activated""" - return self.config.getPlugin(self.__name__, "activated") + """Checks if hook is activated""" + return self.getConfig("activated") + + + def deactivate(self): + """Called when hook was deactivated""" + pass + + #: Deprecated, use method `deactivate` instead + def unload(self): + self.logWarning(_("Deprecated method `unload`, use `deactivate` instead")) + return self.deactivate() - #event methods - overwrite these if needed + + def activate(self): + """Called when hook was activated""" + pass + + + #: Deprecated, use method `activate` instead def coreReady(self): + self.logWarning(_("Deprecated method `coreReady`, use `activate` instead")) + return self.activate() + + + def exit(self): + """Called by core.shutdown just before pyLoad exit""" pass + + #: Deprecated, use method `exit` instead def coreExiting(self): - pass + self.logWarning(_("Deprecated method `coreExiting`, use `exit` instead")) + return self.exit() + def downloadPreparing(self, pyfile): pass + def downloadFinished(self, pyfile): pass + def downloadFailed(self, pyfile): pass + def packageFinished(self, pypack): pass + def beforeReconnecting(self, ip): pass + def afterReconnecting(self, ip): pass + def periodical(self): pass - def newCaptchaTask(self, task): - """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """ + + def captchaTask(self, task): + """New captcha task for the plugin, it MUST set the handler and timeout or will be ignored""" pass + + #: Deprecated, use method `captchaTask` instead + def newCaptchaTask(self, task): + self.logWarning(_("Deprecated method `newCaptchaTask`, use `captchaTask` instead")) + return self.captchaTask() + + def captchaCorrect(self, task): pass + def captchaInvalid(self, task): - pass \ No newline at end of file + pass -- cgit v1.2.3 From 83db7b72d85b74f0248717444a622156fb4d43b9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 14 Jun 2015 18:37:18 +0200 Subject: Update Crypter --- module/plugins/internal/Crypter.py | 127 +++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 46 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index b87ed1882..182354287 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -1,35 +1,36 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from module.plugins.internal.Plugin import Plugin - -class Crypter(Plugin): - __name__ = "Crypter" - __version__ = "0.02" - __pattern__ = None - __type__ = "container" - __description__ = """Base crypter plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") +import urlparse + +from module.plugins.internal.Hoster import Hoster +from module.utils import decode, save_path + + +class Crypter(Hoster): + """ + Base plugin for crypter. + Overwrite `decrypt` in your subclassed plugin. + """ + + __name__ = "Crypter" + __type__ = "crypter" + __version__ = "0.03" + + __pattern__ = r'^unmatchable$' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config.get("general", "folder_per_package") + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Base decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + html = None #: last html loaded #@TODO: Move to Hoster + def __init__(self, pyfile): - Plugin.__init__(self, pyfile) + #: Provide information in dict here + self.info = {} #@TODO: Move to Plugin #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) self.packages = [] @@ -37,36 +38,70 @@ class Crypter(Plugin): #: List of urls, pyLoad will generate packagenames self.urls = [] - self.multiDL = True - self.limitDL = 0 + Plugin.__init__(self, pyfile) + + def process(self, pyfile): + """Main method""" - def preprocessing(self, thread): - """prepare""" - self.setup() - self.thread = thread + self.decrypt(pyfile) - self.decrypt(self.pyfile) + if self.urls: + self._generate_packages() + + elif not self.packages: + self.error(_("No link grabbed"), "decrypt") - self.createPackages() + self._create_packages() def decrypt(self, pyfile): raise NotImplementedError - def createPackages(self): - """ create new packages from self.packages """ - for pack in self.packages: - self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } ) + def _generate_packages(self): + """Generate new packages from self.urls""" - links = [x.decode("utf-8") for x in pack[1]] + packages = [(name, links, None) for name, links in self.core.api.generatePackages(self.urls).iteritems()] + self.packages.extend(packages) - pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue) - if self.pyfile.package().password: - self.core.api.setPackageData(pid, {"password": self.pyfile.package().password}) + def _create_packages(self): + """Create new packages from self.packages""" - if self.urls: - self.core.api.generateAndAddPackages(self.urls) + package_folder = self.pyfile.package().folder + package_password = self.pyfile.package().password + package_queue = self.pyfile.package().queue + + folder_per_package = self.core.config.get('general', 'folder_per_package') + use_subfolder = self.getConfig('use_subfolder', folder_per_package) + subfolder_per_package = self.getConfig('subfolder_per_package', True) + + for name, links, folder in self.packages: + self.logDebug("Parsed package: %s" % name, + "%d links" % len(links), + "Saved to folder: %s" % folder if folder else "Saved to download folder") + + links = map(decode, links) + + pid = self.core.api.addPackage(name, links, package_queue) + + if package_password: + self.core.api.setPackageData(pid, {"password": package_password}) + + setFolder = lambda x: self.core.api.setPackageData(pid, {"folder": x or ""}) #@NOTE: Workaround to do not break API addPackage method + + if use_subfolder: + if not subfolder_per_package: + setFolder(package_folder) + self.logDebug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) + + elif not folder_per_package or name != folder: + if not folder: + folder = urlparse.urlparse(name).path.split("/")[-1] + + setFolder(save_path(folder)) + self.logDebug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) + elif folder_per_package: + setFolder(None) -- cgit v1.2.3 From bae390c536f643932581b6be00c11cc32e84c171 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 14 Jun 2015 19:11:26 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1492 and https://github.com/pyload/pyload/issues/1493 --- module/plugins/internal/Plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index aef8fd450..bbf3191c3 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -61,7 +61,7 @@ class Base(object): msg = " | ".join((fs_encode(a) if isinstance(a, unicode) else #@NOTE: `fs_encode` -> `encode` in 0.4.10 str(a)).strip() for a in args if a) log("%(plugin)s[%(id)s]: %(msg)s" % {'plugin': self.__name__, - 'id' : self.pyfile.id, + 'id' : self.pyfile.id if hasattr(self, 'pyfile') else "", 'msg' : msg or _(level.upper() + " MARK")}) @@ -140,7 +140,7 @@ class Plugin(Base): Overwrite `process` / `decrypt` in your subclassed plugin. """ __name__ = "Plugin" - __version__ = "0.06" + __version__ = "0.07" __pattern__ = None __type__ = "hoster" __config__ = [("name", "type", "desc", "default")] -- cgit v1.2.3 From 3eff29b8f0c6feca5657220f19f6a3ca37a7ec2e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 14 Jun 2015 19:27:37 +0200 Subject: Update Account --- module/plugins/internal/Account.py | 212 ++++++++++++++++++++----------------- 1 file changed, 116 insertions(+), 96 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 7632d15c2..654296f32 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -1,29 +1,13 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. +import random +import threading +import time +import traceback - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. +from module.plugins.internal.Plugin import Base +from module.utils import compare_time, lock, parse_size - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from random import choice -from time import time -from traceback import print_exc -from threading import RLock - -from Plugin import Base -from module.utils import compare_time, parseFileSize, lock class WrongPassword(Exception): pass @@ -35,36 +19,37 @@ class Account(Base): Just overwrite `login` and cookies will be stored and account becomes accessible in\ associated hoster plugin. Plugin should also provide `loadAccountInfo` """ - __name__ = "Account" - __version__ = "0.02" - __type__ = "account" - __description__ = """Account Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + __name__ = "Account" + __type__ = "account" + __version__ = "0.03" - #: after that time [in minutes] pyload will relogin the account - login_timeout = 600 - #: account data will be reloaded after this time - info_threshold = 600 + __description__ = """Base account plugin""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de")] def __init__(self, manager, accounts): - Base.__init__(self, manager.core) - self.manager = manager self.accounts = {} - self.infos = {} # cache for account information - self.lock = RLock() - + self.infos = {} #: cache for account information + self.lock = threading.RLock() self.timestamps = {} - self.setAccounts(accounts) + + self.login_timeout = 10 * 60 #: after that time (in minutes) pyload will relogin the account + self.info_threshold = 10 * 60 #: after that time (in minutes) account data will be reloaded + self.init() + self.setAccounts(accounts) + + def init(self): pass + def login(self, user, data, req): - """login into account, the cookies will be saved so user can be recognized + """ + Login into account, the cookies will be saved so user can be recognized :param user: loginname :param data: data dictionary @@ -72,29 +57,38 @@ class Account(Base): """ pass + @lock def _login(self, user, data): # set timestamp for login - self.timestamps[user] = time() + self.timestamps[user] = time.time() req = self.getAccountRequest(user) try: self.login(user, data, req) + except WrongPassword: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": _("Wrong Password")}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": user, + "msg": _("Wrong Password")}) + success = data['valid'] = False except Exception, e: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": e}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": user, + "msg": e}) + success = data['valid'] = False if self.core.debug: - print_exc() + traceback.print_exc() + + else: + success = True + finally: - if req: req.close() + if req: + req.close() + return success + def relogin(self, user): req = self.getAccountRequest(user) @@ -102,9 +96,10 @@ class Account(Base): req.cj.clear() req.close() if user in self.infos: - del self.infos[user] #delete old information + del self.infos[user] #: delete old information + + return self._login(user, self.accounts[user]) - self._login(user, self.accounts[user]) def setAccounts(self, accounts): self.accounts = accounts @@ -112,24 +107,26 @@ class Account(Base): self._login(user, data) self.infos[user] = {} + def updateAccounts(self, user, password=None, options={}): - """ updates account and return true if anything changed """ + """Updates account and return true if anything changed""" if user in self.accounts: - self.accounts[user]["valid"] = True #do not remove or accounts will not login + self.accounts[user]['valid'] = True #: do not remove or accounts will not login if password: - self.accounts[user]["password"] = password + self.accounts[user]['password'] = password self.relogin(user) return True if options: - before = self.accounts[user]["options"] - self.accounts[user]["options"].update(options) - return self.accounts[user]["options"] != before + before = self.accounts[user]['options'] + self.accounts[user]['options'].update(options) + return self.accounts[user]['options'] != before else: self.accounts[user] = {"password": password, "options": options, "valid": True} self._login(user, self.accounts[user]) return True + def removeAccount(self, user): if user in self.accounts: del self.accounts[user] @@ -138,9 +135,11 @@ class Account(Base): if user in self.timestamps: del self.timestamps[user] + @lock def getAccountInfo(self, name, force=False): - """retrieve account infos for an user, do **not** overwrite this method!\\ + """ + Retrieve account infos for an user, do **not** overwrite this method!\\ just use it to retrieve infos in hoster plugins. see `loadAccountInfo` :param name: username @@ -159,49 +158,54 @@ class Account(Base): raise Exception("Wrong return format") except Exception, e: infos = {"error": str(e)} + if self.core.debug: + traceback.print_exc() - if req: req.close() + if req: + req.close() - self.logDebug("Account Info: %s" % str(infos)) + self.logDebug("Account Info: %s" % infos) - infos["timestamp"] = time() + infos['timestamp'] = time.time() self.infos[name] = infos - elif "timestamp" in self.infos[name] and self.infos[name][ - "timestamp"] + self.info_threshold * 60 < time(): + elif "timestamp" in self.infos[name] and self.infos[name]['timestamp'] + self.info_threshold * 60 < time.time(): self.logDebug("Reached timeout for account data") self.scheduleRefresh(name) data.update(self.infos[name]) return data + def isPremium(self, user): info = self.getAccountInfo(user) - return info["premium"] + return info['premium'] + def loadAccountInfo(self, name, req=None): - """this should be overwritten in account plugin,\ + """ + This should be overwritten in account plugin,\ and retrieving account information for user :param name: :param req: `Request` instance :return: """ - return { - "validuntil": None, # -1 for unlimited - "login": name, - #"password": self.accounts[name]["password"], #@XXX: security - "options": self.accounts[name]["options"], - "valid": self.accounts[name]["valid"], - "trafficleft": None, # in kb, -1 for unlimited - "maxtraffic": None, - "premium": True, #useful for free accounts - "timestamp": 0, #time this info was retrieved - "type": self.__name__, - } + return {"validuntil" : None, #: -1 for unlimited + "login" : name, + # "password" : self.accounts[name]['password'], #: commented due security reason + "options" : self.accounts[name]['options'], + "valid" : self.accounts[name]['valid'], + "trafficleft": None, #: in bytes, -1 for unlimited + "maxtraffic" : None, + "premium" : None, + "timestamp" : 0, #: time this info was retrieved + "type" : self.__name__} + def getAllAccounts(self, force=False): return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] + def getAccountRequest(self, user=None): if not user: user, data = self.selectAccount() @@ -211,6 +215,7 @@ class Account(Base): req = self.core.requestFactory.getRequest(self.__name__, user) return req + def getAccountCookies(self, user=None): if not user: user, data = self.selectAccount() @@ -220,47 +225,58 @@ class Account(Base): cj = self.core.requestFactory.getCookieJar(self.__name__, user) return cj + def getAccountData(self, user): return self.accounts[user] + def selectAccount(self): - """ returns an valid account name and data""" + """Returns an valid account name and data""" usable = [] for user, data in self.accounts.iteritems(): - if not data["valid"]: continue + if not data['valid']: + continue - if "time" in data["options"] and data["options"]["time"]: + if "time" in data['options'] and data['options']['time']: time_data = "" try: - time_data = data["options"]["time"][0] + time_data = data['options']['time'][0] start, end = time_data.split("-") if not compare_time(start.split(":"), end.split(":")): continue - except: + except Exception: self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) if user in self.infos: if "validuntil" in self.infos[user]: - if self.infos[user]["validuntil"] > 0 and time() > self.infos[user]["validuntil"]: + if self.infos[user]['validuntil'] > 0 and time.time() > self.infos[user]['validuntil']: continue if "trafficleft" in self.infos[user]: - if self.infos[user]["trafficleft"] == 0: + if self.infos[user]['trafficleft'] == 0: continue usable.append((user, data)) - if not usable: return None, None - return choice(usable) + if not usable: + return None, None + + return random.choice(usable) + def canUse(self): - return False if self.selectAccount() == (None, None) else True + return self.selectAccount() != (None, None) + + + def parseTraffic(self, value, unit=None): #: return bytes + if not unit and not isinstance(value, basestring): + unit = "KB" + return parseFileSize(value, unit) - def parseTraffic(self, string): #returns kbyte - return parseFileSize(string) / 1024 def wrongPassword(self): raise WrongPassword + def empty(self, user): if user in self.infos: self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) @@ -268,25 +284,29 @@ class Account(Base): self.infos[user].update({"trafficleft": 0}) self.scheduleRefresh(user, 30 * 60) + def expired(self, user): if user in self.infos: self.logWarning(_("Account %s is expired, checking again in 1h") % user) - self.infos[user].update({"validuntil": time() - 1}) + self.infos[user].update({"validuntil": time.time() - 1}) self.scheduleRefresh(user, 60 * 60) + def scheduleRefresh(self, user, time=0, force=True): - """ add task to refresh account info to sheduler """ + """Add task to refresh account info to sheduler""" self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) + @lock def checkLogin(self, user): - """ checks if user is still logged in """ + """Checks if user is still logged in""" if user in self.timestamps: - if self.timestamps[user] + self.login_timeout * 60 < time(): + if self.login_timeout > 0 and self.timestamps[user] + self.login_timeout * 60 < time.time(): self.logDebug("Reached login timeout for %s" % user) - self.relogin(user) - return False - - return True + return self.relogin(user) + else: + return True + else: + return False -- cgit v1.2.3 From 310eff44408075a3563284e74f3a72fd82d2c5ab Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 14 Jun 2015 19:29:47 +0200 Subject: Crypter back to Plugin class --- module/plugins/internal/Crypter.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 182354287..3db36ca8e 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -2,11 +2,11 @@ import urlparse -from module.plugins.internal.Hoster import Hoster +from module.plugins.internal.Plugin import Plugin from module.utils import decode, save_path -class Crypter(Hoster): +class Crypter(Plugin): """ Base plugin for crypter. Overwrite `decrypt` in your subclassed plugin. @@ -29,6 +29,8 @@ class Crypter(Hoster): def __init__(self, pyfile): + super(Crypter, self).__init__(self, pyfile) + #: Provide information in dict here self.info = {} #@TODO: Move to Plugin @@ -38,7 +40,8 @@ class Crypter(Hoster): #: List of urls, pyLoad will generate packagenames self.urls = [] - Plugin.__init__(self, pyfile) + self.multiDL = True + self.limitDL = 0 def process(self, pyfile): -- cgit v1.2.3 From 7dc0b955b6e3d4448c173447c700717e8cdbbe95 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 14 Jun 2015 19:43:37 +0200 Subject: Move OCR to internal plugin folder --- module/plugins/internal/OCR.py | 319 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 module/plugins/internal/OCR.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py new file mode 100644 index 000000000..1874ba07d --- /dev/null +++ b/module/plugins/internal/OCR.py @@ -0,0 +1,319 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +try: + from PIL import Image, GifImagePlugin, JpegImagePlugin, PngImagePlugin, TiffImagePlugin + +except ImportError: + import Image, GifImagePlugin, JpegImagePlugin, PngImagePlugin, TiffImagePlugin + +import logging +import os +import subprocess +#import tempfile + +from module.utils import save_join + + +class OCR(object): + __name__ = "OCR" + __type__ = "ocr" + __version__ = "0.11" + + __description__ = """OCR base plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + + + def __init__(self): + self.logger = logging.getLogger("log") + + + def load_image(self, image): + self.image = Image.open(image) + self.pixels = self.image.load() + self.result_captcha = '' + + + def unload(self): + """delete all tmp images""" + pass + + + def threshold(self, value): + self.image = self.image.point(lambda a: a * value + 10) + + + def run(self, command): + """Run a command""" + + popen = subprocess.Popen(command, bufsize = -1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + popen.wait() + output = popen.stdout.read() +" | "+ popen.stderr.read() + popen.stdout.close() + popen.stderr.close() + self.logger.debug("Tesseract ReturnCode %s Output: %s" % (popen.returncode, output)) + + + def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True): + #tmpTif = tempfile.NamedTemporaryFile(suffix=".tif") + try: + tmpTif = open(save_join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") + tmpTif.close() + + #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") + tmpTxt = open(save_join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") + tmpTxt.close() + + except IOError, e: + self.logError(e) + return + + self.logger.debug("save tiff") + self.image.save(tmpTif.name, 'TIFF') + + if os.name == "nt": + tessparams = [os.path.join(pypath, "tesseract", "tesseract.exe")] + else: + tessparams = ["tesseract"] + + tessparams.extend( [os.path.abspath(tmpTif.name), os.path.abspath(tmpTxt.name).replace(".txt", "")] ) + + if subset and (digits or lowercase or uppercase): + #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") + with open(save_join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") as tmpSub: + tmpSub.write("tessedit_char_whitelist ") + + if digits: + tmpSub.write("0123456789") + if lowercase: + tmpSub.write("abcdefghijklmnopqrstuvwxyz") + if uppercase: + tmpSub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + tmpSub.write("\n") + tessparams.append("nobatch") + tessparams.append(os.path.abspath(tmpSub.name)) + + self.logger.debug("run tesseract") + self.run(tessparams) + self.logger.debug("read txt") + + try: + with open(tmpTxt.name, 'r') as f: + self.result_captcha = f.read().replace("\n", "") + except Exception: + self.result_captcha = "" + + self.logger.debug(self.result_captcha) + try: + os.remove(tmpTif.name) + os.remove(tmpTxt.name) + if subset and (digits or lowercase or uppercase): + os.remove(tmpSub.name) + except Exception: + pass + + + def get_captcha(self, name): + raise NotImplementedError + + + def to_greyscale(self): + if self.image.mode != 'L': + self.image = self.image.convert('L') + + self.pixels = self.image.load() + + + def eval_black_white(self, limit): + self.pixels = self.image.load() + w, h = self.image.size + for x in xrange(w): + for y in xrange(h): + if self.pixels[x, y] > limit: + self.pixels[x, y] = 255 + else: + self.pixels[x, y] = 0 + + + def clean(self, allowed): + pixels = self.pixels + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 255: + continue + # No point in processing white pixels since we only want to remove black pixel + count = 0 + + try: + if pixels[x-1, y-1] != 255: + count += 1 + if pixels[x-1, y] != 255: + count += 1 + if pixels[x-1, y + 1] != 255: + count += 1 + if pixels[x, y + 1] != 255: + count += 1 + if pixels[x + 1, y + 1] != 255: + count += 1 + if pixels[x + 1, y] != 255: + count += 1 + if pixels[x + 1, y-1] != 255: + count += 1 + if pixels[x, y-1] != 255: + count += 1 + except Exception: + pass + + # not enough neighbors are dark pixels so mark this pixel + # to be changed to white + if count < allowed: + pixels[x, y] = 1 + + # second pass: this time set all 1's to 255 (white) + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 1: + pixels[x, y] = 255 + + self.pixels = pixels + + + def derotate_by_average(self): + """rotate by checking each angle and guess most suitable""" + + w, h = self.image.size + pixels = self.pixels + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 155 + + highest = {} + counts = {} + + for angle in xrange(-45, 45): + + tmpimage = self.image.rotate(angle) + + pixels = tmpimage.load() + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + + count = {} + + for x in xrange(w): + count[x] = 0 + for y in xrange(h): + if pixels[x, y] == 155: + count[x] += 1 + + sum = 0 + cnt = 0 + + for x in count.values(): + if x != 0: + sum += x + cnt += 1 + + avg = sum / cnt + counts[angle] = cnt + highest[angle] = 0 + for x in count.values(): + if x > highest[angle]: + highest[angle] = x + + highest[angle] = highest[angle] - avg + + hkey = 0 + hvalue = 0 + + for key, value in highest.iteritems(): + if value > hvalue: + hkey = key + hvalue = value + + self.image = self.image.rotate(hkey) + pixels = self.image.load() + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + if pixels[x, y] == 155: + pixels[x, y] = 0 + + self.pixels = pixels + + + def split_captcha_letters(self): + captcha = self.image + started = False + letters = [] + width, height = captcha.size + bottomY, topY = 0, height + pixels = captcha.load() + + for x in xrange(width): + black_pixel_in_col = False + for y in xrange(height): + if pixels[x, y] != 255: + if not started: + started = True + firstX = x + lastX = x + + if y > bottomY: + bottomY = y + if y < topY: + topY = y + if x > lastX: + lastX = x + + black_pixel_in_col = True + + if black_pixel_in_col is False and started is True: + rect = (firstX, topY, lastX, bottomY) + new_captcha = captcha.crop(rect) + + w, h = new_captcha.size + if w > 5 and h > 5: + letters.append(new_captcha) + + started = False + bottomY, topY = 0, height + + return letters + + + def correct(self, values, var=None): + if var: + result = var + else: + result = self.result_captcha + + for key, item in values.iteritems(): + + if key.__class__ == str: + result = result.replace(key, item) + else: + for expr in key: + result = result.replace(expr, item) + + if var: + return result + else: + self.result_captcha = result -- cgit v1.2.3 From 312f6decf9f0f07d46a3470c97707dd74698172c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 14 Jun 2015 19:46:30 +0200 Subject: Hook back to Base class --- module/plugins/internal/Hook.py | 4 ++-- module/plugins/internal/Plugin.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 5f7f3db55..d34435349 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -2,7 +2,7 @@ import traceback -from module.plugins.internal.Plugin import Plugin +from module.plugins.internal.Plugin import Base class Expose(object): @@ -21,7 +21,7 @@ def threaded(fn): return run -class Hook(Plugin): +class Hook(Base): __name__ = "Hook" __type__ = "hook" __version__ = "0.03" diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index bbf3191c3..c13d94aac 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -32,7 +32,7 @@ if os.name != "nt": from itertools import islice from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload #@TODO: Remove in 0.4.10 -from module.utils import decode, save_join, save_path, fs_encode, fs_decode +from module.utils import save_join, save_path, fs_encode, fs_decode def chunks(iterable, size): it = iter(iterable) -- cgit v1.2.3 From da62d00a301076bda1f645f31cc461a7b0f4c674 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 14 Jun 2015 19:56:43 +0200 Subject: Update Container --- module/plugins/internal/Container.py | 77 ++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 44 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 43ca95e77..fa3508ae4 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -1,75 +1,64 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. +from __future__ import with_statement - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" +import os +import re from module.plugins.internal.Crypter import Crypter +from pyload.utils import save_join -from os.path import join, exists, basename -from os import remove -import re class Container(Crypter): - __name__ = "Container" - __version__ = "0.02" - __pattern__ = None - __type__ = "container" - __description__ = """Base container plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + __name__ = "Container" + __type__ = "container" + __version__ = "0.03" + + __pattern__ = r'^unmatchable$' + __config__ = [] #: [("name", "type", "desc", "default")] + + __description__ = """Base container decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("mkaay", "mkaay@mkaay.de")] def preprocessing(self, thread): - """prepare""" + """Prepare""" self.setup() self.thread = thread - self.loadToDisk() + self._load2disk() self.decrypt(self.pyfile) - self.deleteTmp() + self.delete_tmp() - self.createPackages() + self._create_packages() - def loadToDisk(self): - """loads container to disk if its stored remotely and overwrite url, + def _load2disk(self): + """Loads container to disk if its stored remotely and overwrite url, or check existent on several places at disk""" if self.pyfile.url.startswith("http"): self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] content = self.load(self.pyfile.url) - self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name) - f = open(self.pyfile.url, "wb" ) - f.write(content) - f.close() + self.pyfile.url = save_join(self.core.config.get("general", "download_folder"), self.pyfile.name) + try: + with open(self.pyfile.url, "wb") as f: + f.write(content) + except IOError, e: + self.fail(str(e)) else: - self.pyfile.name = basename(self.pyfile.url) - if not exists(self.pyfile.url): - if exists(join(pypath, self.pyfile.url)): - self.pyfile.url = join(pypath, self.pyfile.url) + self.pyfile.name = os.path.basename(self.pyfile.url) + if not os.path.exists(self.pyfile.url): + if os.path.exists(save_join(pypath, self.pyfile.url)): + self.pyfile.url = save_join(pypath, self.pyfile.url) else: - self.fail(_("File not exists.")) + self.fail(_("File not exists")) - def deleteTmp(self): + def delete_tmp(self): if self.pyfile.name.startswith("tmp_"): - remove(self.pyfile.url) - - + os.remove(self.pyfile.url) -- cgit v1.2.3 From 71d5584e82050a66aa5306ce99e43e7cbc3a62a7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 00:17:49 +0200 Subject: Fix new internal plugins --- module/plugins/internal/Account.py | 9 +++------ module/plugins/internal/Crypter.py | 7 +------ module/plugins/internal/Hook.py | 16 ++++++++++------ module/plugins/internal/Plugin.py | 4 ++-- 4 files changed, 16 insertions(+), 20 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 654296f32..27a040e24 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -6,7 +6,7 @@ import time import traceback from module.plugins.internal.Plugin import Base -from module.utils import compare_time, lock, parse_size +from module.utils import compare_time, lock, parseFileSize class WrongPassword(Exception): @@ -14,11 +14,6 @@ class WrongPassword(Exception): class Account(Base): - """ - Base class for every Account plugin. - Just overwrite `login` and cookies will be stored and account becomes accessible in\ - associated hoster plugin. Plugin should also provide `loadAccountInfo` - """ __name__ = "Account" __type__ = "account" __version__ = "0.03" @@ -29,6 +24,8 @@ class Account(Base): def __init__(self, manager, accounts): + super(Account, self).__init__(manager.core) + self.manager = manager self.accounts = {} self.infos = {} #: cache for account information diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 3db36ca8e..5fb9bc5c2 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -7,11 +7,6 @@ from module.utils import decode, save_path class Crypter(Plugin): - """ - Base plugin for crypter. - Overwrite `decrypt` in your subclassed plugin. - """ - __name__ = "Crypter" __type__ = "crypter" __version__ = "0.03" @@ -29,7 +24,7 @@ class Crypter(Plugin): def __init__(self, pyfile): - super(Crypter, self).__init__(self, pyfile) + super(Crypter, self).__init__(pyfile) #: Provide information in dict here self.info = {} #@TODO: Move to Plugin diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index d34435349..6931a7dba 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -26,7 +26,8 @@ class Hook(Base): __type__ = "hook" __version__ = "0.03" - __config__ = [] #: [("name", "type", "desc", "default")] + __config__ = [] #: [("name", "type", "desc", "default")] + __threaded__ = [] #@TODO: Remove in 0.4.10 __description__ = """Base hook plugin""" __license__ = "GPLv3" @@ -36,6 +37,8 @@ class Hook(Base): def __init__(self, core, manager): + super(Hook, self).__init__(core) + #: Provide information in dict here, usable by API `getInfo` self.info = {} @@ -55,7 +58,12 @@ class Hook(Base): #: List of events the plugin can handle, name the functions exactly like eventname. self.event_list = [] #@NOTE: dont make duplicate entries in event_map - # register events + self.initEvents() + # self.initPeriodical(10) + + + def initEvents(self): + # register events if self.event_map: for event, funcs in self.event_map.iteritems(): if type(funcs) in (list, tuple): @@ -117,7 +125,6 @@ class Hook(Base): #: Deprecated, use method `deactivate` instead def unload(self): - self.logWarning(_("Deprecated method `unload`, use `deactivate` instead")) return self.deactivate() @@ -128,7 +135,6 @@ class Hook(Base): #: Deprecated, use method `activate` instead def coreReady(self): - self.logWarning(_("Deprecated method `coreReady`, use `activate` instead")) return self.activate() @@ -139,7 +145,6 @@ class Hook(Base): #: Deprecated, use method `exit` instead def coreExiting(self): - self.logWarning(_("Deprecated method `coreExiting`, use `exit` instead")) return self.exit() @@ -178,7 +183,6 @@ class Hook(Base): #: Deprecated, use method `captchaTask` instead def newCaptchaTask(self, task): - self.logWarning(_("Deprecated method `newCaptchaTask`, use `captchaTask` instead")) return self.captchaTask() diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index c13d94aac..950879c6e 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -60,8 +60,8 @@ class Base(object): log = getattr(self.core.log, level) msg = " | ".join((fs_encode(a) if isinstance(a, unicode) else #@NOTE: `fs_encode` -> `encode` in 0.4.10 str(a)).strip() for a in args if a) - log("%(plugin)s[%(id)s]: %(msg)s" % {'plugin': self.__name__, - 'id' : self.pyfile.id if hasattr(self, 'pyfile') else "", + log("%(plugin)s%(id)s: %(msg)s" % {'plugin': self.__name__, + 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", 'msg' : msg or _(level.upper() + " MARK")}) -- cgit v1.2.3 From 725f4ee53a8e8797492329d015af18262e9ca369 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 02:05:56 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1496 --- module/plugins/internal/Hook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 6931a7dba..71e8866f3 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -24,7 +24,7 @@ def threaded(fn): class Hook(Base): __name__ = "Hook" __type__ = "hook" - __version__ = "0.03" + __version__ = "0.04" __config__ = [] #: [("name", "type", "desc", "default")] __threaded__ = [] #@TODO: Remove in 0.4.10 @@ -183,7 +183,7 @@ class Hook(Base): #: Deprecated, use method `captchaTask` instead def newCaptchaTask(self, task): - return self.captchaTask() + return self.captchaTask(task) def captchaCorrect(self, task): -- cgit v1.2.3 From 963c62e6f5104e700fbd9aed81a7595d12288b20 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 05:14:06 +0200 Subject: [XFSCrypter] Improve OFFLINE_PATTERN --- module/plugins/internal/XFSCrypter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 8b333b45c..84317c615 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.09" + __version__ = "0.10" __pattern__ = r'^unmatchable$' @@ -22,7 +22,7 @@ class XFSCrypter(SimpleCrypter): NAME_PATTERN = r'<[Tt]itle>.*?\: (?P.+) folder' LINK_PATTERN = r'<(?:td|TD).*?>\s*(?:<.+>\s*)?.+?(?:)?\s*(?:<.+>\s*)?' - OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + OFFLINE_PATTERN = r'>\s*(No such user|\w+ (Not Found|file (was|has been) removed|no longer available)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' -- cgit v1.2.3 From b2cd27adb0912003d8a71cff89e49c7a4d594d3f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 05:14:32 +0200 Subject: [SimpleCrypter] Fixup --- module/plugins/internal/SimpleCrypter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index be0d5e9d5..6a339b46e 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -11,7 +11,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.53" + __version__ = "0.54" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] @@ -114,7 +114,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.preload() self.checkInfo() - self.links = self.getLinks() + self.links = self.getLinks() or list() if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): self.handlePages(pyfile) -- cgit v1.2.3 From 8fdf7ff12b8a7c80292c4599e0220439cb9cfcac Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 05:31:32 +0200 Subject: Update Hook (2) --- module/plugins/internal/Hook.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 71e8866f3..4f9a01bb3 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -24,7 +24,7 @@ def threaded(fn): class Hook(Base): __name__ = "Hook" __type__ = "hook" - __version__ = "0.04" + __version__ = "0.05" __config__ = [] #: [("name", "type", "desc", "default")] __threaded__ = [] #@TODO: Remove in 0.4.10 @@ -59,11 +59,10 @@ class Hook(Base): self.event_list = [] #@NOTE: dont make duplicate entries in event_map self.initEvents() - # self.initPeriodical(10) + self.initPeriodical(10) def initEvents(self): - # register events if self.event_map: for event, funcs in self.event_map.iteritems(): if type(funcs) in (list, tuple): @@ -123,7 +122,7 @@ class Hook(Base): pass - #: Deprecated, use method `deactivate` instead + #: Deprecated method, use `deactivate` instead def unload(self): return self.deactivate() @@ -133,7 +132,7 @@ class Hook(Base): pass - #: Deprecated, use method `activate` instead + #: Deprecated method, use `activate` instead def coreReady(self): return self.activate() @@ -143,7 +142,7 @@ class Hook(Base): pass - #: Deprecated, use method `exit` instead + #: Deprecated method, use `exit` instead def coreExiting(self): return self.exit() @@ -176,14 +175,14 @@ class Hook(Base): pass - def captchaTask(self, task): + def captcha_task(self, task): """New captcha task for the plugin, it MUST set the handler and timeout or will be ignored""" pass - #: Deprecated, use method `captchaTask` instead + #: Deprecated method, use `captcha_task` instead def newCaptchaTask(self, task): - return self.captchaTask(task) + return self.captcha_task(task) def captchaCorrect(self, task): -- cgit v1.2.3 From 8319d31956f7c1a09684009b7ec8c64bd0f3f631 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 05:36:33 +0200 Subject: Update Base --- module/plugins/internal/Plugin.py | 125 ++++++++++++++++++++------------ module/plugins/internal/SimpleHoster.py | 16 ---- 2 files changed, 77 insertions(+), 64 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 950879c6e..70bd8574d 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -31,7 +31,7 @@ if os.name != "nt": from itertools import islice -from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload #@TODO: Remove in 0.4.10 +from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 from module.utils import save_join, save_path, fs_encode, fs_decode def chunks(iterable, size): @@ -55,7 +55,8 @@ class Base(object): #: core config self.config = core.config - # Log functions + + #: Log functions def _log(self, level, args): log = getattr(self.core.log, level) msg = " | ".join((fs_encode(a) if isinstance(a, unicode) else #@NOTE: `fs_encode` -> `encode` in 0.4.10 @@ -86,67 +87,96 @@ class Base(object): return self._log("critical", args) - def setConf(self, option, value): - """ see `setConfig` """ - self.core.config.setPlugin(self.__name__, option, value) - def setConfig(self, option, value): - """ Set config value for current plugin + """ + Set config value for current plugin :param option: :param value: :return: """ - self.setConf(option, value) + self.core.config.setPlugin(self.__name__, option, value) + + + #: Deprecated method + def setConf(self, *args, **kwargs): + """See `setConfig`""" + return self.setConfig(*args, **kwargs) - def getConf(self, option): - """ see `getConfig` """ - return self.core.config.getPlugin(self.__name__, option) def getConfig(self, option): - """ Returns config value for current plugin + """ + Returns config value for current plugin :param option: :return: """ - return self.getConf(option) + return self.core.config.getPlugin(self.__name__, option) + + + #: Deprecated method + def getConf(self, *args, **kwargs): + """See `getConfig`""" + return self.getConfig(*args, **kwargs) - def setStorage(self, key, value): - """ Saves a value persistently to the database """ - self.core.db.setStorage(self.__name__, key, value) def store(self, key, value): - """ same as `setStorage` """ + """Saves a value persistently to the database""" self.core.db.setStorage(self.__name__, key, value) - def getStorage(self, key=None, default=None): - """ Retrieves saved value or dict of all saved entries if key is None """ - if key is not None: - return self.core.db.getStorage(self.__name__, key) or default - return self.core.db.getStorage(self.__name__, key) - def retrieve(self, *args, **kwargs): - """ same as `getStorage` """ - return self.getStorage(*args, **kwargs) + #: Deprecated method + def setStorage(self, *args, **kwargs): + """Same as `setStorage`""" + return self.store(*args, **kwargs) + + + def retrieve(self, key, default=None): + """Retrieves saved value or dict of all saved entries if key is None""" + return self.core.db.getStorage(self.__name__, key) or default + + + #: Deprecated method + def getStorage(self, *args, **kwargs): + """Same as `getStorage`""" + return self.retrieve(*args, **kwargs) + def delStorage(self, key): - """ Delete entry in db """ + """Delete entry in db""" self.core.db.delStorage(self.__name__, key) + def fail(self, reason): + """Fail and give reason""" + raise Fail(reason) + + + def error(self, reason="", type=_("Parse")): + if not reason and not type: + type = _("Unknown") + + msg = _("%s error") % type.strip().capitalize() if type else _("Error") + msg += (": %s" % reason.strip()) if reason else "" + msg += _(" | Plugin may be out of date") + + raise Fail(msg) + + class Plugin(Base): - """ - Base plugin for hoster/crypter. - Overwrite `process` / `decrypt` in your subclassed plugin. - """ - __name__ = "Plugin" - __version__ = "0.07" - __pattern__ = None - __type__ = "hoster" - __config__ = [("name", "type", "desc", "default")] - __description__ = """Base Plugin""" - __author_name__ = ("RaNaN", "spoob", "mkaay") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") + __name__ = "Plugin" + __type__ = "hoster" + __version__ = "0.08" + + __pattern__ = r'^unmatchable$' + __config__ = [] #: [("name", "type", "desc", "default")] + + __description__ = """Base plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("spoob", "spoob@pyload.org"), + ("mkaay", "mkaay@mkaay.de" )] + def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) @@ -286,10 +316,6 @@ class Plugin(Base): self.waiting = False self.pyfile.setStatus("starting") - def fail(self, reason): - """ fail and give reason """ - raise Fail(reason) - def offline(self): """ fail and indicate file is offline """ raise Fail("offline") @@ -298,6 +324,9 @@ class Plugin(Base): """ fail and indicates file ist temporary offline, the core may take consequences """ raise Fail("temp. offline") + def skip(self, reason) + raise Skip(reason) + def retry(self, max_tries=3, wait_time=1, reason=""): """Retries and begin again from the beginning @@ -574,7 +603,7 @@ class Plugin(Base): """ checks if same file was/is downloaded within same package :param starting: indicates that the current download is going to start - :raises SkipDownload: + :raises Skip: """ pack = self.pyfile.package() @@ -582,10 +611,10 @@ class Plugin(Base): for pyfile in self.core.files.cache.values(): if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: if pyfile.status in (0, 12): #finished or downloading - raise SkipDownload(pyfile.pluginname) + self.skip(pyfile.pluginname) elif pyfile.status in ( 5, 7) and starting: #a download is waiting/starting and was appenrently started before - raise SkipDownload(pyfile.pluginname) + self.skip(pyfile.pluginname) download_folder = self.config['general']['download_folder'] location = save_join(download_folder, pack.folder, self.pyfile.name) @@ -593,14 +622,14 @@ class Plugin(Base): if starting and self.core.config['download']['skip_existing'] and exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: - raise SkipDownload("File exists.") + self.skip("File exists") pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: if exists(location): - raise SkipDownload(pyfile[0]) + self.skip(pyfile[0]) - self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) + self.log.debug("File %s not skipped, because it does not exists" % self.pyfile.name) def clean(self): """ clean everything and remove references """ diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index d1563ac89..ad76e27a6 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -23,18 +23,6 @@ from module.utils import fixup, fs_encode, html_unescape, parseFileSize statusMap = dict((v, k) for k, v in _statusMap.iteritems()) -#@TODO: Remove in 0.4.10 and redirect to self.error instead -def _error(self, reason, type): - if not reason and not type: - type = "unknown" - - msg = _("%s error") % type.strip().capitalize() if type else _("Error") - msg += (": %s" % reason.strip()) if reason else "" - msg += _(" | Plugin may be out of date") - - raise Fail(msg) - - #@TODO: Remove in 0.4.10 def _wait(self, seconds, reconnect): if seconds: @@ -852,7 +840,3 @@ class SimpleHoster(Hoster): #@TODO: Remove in 0.4.10 def wait(self, seconds=0, reconnect=None): return _wait(self, seconds, reconnect) - - - def error(self, reason="", type="parse"): - return _error(self, reason, type) -- cgit v1.2.3 From 743fb50e0f6e33077b8d1ea661f80c2f34e2711a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 05:41:47 +0200 Subject: [Plugin] Typo --- module/plugins/internal/Plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 70bd8574d..b89772052 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -166,7 +166,7 @@ class Base(object): class Plugin(Base): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.08" + __version__ = "0.09" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] @@ -324,7 +324,7 @@ class Plugin(Base): """ fail and indicates file ist temporary offline, the core may take consequences """ raise Fail("temp. offline") - def skip(self, reason) + def skip(self, reason): raise Skip(reason) def retry(self, max_tries=3, wait_time=1, reason=""): -- cgit v1.2.3 From 2b1399197d6090e8672edabbbfaa59d0ecb4ce8d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 06:54:39 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1498 --- module/plugins/internal/MultiHook.py | 11 +---------- module/plugins/internal/Plugin.py | 10 +++++++--- module/plugins/internal/SimpleHoster.py | 10 ---------- 3 files changed, 8 insertions(+), 23 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 4a2c6654b..0a85ba277 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -11,7 +11,7 @@ from module.utils import decode, remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.46" + __version__ = "0.47" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), @@ -106,15 +106,6 @@ class MultiHook(Hook): return rep - def getConfig(self, option, default=''): #@TODO: Remove in 0.4.10 - """getConfig with default value - sublass may not implements all config options""" - try: - return self.getConf(option) - - except KeyError: - return default - - def pluginsCached(self): if self.plugins: return self.plugins diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index b89772052..16aa9cc45 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -104,14 +104,18 @@ class Base(object): return self.setConfig(*args, **kwargs) - def getConfig(self, option): + def getConfig(self, option, default=""): """ Returns config value for current plugin :param option: :return: """ - return self.core.config.getPlugin(self.__name__, option) + try: + return self.core.config.getPlugin(self.__name__, option) + + except KeyError: + return default #: Deprecated method @@ -166,7 +170,7 @@ class Base(object): class Plugin(Base): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.09" + __version__ = "0.10" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ad76e27a6..a00676aa1 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -817,16 +817,6 @@ class SimpleHoster(Hoster): return size <= traffic - #@TODO: Remove in 0.4.10 - def getConfig(self, option, default=''): - """getConfig with default value - sublass may not implements all config options""" - try: - return self.getConf(option) - - except KeyError: - return default - - def retryFree(self): if not self.premium: return -- cgit v1.2.3 From d79f089fd47ba664cf9c3136b70961727b00bbe5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 07:11:45 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1498 (2) --- module/plugins/internal/MultiHook.py | 37 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 23 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 0a85ba277..bde0b7dc6 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -11,7 +11,7 @@ from module.utils import decode, remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.47" + __version__ = "0.48" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), @@ -93,8 +93,12 @@ class MultiHook(Hook): self.setConfig('activated', False) + def activate(self): + self.initPeriodical(threaded=True) + + def getURL(self, *args, **kwargs): #@TODO: Remove in 0.4.10 - """ see HTTPRequest for argument list """ + """See HTTPRequest for argument list""" h = pyreq.getHTTPRequest(timeout=120) try: if not 'decode' in kwargs: @@ -124,9 +128,9 @@ class MultiHook(Hook): return list() try: - configmode = self.getConfig("pluginmode", 'all') + configmode = self.getConfig('pluginmode', 'all') if configmode in ("listed", "unlisted"): - pluginlist = self.getConfig("pluginlist", '').replace('|', ',').replace(';', ',').split(',') + pluginlist = self.getConfig('pluginlist', '').replace('|', ',').replace(';', ',').split(',') configset = self._pluginSet(pluginlist) if configmode == "listed": @@ -155,33 +159,20 @@ class MultiHook(Hook): def getHosters(self): - """Load list of supported hoster + """ + Load list of supported hoster :return: List of domain names """ raise NotImplementedError - #: Threaded _periodical, remove in 0.4.10 and use built-in flag for that - def _periodical(self): - try: - if self.isActivated(): - self.periodical() - - except Exception, e: - self.core.log.error(_("Error executing hooks: %s") % str(e)) - if self.core.debug: - traceback.print_exc() - - self.cb = self.core.scheduler.addJob(self.interval, self._periodical) - - def periodical(self): - """reload plugin list periodically""" + """Reload plugin list periodically""" self.loadAccount() - if self.getConfig("reload", True): - self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_RELOAD_INTERVAL) + if self.getConfig('reload', True): + self.interval = max(self.getConfig('reloadinterval', 12) * 60 * 60, self.MIN_RELOAD_INTERVAL) else: self.core.scheduler.removeJob(self.cb) self.cb = None @@ -247,7 +238,7 @@ class MultiHook(Hook): # create new regexp regexp = r'.*(?P%s).*' % "|".join(x.replace('.', '\.') for x in plugins) - if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and '://' in self.pluginclass.__pattern__: + if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and "://" in self.pluginclass.__pattern__: regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) self.logDebug("Regexp: %s" % regexp) -- cgit v1.2.3 From c9144f451b74e4d3cc67935b9e73c662ac870c6e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 07:18:39 +0200 Subject: Hook plugin code cosmetics (2) --- module/plugins/internal/MultiHook.py | 2 +- module/plugins/internal/OCR.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index bde0b7dc6..a45d52e8a 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -258,7 +258,7 @@ class MultiHook(Hook): hdict.pop('new_name', None) - def unload(self): + def deactivate(self): """Remove override for all plugins. Scheduler job is removed by hookmanager""" for plugin in self.supported: self.unloadPlugin(plugin) diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 1874ba07d..2349d32af 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -36,7 +36,7 @@ class OCR(object): self.result_captcha = '' - def unload(self): + def deactivate(self): """delete all tmp images""" pass -- cgit v1.2.3 From a0d0d3ccd041097c01a100861d3b6d1b1460df16 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 18:07:17 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1502 --- module/plugins/internal/Hook.py | 87 ++++++++++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 19 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 4f9a01bb3..deeb5a097 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -24,7 +24,7 @@ def threaded(fn): class Hook(Base): __name__ = "Hook" __type__ = "hook" - __version__ = "0.05" + __version__ = "0.06" __config__ = [] #: [("name", "type", "desc", "default")] __threaded__ = [] #@TODO: Remove in 0.4.10 @@ -39,17 +39,11 @@ class Hook(Base): def __init__(self, core, manager): super(Hook, self).__init__(core) - #: Provide information in dict here, usable by API `getInfo` - self.info = {} - - #: Callback of periodical job task, used by HookManager - self.cb = None - self.interval = 60 - #: `HookManager` self.manager = manager - self.setup() + #: Provide information in dict here, usable by API `getInfo` + self.info = {} #: automatically register event listeners for functions, attribute will be deleted dont use it yourself self.event_map = {} @@ -58,11 +52,16 @@ class Hook(Base): #: List of events the plugin can handle, name the functions exactly like eventname. self.event_list = [] #@NOTE: dont make duplicate entries in event_map - self.initEvents() - self.initPeriodical(10) + #: Callback of periodical job task, used by HookManager + self.cb = None + self.interval = 60 + + self.setup() + self.init_events() + self.init_periodical(10) - def initEvents(self): + def init_events(self): if self.event_map: for event, funcs in self.event_map.iteritems(): if type(funcs) in (list, tuple): @@ -83,10 +82,15 @@ class Hook(Base): self.event_list = None - def initPeriodical(self, delay=0, threaded=False): + def init_periodical(self, delay=0, threaded=False): self.cb = self.core.scheduler.addJob(max(0, delay), self._periodical, [threaded], threaded=threaded) + #: Deprecated method, use `init_periodical` instead + def initPeriodical(self): + return self.init_periodical() + + def _periodical(self, threaded): if self.interval < 0: self.cb = None @@ -103,6 +107,10 @@ class Hook(Base): self.cb = self.core.scheduler.addJob(self.interval, self._periodical, [threaded], threaded=threaded) + def periodical(self): + pass + + def __repr__(self): return "" % self.__name__ @@ -112,11 +120,16 @@ class Hook(Base): pass - def isActivated(self): + def is_activated(self): """Checks if hook is activated""" return self.getConfig("activated") + #: Deprecated method, use `is_activated` instead + def isActivated(self): + return self.is_activated() + + def deactivate(self): """Called when hook was deactivated""" pass @@ -147,32 +160,58 @@ class Hook(Base): return self.exit() + def download_preparing(self, pyfile): + pass + + + #: Deprecated method, use `download_preparing` instead def downloadPreparing(self, pyfile): + return self.download_preparing(pyfile) + + + def download_finished(self, pyfile): pass + #: Deprecated method, use `download_finished` instead def downloadFinished(self, pyfile): + return self.download_finished(pyfile) + + + def download_failed(self, pyfile): pass + #: Deprecated method, use `download_failed` instead def downloadFailed(self, pyfile): + return self.download_failed(pyfile) + + + def package_finished(self, pypack): pass + #: Deprecated method, use `package_finished` instead def packageFinished(self, pypack): + return self.package_finished(pyfile) + + + def before_reconnect(self, ip): pass + #: Deprecated method, use `before_reconnect` instead def beforeReconnecting(self, ip): - pass + return self.before_reconnect(ip) - def afterReconnecting(self, ip): + def after_reconnect(self, ip, oldip): pass - def periodical(self): - pass + #: Deprecated method, use `after_reconnect` instead + def afterReconnecting(self, ip): + return self.after_reconnect(ip, None) def captcha_task(self, task): @@ -185,9 +224,19 @@ class Hook(Base): return self.captcha_task(task) + def captcha_correct(self, task): + pass + + + #: Deprecated method, use `captcha_correct` instead def captchaCorrect(self, task): + return self.captcha_correct(task) + + + def captcha_invalid(self, task): pass + #: Deprecated method, use `captcha_invalid` instead def captchaInvalid(self, task): - pass + return self.captcha_invalid(task) -- cgit v1.2.3 From 12b883227b113efaf33d48e9158fc3d76d3aa490 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 18:09:18 +0200 Subject: [MultiHook] Update --- module/plugins/internal/MultiHook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index a45d52e8a..10b2e7b0e 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -11,7 +11,7 @@ from module.utils import decode, remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.48" + __version__ = "0.49" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), @@ -94,7 +94,7 @@ class MultiHook(Hook): def activate(self): - self.initPeriodical(threaded=True) + self.init_periodical(threaded=True) def getURL(self, *args, **kwargs): #@TODO: Remove in 0.4.10 -- cgit v1.2.3 From 40092f35f8bf79a88e435acb056cf0129f1b5711 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 18:30:32 +0200 Subject: Fix http://forum.pyload.org/viewtopic.php?f=12&t=4332 --- module/plugins/internal/Container.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index fa3508ae4..3b3eb88aa 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -6,13 +6,13 @@ import os import re from module.plugins.internal.Crypter import Crypter -from pyload.utils import save_join +from module.utils import save_join class Container(Crypter): __name__ = "Container" __type__ = "container" - __version__ = "0.03" + __version__ = "0.04" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] -- cgit v1.2.3 From eb7a80a8c30a16f5a828823e79893bdc4764704c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 18:45:14 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1504 --- module/plugins/internal/Hook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index deeb5a097..0eec4c008 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -74,7 +74,7 @@ class Hook(Base): self.event_map = None if self.event_list: - self.logWarning(_("Deprecated method `event_list`, use `event_map` instead")) + self.logDebug("Deprecated method `event_list`, use `event_map` instead") for f in self.event_list: self.manager.addEvent(f, getattr(self, f)) -- cgit v1.2.3 From 7bc0b193735b5e98c33ae6f0f10e3d49d441fdf0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 19:00:15 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1503 --- module/plugins/internal/Hook.py | 2 +- module/plugins/internal/SimpleHoster.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 0eec4c008..0eeca5556 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -24,7 +24,7 @@ def threaded(fn): class Hook(Base): __name__ = "Hook" __type__ = "hook" - __version__ = "0.06" + __version__ = "0.07" __config__ = [] #: [("name", "type", "desc", "default")] __threaded__ = [] #@TODO: Remove in 0.4.10 diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index a00676aa1..58d8158ca 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -229,7 +229,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.67" + __version__ = "1.68" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), -- cgit v1.2.3 From 5e5775d7ef1a59f9ad166995483dc7a38e096f22 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 20:31:11 +0200 Subject: Fix http://forum.pyload.org/viewtopic.php?f=10&t=4328 --- module/plugins/internal/Hook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 0eeca5556..7d785ded0 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -24,7 +24,7 @@ def threaded(fn): class Hook(Base): __name__ = "Hook" __type__ = "hook" - __version__ = "0.07" + __version__ = "0.08" __config__ = [] #: [("name", "type", "desc", "default")] __threaded__ = [] #@TODO: Remove in 0.4.10 @@ -193,7 +193,7 @@ class Hook(Base): #: Deprecated method, use `package_finished` instead def packageFinished(self, pypack): - return self.package_finished(pyfile) + return self.package_finished(pypack) def before_reconnect(self, ip): -- cgit v1.2.3 From 5a139055ae658d3a05cbb658cbd66aeae0d01db5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 21:06:10 +0200 Subject: Spare code cosmetics --- module/plugins/internal/Account.py | 4 ++-- module/plugins/internal/Captcha.py | 4 ++-- module/plugins/internal/Container.py | 8 +++---- module/plugins/internal/Crypter.py | 4 ++-- module/plugins/internal/Hook.py | 4 ++-- module/plugins/internal/OCR.py | 8 +++---- module/plugins/internal/Plugin.py | 44 ++++++++++++++++-------------------- module/plugins/internal/SevenZip.py | 4 ++-- module/plugins/internal/UnRar.py | 10 ++++---- 9 files changed, 43 insertions(+), 47 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 27a040e24..aa472f297 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -5,7 +5,7 @@ import threading import time import traceback -from module.plugins.internal.Plugin import Base +from module.plugins.internal.Plugin import Plugin from module.utils import compare_time, lock, parseFileSize @@ -13,7 +13,7 @@ class WrongPassword(Exception): pass -class Account(Base): +class Account(Plugin): __name__ = "Account" __type__ = "account" __version__ = "0.03" diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index e5a44d750..8d040515c 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.Plugin import Base +from module.plugins.internal.Plugin import Plugin #@TODO: Extend (new) Plugin class; remove all `html` args -class Captcha(Base): +class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" __version__ = "0.30" diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 3b3eb88aa..097b04ac7 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -6,7 +6,7 @@ import os import re from module.plugins.internal.Crypter import Crypter -from module.utils import save_join +from module.utils import save_join as fs_join class Container(Crypter): @@ -43,7 +43,7 @@ class Container(Crypter): if self.pyfile.url.startswith("http"): self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] content = self.load(self.pyfile.url) - self.pyfile.url = save_join(self.core.config.get("general", "download_folder"), self.pyfile.name) + self.pyfile.url = fs_join(self.core.config.get("general", "download_folder"), self.pyfile.name) try: with open(self.pyfile.url, "wb") as f: f.write(content) @@ -53,8 +53,8 @@ class Container(Crypter): else: self.pyfile.name = os.path.basename(self.pyfile.url) if not os.path.exists(self.pyfile.url): - if os.path.exists(save_join(pypath, self.pyfile.url)): - self.pyfile.url = save_join(pypath, self.pyfile.url) + if os.path.exists(fs_join(pypath, self.pyfile.url)): + self.pyfile.url = fs_join(pypath, self.pyfile.url) else: self.fail(_("File not exists")) diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 5fb9bc5c2..39b09129f 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -3,7 +3,7 @@ import urlparse from module.plugins.internal.Plugin import Plugin -from module.utils import decode, save_path +from module.utils import decode, save_path as safe_filename class Crypter(Plugin): @@ -98,7 +98,7 @@ class Crypter(Plugin): if not folder: folder = urlparse.urlparse(name).path.split("/")[-1] - setFolder(save_path(folder)) + setFolder(safe_filename(folder)) self.logDebug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) elif folder_per_package: diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 7d785ded0..8d620e794 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -2,7 +2,7 @@ import traceback -from module.plugins.internal.Plugin import Base +from module.plugins.internal.Plugin import Plugin class Expose(object): @@ -21,7 +21,7 @@ def threaded(fn): return run -class Hook(Base): +class Hook(Plugin): __name__ = "Hook" __type__ = "hook" __version__ = "0.08" diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 2349d32af..1782e17f0 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -13,7 +13,7 @@ import os import subprocess #import tempfile -from module.utils import save_join +from module.utils import save_join as fs_join class OCR(object): @@ -59,11 +59,11 @@ class OCR(object): def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True): #tmpTif = tempfile.NamedTemporaryFile(suffix=".tif") try: - tmpTif = open(save_join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") + tmpTif = open(fs_join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") tmpTif.close() #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") - tmpTxt = open(save_join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") + tmpTxt = open(fs_join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") tmpTxt.close() except IOError, e: @@ -82,7 +82,7 @@ class OCR(object): if subset and (digits or lowercase or uppercase): #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") - with open(save_join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") as tmpSub: + with open(fs_join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") as tmpSub: tmpSub.write("tessedit_char_whitelist ") if digits: diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 16aa9cc45..da597ef42 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -32,7 +32,7 @@ if os.name != "nt": from itertools import islice from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 -from module.utils import save_join, save_path, fs_encode, fs_decode +from module.utils import save_join as fs_join, save_path as safe_filename, fs_encode, fs_decode def chunks(iterable, size): it = iter(iterable) @@ -50,10 +50,6 @@ class Base(object): def __init__(self, core): #: Core instance self.core = core - #: logging instance - self.log = core.log - #: core config - self.config = core.config #: Log functions @@ -183,7 +179,7 @@ class Plugin(Base): def __init__(self, pyfile): - Base.__init__(self, pyfile.m.core) + super(Plugin, self).__init__(pyfile.m.core) self.wantReconnect = False #: enables simultaneous processing of multiple downloads @@ -239,8 +235,8 @@ class Plugin(Base): def getChunkCount(self): if self.chunkLimit <= 0: - return self.config["download"]["chunks"] - return min(self.config["download"]["chunks"], self.chunkLimit) + return self.core.config["download"]["chunks"] + return min(self.core.config["download"]["chunks"], self.chunkLimit) def __call__(self): return self.__name__ @@ -416,7 +412,7 @@ class Plugin(Base): self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) result = task.result - self.log.debug("Received captcha result: %s" % str(result)) + self.core.log.debug("Received captcha result: %s" % str(result)) if not self.core.debug: try: @@ -504,25 +500,25 @@ class Plugin(Base): self.pyfile.setStatus("downloading") - download_folder = self.config['general']['download_folder'] + download_folder = self.core.config['general']['download_folder'] - location = save_join(download_folder, self.pyfile.package().folder) + location = fs_join(download_folder, self.pyfile.package().folder) if not exists(location): makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] + uid = getpwnam(self.core.config["permission"]["user"])[2] + gid = getgrnam(self.core.config["permission"]["group"])[2] chown(location, uid, gid) except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) + self.core.log.warning(_("Setting User and Group failed: %s") % str(e)) # convert back to unicode location = fs_decode(location) - name = save_path(self.pyfile.name) + name = safe_filename(self.pyfile.name) filename = join(location, name) @@ -536,7 +532,7 @@ class Plugin(Base): self.pyfile.size = self.req.size if disposition and newname and newname != name: #triple check, just to be sure - self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.core.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) self.pyfile.name = newname filename = join(location, newname) @@ -547,12 +543,12 @@ class Plugin(Base): if self.core.config["permission"]["change_dl"] and os.name != "nt": try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] + uid = getpwnam(self.core.config["permission"]["user"])[2] + gid = getgrnam(self.core.config["permission"]["group"])[2] chown(fs_filename, uid, gid) except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) + self.core.log.warning(_("Setting User and Group failed: %s") % str(e)) self.lastDownload = filename return self.lastDownload @@ -575,12 +571,12 @@ class Plugin(Base): if api_size and api_size <= size: return None elif size > max_size and not read_size: return None - self.log.debug("Download Check triggered") + self.core.log.debug("Download Check triggered") f = open(lastDownload, "rb") content = f.read(read_size if read_size else -1) f.close() #produces encoding errors, better log to other file in the future? - #self.log.debug("Content: %s" % content) + #self.core.log.debug("Content: %s" % content) for name, rule in rules.iteritems(): if type(rule) in (str, unicode): if rule in content: @@ -620,8 +616,8 @@ class Plugin(Base): 5, 7) and starting: #a download is waiting/starting and was appenrently started before self.skip(pyfile.pluginname) - download_folder = self.config['general']['download_folder'] - location = save_join(download_folder, pack.folder, self.pyfile.name) + download_folder = self.core.config['general']['download_folder'] + location = fs_join(download_folder, pack.folder, self.pyfile.name) if starting and self.core.config['download']['skip_existing'] and exists(location): size = os.stat(location).st_size @@ -633,7 +629,7 @@ class Plugin(Base): if exists(location): self.skip(pyfile[0]) - self.log.debug("File %s not skipped, because it does not exists" % self.pyfile.name) + self.core.log.debug("File %s not skipped, because it does not exists" % self.pyfile.name) def clean(self): """ clean everything and remove references """ diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index 624f6c939..cad4cfa1c 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -5,7 +5,7 @@ import re import subprocess from module.plugins.internal.UnRar import ArchiveError, CRCError, PasswordError, UnRar, renice -from module.utils import fs_encode, save_join +from module.utils import fs_encode, save_join as fs_join class SevenZip(UnRar): @@ -126,7 +126,7 @@ class SevenZip(UnRar): result = set() for groups in self.re_filelist.findall(out): f = groups[-1].strip() - result.add(save_join(self.out, f)) + result.add(fs_join(self.out, f)) return list(result) diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 5b9f2e1c3..85e939b6a 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -8,7 +8,7 @@ from glob import glob from string import digits from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import fs_decode, fs_encode, save_join +from module.utils import fs_decode, fs_encode, save_join as fs_join def renice(pid, value): @@ -175,7 +175,7 @@ class UnRar(Extractor): files = [self.filename] # eventually Multipart Files - files.extend(save_join(dir, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir)) + files.extend(fs_join(dir, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir)) if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file)) return files @@ -197,13 +197,13 @@ class UnRar(Extractor): if not self.fullpath and self.VERSION.startswith('5'): # NOTE: Unrar 5 always list full path for f in fs_decode(out).splitlines(): - f = save_join(self.out, os.path.basename(f.strip())) + f = fs_join(self.out, os.path.basename(f.strip())) if os.path.isfile(f): - result.add(save_join(self.out, os.path.basename(f))) + result.add(fs_join(self.out, os.path.basename(f))) else: for f in fs_decode(out).splitlines(): f = f.strip() - result.add(save_join(self.out, f)) + result.add(fs_join(self.out, f)) return list(result) -- cgit v1.2.3 From aeba469412a08a52f8fcd61b02dd5b9e481a4dff Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 15 Jun 2015 22:24:08 +0200 Subject: [SimpleHoster] fixurl --- module/plugins/internal/SimpleCrypter.py | 14 ++++---------- module/plugins/internal/SimpleHoster.py | 25 ++++++++++++++++--------- module/plugins/internal/XFSHoster.py | 1 - 3 files changed, 20 insertions(+), 20 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 6a339b46e..3e8b89f79 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import re -import urlparse from module.plugins.internal.Crypter import Crypter from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies @@ -11,7 +10,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.54" + __version__ = "0.55" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] @@ -108,13 +107,13 @@ class SimpleCrypter(Crypter, SimpleHoster): self.handleDirect(pyfile) if self.link: - self.urls = [self.link] + self.urls = self.fixurls([self.link]) else: self.preload() self.checkInfo() - self.links = self.getLinks() or list() + self.links = self.fixurls(self.getLinks()) or list() if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): self.handlePages(pyfile) @@ -122,7 +121,6 @@ class SimpleCrypter(Crypter, SimpleHoster): self.logDebug("Package has %d links" % len(self.links)) if self.links: - self.links = [html_unescape(l.decode('unicode-escape').strip()) for l in self.links] #@TODO: Move to Crypter in 0.4.10 self.packages = [(self.info['name'], self.links, self.info['folder'])] elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 @@ -159,11 +157,7 @@ class SimpleCrypter(Crypter, SimpleHoster): Returns the links extracted from self.html You should override this only if it's impossible to extract links using only the LINK_PATTERN. """ - url_p = urlparse.urlparse(self.pyfile.url) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - - return [urlparse.urljoin(baseurl, link) if not urlparse.urlparse(link).scheme else link \ - for link in re.findall(self.LINK_PATTERN, self.html)] + return re.findall(self.LINK_PATTERN, self.html) def handlePages(self, pyfile): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 58d8158ca..cec5f55f1 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -229,7 +229,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.68" + __version__ = "1.69" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -517,20 +517,27 @@ class SimpleHoster(Hoster): raise Fail(err) + def fixurl(self, url): + return self.fixurls([url])[0] + + + def fixurls(self, urls): + url_p = urlparse.urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + + urls = (html_unescape(url.decode('unicode-escape').strip()) for url in urls) + + return [urlparse.urljoin(baseurl, url) if not urlparse.urlparse(url).scheme else url \ + for url in urls] + + def download(self, url, *args, **kwargs): if not url or not isinstance(url, basestring): return self.correctCaptcha() - url = html_unescape(url.decode('unicode-escape').strip()) #@TODO: Move to Hoster in 0.4.10 - - if not urlparse.urlparse(url).scheme: - url_p = urlparse.urlparse(self.pyfile.url) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - url = urlparse.urljoin(baseurl, url) - - return super(SimpleHoster, self).download(url, *args, **kwargs) + return super(SimpleHoster, self).download(self.fixurl(url), *args, **kwargs) def checkFile(self): diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 0591f1baf..b2642bf1a 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -3,7 +3,6 @@ import pycurl import random import re -import urlparse from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SolveMedia import SolveMedia -- cgit v1.2.3 From c1764e2fea0bb05164c83a876e8cd58b97f58f25 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 16 Jun 2015 17:31:38 +0200 Subject: Update all --- module/plugins/internal/Account.py | 11 +- module/plugins/internal/AdYouLike.py | 2 +- module/plugins/internal/AdsCaptcha.py | 2 +- module/plugins/internal/Captcha.py | 6 - module/plugins/internal/Crypter.py | 21 +- module/plugins/internal/Hook.py | 3 - module/plugins/internal/Hoster.py | 704 ++++++++++++++++++++++++++++++- module/plugins/internal/MultiHook.py | 15 - module/plugins/internal/Plugin.py | 608 ++++++-------------------- module/plugins/internal/ReCaptcha.py | 33 +- module/plugins/internal/SimpleCrypter.py | 19 +- module/plugins/internal/SimpleHoster.py | 259 +----------- module/plugins/internal/SolveMedia.py | 6 +- module/plugins/internal/XFSAccount.py | 13 +- module/plugins/internal/XFSCrypter.py | 2 +- module/plugins/internal/XFSHoster.py | 8 +- 16 files changed, 874 insertions(+), 838 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index aa472f297..0e2996aaa 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -154,7 +154,9 @@ class Account(Plugin): if not type(infos) == dict: raise Exception("Wrong return format") except Exception, e: - infos = {"error": str(e)} + infos = super(self.__class__, self).loadAccountInfo(name, req) + infos['error'] = str(e) + if self.core.debug: traceback.print_exc() @@ -206,11 +208,8 @@ class Account(Plugin): def getAccountRequest(self, user=None): if not user: user, data = self.selectAccount() - if not user: - return None - req = self.core.requestFactory.getRequest(self.__name__, user) - return req + return self.core.requestFactory.getRequest(self.__name__, user) def getAccountCookies(self, user=None): @@ -264,7 +263,7 @@ class Account(Plugin): return self.selectAccount() != (None, None) - def parseTraffic(self, value, unit=None): #: return bytes + def parseTraffic(self, value, unit=None): #: return kilobytes if not unit and not isinstance(value, basestring): unit = "KB" return parseFileSize(value, unit) diff --git a/module/plugins/internal/AdYouLike.py b/module/plugins/internal/AdYouLike.py index a9c194dda..b7324ef8e 100644 --- a/module/plugins/internal/AdYouLike.py +++ b/module/plugins/internal/AdYouLike.py @@ -41,7 +41,7 @@ class AdYouLike(Captcha): # "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} ayl = json_loads(ayl) - html = self.plugin.req.load("http://api-ayl.appspot.com/challenge", + html = self.plugin.load("http://api-ayl.appspot.com/challenge", get={'key' : ayl['adyoulike']['key'], 'env' : ayl['all']['env'], 'callback': callback}) diff --git a/module/plugins/internal/AdsCaptcha.py b/module/plugins/internal/AdsCaptcha.py index 9cab99151..e058352dd 100644 --- a/module/plugins/internal/AdsCaptcha.py +++ b/module/plugins/internal/AdsCaptcha.py @@ -37,7 +37,7 @@ class AdsCaptcha(Captcha): def challenge(self, key=None, html=None): PublicKey, CaptchaId = key or self.retrieve_key(html) - html = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", + html = self.plugin.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}) try: diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 8d040515c..8dbc33da2 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -22,12 +22,6 @@ class Captcha(Plugin): super(Captcha, self).__init__(plugin.core) - #@TODO: Recheck in 0.4.10 - def fail(self, reason): - self.plugin.fail(reason) - raise AttributeError(reason) - - #@TODO: Recheck in 0.4.10 def retrieve_key(self, html): if self.detect_key(html): diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 39b09129f..a713c7da6 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -2,14 +2,14 @@ import urlparse -from module.plugins.internal.Plugin import Plugin -from module.utils import decode, save_path as safe_filename +from module.plugins.internal.Hoster import Hoster +from module.utils import save_path as safe_filename -class Crypter(Plugin): +class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.03" + __version__ = "0.04" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config.get("general", "folder_per_package") @@ -26,18 +26,12 @@ class Crypter(Plugin): def __init__(self, pyfile): super(Crypter, self).__init__(pyfile) - #: Provide information in dict here - self.info = {} #@TODO: Move to Plugin - #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) self.packages = [] #: List of urls, pyLoad will generate packagenames self.urls = [] - self.multiDL = True - self.limitDL = 0 - def process(self, pyfile): """Main method""" @@ -80,14 +74,13 @@ class Crypter(Plugin): "%d links" % len(links), "Saved to folder: %s" % folder if folder else "Saved to download folder") - links = map(decode, links) - - pid = self.core.api.addPackage(name, links, package_queue) + pid = self.core.api.addPackage(name, self.fixurl(links), package_queue) if package_password: self.core.api.setPackageData(pid, {"password": package_password}) - setFolder = lambda x: self.core.api.setPackageData(pid, {"folder": x or ""}) #@NOTE: Workaround to do not break API addPackage method + # Workaround to do not break API addPackage method + setFolder = lambda x: self.core.api.setPackageData(pid, {"folder": x or ""}) if use_subfolder: if not subfolder_per_package: diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 8d620e794..01ffbc5f2 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -42,9 +42,6 @@ class Hook(Plugin): #: `HookManager` self.manager = manager - #: Provide information in dict here, usable by API `getInfo` - self.info = {} - #: automatically register event listeners for functions, attribute will be deleted dont use it yourself self.event_map = {} diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 3fed8a7c6..9db22d1d7 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -1,33 +1,691 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. +from __future__ import with_statement - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. +import inspect +import os +import random +import time +import urlparse - You should have received a copy of the GNU General Public License - along with this program; if not, see . +if os.name != "nt": + import grp + import pwd - @author: mkaay -""" +from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip +from module.utils import fs_decode, fs_encode, save_join as fs_join -from module.plugins.internal.Plugin import Plugin -def getInfo(self): - #result = [ .. (name, size, status, url) .. ] - return +def getInfo(urls): + #result = [ .. (name, size, status, url) .. ] + pass + class Hoster(Plugin): - __name__ = "Hoster" - __version__ = "0.02" - __pattern__ = None - __type__ = "hoster" + __name__ = "Hoster" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'^unmatchable$' + __config__ = [] #: [("name", "type", "desc", "default")] + __description__ = """Base hoster plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + __license__ = "GPLv3" + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), + ("spoob" , "spoob@pyload.org" ), + ("mkaay" , "mkaay@mkaay.de" ), + ("Walter Purcaro", "vuolter@gmail.com")] + + + def __init__(self, pyfile): + super(Hoster, self).__init__(pyfile.m.core) + + #: engage wan reconnection + self.wantReconnect = False + + #: enable simultaneous processing of multiple downloads + self.multiDL = True + self.limitDL = 0 + + #: chunk limit + self.chunkLimit = 1 + self.resumeDownload = False + + #: time.time() + wait in seconds + self.waitUntil = 0 + self.waiting = False + + #: captcha reader instance + self.ocr = None + + #: account handler instance, see :py:class:`Account` + self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + + #: premium status + self.premium = False + + #: username/login + self.user = None + + if self.account and not self.account.canUse(): + self.account = None + + if self.account: + self.user, data = self.account.selectAccount() + + #: Browser instance, see `network.Browser` + self.req = self.account.getAccountRequest(self.user) + self.chunkLimit = -1 #: chunk limit, -1 for unlimited + + #: enables resume (will be ignored if server dont accept chunks) + self.resumeDownload = True + + #: premium status + self.premium = self.account.isPremium(self.user) + else: + self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) + + #: associated pyfile instance, see `PyFile` + self.pyfile = pyfile + + self.thread = None #: holds thread in future + + #: location where the last call to download was saved + self.lastDownload = "" + + #: re match of the last call to `checkDownload` + self.lastCheck = None + + #: js engine, see `JsEngine` + self.js = self.core.js + + #: captcha task + self.cTask = None + + #: some plugins store html code here + self.html = None + + #: dict of the amount of retries already made + self.retries = {} + + self.init() + + + def init(self): + """ + Initialize the plugin (in addition to `__init__`) + """ + pass + + + def setup(self): + """ + Setup for enviroment and other things, called before downloading (possibly more than one time) + """ + pass + + + def preprocessing(self, thread): + """ + Handles important things to do before starting + """ + self.thread = thread + + if self.account: + self.account.checkLogin(self.user) + else: + self.req.clearCookies() + + self.setup() + + self.pyfile.setStatus("starting") + + return self.process(self.pyfile) + + + def process(self, pyfile): + """ + The 'main' method of every plugin, you **have to** overwrite it + """ + raise NotImplementedError + + + def getChunkCount(self): + if self.chunkLimit <= 0: + return self.core.config.get("download", "chunks") + return min(self.core.config.get("download", "chunks"), self.chunkLimit) + + + def resetAccount(self): + """ + Don't use account and retry download + """ + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retry() + + + def setReconnect(self, reconnect): + reconnect = bool(reconnect) + self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + self.wantReconnect = reconnect + + + def setWait(self, seconds, reconnect=None): + """ + Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + wait_time = int(seconds) + 1 + wait_until = time.time() + wait_time + + self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), + "Wait: %d seconds" % wait_time) + + self.pyfile.waitUntil = wait_until + + if reconnect is not None: + self.setReconnect(reconnect) + + + def wait(self, seconds=None, reconnect=None): + """ + Waits the time previously set + """ + pyfile = self.pyfile + + if seconds is not None: + self.setWait(seconds) + + if reconnect is not None: + self.setReconnect(reconnect) + + self.waiting = True + + status = pyfile.status + pyfile.setStatus("waiting") + + self.logInfo(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), + _("Reconnect: %s") % self.wantReconnect) + + if self.account: + self.logDebug("Ignore reconnection due account logged") + + while pyfile.waitUntil > time.time(): + if pyfile.abort: + self.abort() + + time.sleep(1) + else: + while pyfile.waitUntil > time.time(): + self.thread.m.reconnecting.wait(2) + + if pyfile.abort: + self.abort() + + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + time.sleep(1) + + self.waiting = False + + pyfile.status = status + + + def skip(self, reason=""): + """ + Skip and give reason + """ + raise Skip(fs_encode(reason)) + + + def abort(self, reason=""): + """ + Abort and give reason + """ + if reason: + self.pyfile.error = fs_encode(reason) + raise Abort + + + def offline(self, reason=""): + """ + Fail and indicate file is offline + """ + if reason: + self.pyfile.error = fs_encode(reason) + raise Fail("offline") + + + def tempOffline(self, reason=""): + """ + Fail and indicates file ist temporary offline, the core may take consequences + """ + if reason: + self.pyfile.error = fs_encode(reason) + raise Fail("temp. offline") + + + def retry(self, max_tries=5, wait_time=1, reason=""): + """ + Retries and begin again from the beginning + + :param max_tries: number of maximum retries + :param wait_time: time to wait in seconds + :param reason: reason for retrying, will be passed to fail if max_tries reached + """ + id = inspect.currentframe().f_back.f_lineno + if id not in self.retries: + self.retries[id] = 0 + + if 0 < max_tries <= self.retries[id]: + self.fail(reason or _("Max retries reached"), "retry") + + self.wait(wait_time, False) + + self.retries[id] += 1 + raise Retry(reason) + + + def invalidCaptcha(self): + self.logError(_("Invalid captcha")) + if self.cTask: + self.cTask.invalid() + + + def correctCaptcha(self): + self.logInfo(_("Correct captcha")) + if self.cTask: + self.cTask.correct() + + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, + imgtype='jpg', result_type='textual'): + """ + Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time.time())[-6:].replace(".", "") + + with open(os.path.join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: + tmpCaptcha.write(img) + + has_plugin = self.__name__ in self.core.pluginManager.ocrPlugins + + if self.core.captcha: + Ocr = self.core.pluginManager.loadClass("ocr", self.__name__) + else: + Ocr = None + + if Ocr and not forceUser: + time.sleep(random.randint(3000, 5000) / 1000.0) + if self.pyfile.abort: + self.abort() + + ocr = Ocr() + result = ocr.get_captcha(tmpCaptcha.name) + else: + captchaManager = self.core.captchaManager + task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) + self.cTask = task + captchaManager.handleCaptcha(task) + + while task.isWaiting(): + if self.pyfile.abort: + captchaManager.removeTask(task) + self.abort() + time.sleep(1) + + captchaManager.removeTask(task) + + if task.error and has_plugin: #: ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) + + result = task.result + self.logDebug("Received captcha result: %s" % result) + + if not self.core.debug: + try: + os.remove(tmpCaptcha.name) + except Exception: + pass + + return result + + + def fixurl(self, url): + url_p = urlparse.urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + + url = super(Hoster, self).fixurl(url) + + if not urlparse.urlparse(url).scheme: + url = urlparse.urljoin(baseurl, url) + + return url + + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + """ + Downloads the content at url to download folder + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param disposition: if True and server provides content-disposition header\ + the filename will be changed if needed + :return: The location where the file was saved + """ + if self.pyfile.abort: + self.abort() + + url = self.fixurl(url) + + if not url or not isinstance(url, basestring): + self.fail(_("No url given")) + + if self.core.debug: + self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + + self.correctCaptcha() + self.checkForSameFiles() + + self.pyfile.setStatus("downloading") + + if disposition: + self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name + + download_folder = self.core.config.get("general", "download_folder") + + location = fs_join(download_folder, self.pyfile.package().folder) + + if not os.path.exists(location): + try: + os.makedirs(location, int(self.core.config.get("permission", "folder"), 8)) + + if self.core.config.get("permission", "change_dl") and os.name != "nt": + uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2] + gid = grp.getgrnam(self.core.config.get("permission", "group"))[2] + os.chown(location, uid, gid) + + except Exception, e: + self.fail(e) + + # convert back to unicode + location = fs_decode(location) + name = safe_filename(self.pyfile.name) + + filename = os.path.join(location, name) + + self.core.addonManager.dispatchEvent("download-start", self.pyfile, url, filename) + + try: + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.setProgress, disposition=disposition) + finally: + self.pyfile.size = self.req.size + + if newname: + newname = urlparse.urlparse(newname).path.split('/')[-1] + + if disposition and newname != name: + self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = os.path.join(location, newname) + + fs_filename = fs_encode(filename) + + if self.core.config.get("permission", "change_file"): + try: + os.chmod(fs_filename, int(self.core.config.get("permission", "file"), 8)) + except Exception, e: + self.logWarning(_("Setting file mode failed"), e) + + if self.core.config.get("permission", "change_dl") and os.name != "nt": + try: + uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2] + gid = grp.getgrnam(self.core.config.get("permission", "group"))[2] + os.chown(fs_filename, uid, gid) + + except Exception, e: + self.logWarning(_("Setting User and Group failed"), e) + + self.lastDownload = filename + return self.lastDownload + + + def checkDownload(self, rules, delete=True, file_size=None, size_tolerance=1000, read_size=100000): + """ + Checks the content of the last downloaded file, re match is saved to `lastCheck` + + :param rules: dict with names and rules to match (compiled regexp or strings) + :param delete: delete if matched + :param file_size: expected file size + :param size_tolerance: size check tolerance + :param read_size: amount of bytes to read from files + :return: dictionary key of the first rule that matched + """ + lastDownload = fs_encode(self.lastDownload) + + if not self.lastDownload or not os.path.exists(lastDownload): + self.lastDownload = "" + self.fail(self.pyfile.error or _("No file downloaded")) + + download_size = os.stat(lastDownload).st_size + + if download_size < 1 or (file_size and abs(file_size - download_size) > size_tolerance): + if delete: + os.remove(lastDownload) + self.fail(_("Empty file")) + + self.logDebug("Download Check triggered") + + with open(lastDownload, "rb") as f: + content = f.read(read_size) + + # produces encoding errors, better log to other file in the future? + # self.logDebug("Content: %s" % content) + for name, rule in rules.iteritems(): + if isinstance(rule, basestring): + if rule in content: + if delete: + os.remove(lastDownload) + return name + + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + if delete: + os.remove(lastDownload) + self.lastCheck = m + return name + + + def directLink(self, url, follow_location=None): + link = "" + + if follow_location is None: + redirect = 1 + + elif type(follow_location) is int: + redirect = max(follow_location, 1) + + else: + redirect = self.getConfig("maxredirs", plugin="UserAgentSwitcher") + + for i in xrange(redirect): + try: + self.logDebug("Redirect #%d to: %s" % (i, url)) + header = self.load(url, just_header=True) + + except Exception: #: Bad bad bad... rewrite this part in 0.4.10 + req = pyreq.getHTTPRequest() + res = self.load(url, just_header=True) + + req.close() + + header = {"code": req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: + continue + + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() + + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value + + if 'content-disposition' in header: + link = url + + elif 'location' in header and header['location']: + location = header['location'] + + if not urlparse.urlparse(location).scheme: + url_p = urlparse.urlparse(url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + location = urlparse.urljoin(baseurl, location) + + if 'code' in header and header['code'] == 302: + link = location + + if follow_location: + url = location + continue + + else: + extension = os.path.splitext(urlparse.urlparse(url).path.split('/')[-1])[-1] + + if 'content-type' in header and header['content-type']: + mimetype = header['content-type'].split(';')[0].strip() + + elif extension: + mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" + + else: + mimetype = "" + + if mimetype and (link or 'html' not in mimetype): + link = url + else: + link = "" + + break + + else: + try: + self.logError(_("Too many redirects")) + except Exception: + pass + + return link + + + def parseHtmlForm(self, attr_str="", input_names={}): + return parseHtmlForm(attr_str, self.html, input_names) + + + def checkTrafficLeft(self): + if not self.account: + return True + + traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] + + if traffic is None: + return False + elif traffic == -1: + return True + else: + size = self.pyfile.size / 1024 + self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) + return size <= traffic + + + def getPassword(self): + """ + Get the password the user provided in the package + """ + return self.pyfile.package().password or "" + + + def checkForSameFiles(self, starting=False): + """ + Checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises Skip: + """ + pack = self.pyfile.package() + + for pyfile in self.core.files.cache.values(): + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #: finished or downloading + self.skip(pyfile.pluginname) + elif pyfile.status in (5, 7) and starting: #: a download is waiting/starting and was appenrently started before + self.skip(pyfile.pluginname) + + download_folder = self.core.config.get("general", "download_folder") + location = fs_join(download_folder, pack.folder, self.pyfile.name) + + if starting and self.core.config.get("download", "skip_existing") and os.path.exists(location): + size = os.stat(location).st_size + if size >= self.pyfile.size: + self.skip("File exists") + + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + if pyfile: + if os.path.exists(location): + self.skip(pyfile[0]) + + self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name) + + + def clean(self): + """ + Clean everything and remove references + """ + if hasattr(self, "pyfile"): + del self.pyfile + + if hasattr(self, "req"): + self.req.close() + del self.req + + if hasattr(self, "thread"): + del self.thread + + if hasattr(self, "html"): + del self.html diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 10b2e7b0e..0bcab5915 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -55,8 +55,6 @@ class MultiHook(Hook): def setup(self): - self.info = {} #@TODO: Remove in 0.4.10 - self.plugins = [] self.supported = [] self.new_supported = [] @@ -97,19 +95,6 @@ class MultiHook(Hook): self.init_periodical(threaded=True) - def getURL(self, *args, **kwargs): #@TODO: Remove in 0.4.10 - """See HTTPRequest for argument list""" - h = pyreq.getHTTPRequest(timeout=120) - try: - if not 'decode' in kwargs: - kwargs['decode'] = True - rep = h.load(*args, **kwargs) - finally: - h.close() - - return rep - - def pluginsCached(self): if self.plugins: return self.plugins diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index da597ef42..0d4c3b165 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -1,65 +1,108 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. +from __future__ import with_statement - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. +import os +import re +import urllib - You should have received a copy of the GNU General Public License - along with this program; if not, see . +from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 +from module.utils import fs_encode, fs_decode, html_unescape, save_join as fs_join - @author: RaNaN, spoob, mkaay -""" -from time import time, sleep -from random import randint +def replace_patterns(string, ruleslist): + for r in ruleslist: + rf, rt = r + string = re.sub(rf, rt, string) + return string -import os -from os import remove, makedirs, chmod, stat -from os.path import exists, join -if os.name != "nt": - from os import chown - from pwd import getpwnam - from grp import getgrnam +def set_cookies(cj, cookies): + for cookie in cookies: + if isinstance(cookie, tuple) and len(cookie) == 3: + domain, name, value = cookie + cj.setCookie(domain, name, value) -from itertools import islice -from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 -from module.utils import save_join as fs_join, save_path as safe_filename, fs_encode, fs_decode +def parseHtmlTagAttrValue(attr_name, tag): + m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^>\s\"'][^>\s]*)\1" % attr_name, tag, re.I) + return m.group(2) if m else None + + +def parseHtmlForm(attr_str, html, input_names={}): + for form in re.finditer(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, + html, re.S | re.I): + inputs = {} + action = parseHtmlTagAttrValue("action", form.group('TAG')) + + for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?= `encode` in 0.4.10 - str(a)).strip() for a in args if a) + msg = fs_encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in args if a)) #@NOTE: `fs_encode` -> `encode` in 0.4.10 log("%(plugin)s%(id)s: %(msg)s" % {'plugin': self.__name__, - 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", - 'msg' : msg or _(level.upper() + " MARK")}) + 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", + 'msg' : msg or _(level.upper() + " MARK")}) def logDebug(self, *args): @@ -100,7 +143,7 @@ class Base(object): return self.setConfig(*args, **kwargs) - def getConfig(self, option, default=""): + def getConfig(self, option, default="", plugin=None): """ Returns config value for current plugin @@ -108,9 +151,10 @@ class Base(object): :return: """ try: - return self.core.config.getPlugin(self.__name__, option) + return self.core.config.getPlugin(plugin or self.__name__, option) except KeyError: + self.logWarning(_("Config option or plugin not found")) return default @@ -149,11 +193,11 @@ class Base(object): def fail(self, reason): """Fail and give reason""" - raise Fail(reason) + raise Fail(fs_encode(reason)) def error(self, reason="", type=_("Parse")): - if not reason and not type: + if not reason: type = _("Unknown") msg = _("%s error") % type.strip().capitalize() if type else _("Error") @@ -163,313 +207,73 @@ class Base(object): raise Fail(msg) -class Plugin(Base): - __name__ = "Plugin" - __type__ = "hoster" - __version__ = "0.10" + def fixurl(self, url): + return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip() - __pattern__ = r'^unmatchable$' - __config__ = [] #: [("name", "type", "desc", "default")] - __description__ = """Base plugin""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("spoob", "spoob@pyload.org"), - ("mkaay", "mkaay@mkaay.de" )] - - - def __init__(self, pyfile): - super(Plugin, self).__init__(pyfile.m.core) - - self.wantReconnect = False - #: enables simultaneous processing of multiple downloads - self.multiDL = True - self.limitDL = 0 - #: chunk limit - self.chunkLimit = 1 - self.resumeDownload = False - - #: time() + wait in seconds - self.waitUntil = 0 - self.waiting = False - - self.ocr = None #captcha reader instance - #: account handler instance, see :py:class:`Account` - self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) - - #: premium status - self.premium = False - #: username/login - self.user = None - - if self.account and not self.account.canUse(): self.account = None - if self.account: - self.user, data = self.account.selectAccount() - #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest(self.user) - self.chunkLimit = -1 # chunk limit, -1 for unlimited - #: enables resume (will be ignored if server dont accept chunks) - self.resumeDownload = True - self.multiDL = True #every hoster with account should provide multiple downloads - #: premium status - self.premium = self.account.isPremium(self.user) - else: - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) - - #: associated pyfile instance, see `PyFile` - self.pyfile = pyfile - self.thread = None # holds thread in future - - #: location where the last call to download was saved - self.lastDownload = "" - #: re match of the last call to `checkDownload` - self.lastCheck = None - #: js engine, see `JsEngine` - self.js = self.core.js - self.cTask = None #captcha task - - self.retries = 0 # amount of retries already made - self.html = None # some plugins store html code here - - self.init() - - def getChunkCount(self): - if self.chunkLimit <= 0: - return self.core.config["download"]["chunks"] - return min(self.core.config["download"]["chunks"], self.chunkLimit) - - def __call__(self): - return self.__name__ - - def init(self): - """initialize the plugin (in addition to `__init__`)""" - pass - - def setup(self): - """ setup for enviroment and other things, called before downloading (possibly more than one time)""" - pass - - def preprocessing(self, thread): - """ handles important things to do before starting """ - self.thread = thread - - if self.account: - self.account.checkLogin(self.user) - else: - self.req.clearCookies() - - self.setup() - - self.pyfile.setStatus("starting") - - return self.process(self.pyfile) - - - def process(self, pyfile): - """the 'main' method of every plugin, you **have to** overwrite it""" - raise NotImplementedError - - def resetAccount(self): - """ dont use account and retry download """ - self.account = None - self.req = self.core.requestFactory.getRequest(self.__name__) - self.retry() - - def checksum(self, local_file=None): - """ - return codes: - 0 - checksum ok - 1 - checksum wrong - 5 - can't get checksum - 10 - not implemented - 20 - unknown error - """ - #@TODO checksum check hook - - return True, 10 - - - def setWait(self, seconds, reconnect=False): - """Set a specific wait time later used with `wait` - - :param seconds: wait time in seconds - :param reconnect: True if a reconnect would avoid wait time + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=True, req=None): """ - if reconnect: - self.wantReconnect = True - self.pyfile.waitUntil = time() + int(seconds) - - def wait(self): - """ waits the time previously set """ - self.waiting = True - self.pyfile.setStatus("waiting") - - while self.pyfile.waitUntil > time(): - self.thread.m.reconnecting.wait(2) - - if self.pyfile.abort: raise Abort - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - self.waiting = False - self.pyfile.setStatus("starting") - - def offline(self): - """ fail and indicate file is offline """ - raise Fail("offline") - - def tempOffline(self): - """ fail and indicates file ist temporary offline, the core may take consequences """ - raise Fail("temp. offline") - - def skip(self, reason): - raise Skip(reason) - - def retry(self, max_tries=3, wait_time=1, reason=""): - """Retries and begin again from the beginning - - :param max_tries: number of maximum retries - :param wait_time: time to wait in seconds - :param reason: reason for retrying, will be passed to fail if max_tries reached - """ - if 0 < max_tries <= self.retries: - if not reason: reason = "Max retries reached" - raise Fail(reason) - - self.wantReconnect = False - self.setWait(wait_time) - self.wait() - - self.retries += 1 - raise Retry(reason) - - def invalidCaptcha(self): - if self.cTask: - self.cTask.invalid() - - def correctCaptcha(self): - if self.cTask: - self.cTask.correct() - - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual'): - """ Loads a captcha and decrypts it with ocr, plugin, user input - - :param url: url of captcha image - :param get: get part for request - :param post: post part for request - :param cookies: True if cookies should be enabled - :param forceUser: if True, ocr is not used - :param imgtype: Type of the Image - :param result_type: 'textual' if text is written on the captcha\ - or 'positional' for captcha where the user have to click\ - on a specific region on the captcha - - :return: result of decrypting - """ - - img = self.load(url, get=get, post=post, cookies=cookies) - - id = ("%.2f" % time())[-6:].replace(".", "") - temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - temp_file.write(img) - temp_file.close() - - has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins - - if self.core.captcha: - Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) - else: - Ocr = None - - if Ocr and not forceUser: - sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: raise Abort - - ocr = Ocr() - result = ocr.get_captcha(temp_file.name) - else: - captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) - self.cTask = task - captchaManager.handleCaptcha(task) - - while task.isWaiting(): - if self.pyfile.abort: - captchaManager.removeTask(task) - raise Abort - sleep(1) - - captchaManager.removeTask(task) - - if task.error and has_plugin: #ignore default error message since the user could use OCR - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) - - result = task.result - self.core.log.debug("Received captcha result: %s" % str(result)) - - if not self.core.debug: - try: - remove(temp_file.name) - except: - pass - - return result - - - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): - """Load content at url and returns it + Load content at url and returns it :param url: :param get: :param post: :param ref: :param cookies: - :param just_header: if True only the header will be retrieved and returned as dict + :param just_header: If True only the header will be retrieved and returned as dict :param decode: Wether to decode the output according to http header, should be True in most cases :return: Loaded content """ - if self.pyfile.abort: raise Abort - #utf8 vs decode -> please use decode attribute in all future plugins - if type(url) == unicode: url = str(url) + if hasattr(self, 'pyfile') and self.pyfile.abort: + self.abort() + + url = self.fixurl(url) - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) + if not url or not isinstance(url, basestring): + self.fail(_("No url given")) if self.core.debug: - from inspect import currentframe + self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + + if req is None: + if hasattr(self, "req"): + req = self.req + else: + req = self.core.requestFactory.getRequest(self.__name__) - frame = currentframe() - if not exists(join("tmp", self.__name__)): - makedirs(join("tmp", self.__name__)) + res = req.load(url, get, post, ref, cookies, just_header, True, bool(decode)) - f = open( - join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) - , "wb") - del frame # delete the frame or it wont be cleaned + if decode: + res = html_unescape(res) + if isinstance(decode, basestring): + res = res.decode(decode) + + if self.core.debug: + import inspect + + frame = inspect.currentframe() + framefile = fs_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) try: - tmp = res.encode("utf8") - except: - tmp = res + if not os.path.exists(os.path.join("tmp", self.__name__)): + os.makedirs(os.path.join("tmp", self.__name__)) - f.write(tmp) - f.close() + with open(framefile, "wb") as f: + del frame #: delete the frame or it wont be cleaned + f.write(res.encode('utf8')) + except IOError, e: + self.logError(e) if just_header: - #parse header - header = {"code": self.req.code} + # parse header + header = {"code": req.code} for line in res.splitlines(): line = line.strip() - if not line or ":" not in line: continue + if not line or ":" not in line: + continue key, none, value = line.partition(":") - key = key.lower().strip() + key = key.strip().lower() value = value.strip() if key in header: @@ -481,164 +285,4 @@ class Plugin(Base): header[key] = value res = header - return res - - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): - """Downloads the content at url to download folder - - :param url: - :param get: - :param post: - :param ref: - :param cookies: - :param disposition: if True and server provides content-disposition header\ - the filename will be changed if needed - :return: The location where the file was saved - """ - - self.checkForSameFiles() - - self.pyfile.setStatus("downloading") - - download_folder = self.core.config['general']['download_folder'] - - location = fs_join(download_folder, self.pyfile.package().folder) - - if not exists(location): - makedirs(location, int(self.core.config["permission"]["folder"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.core.config["permission"]["user"])[2] - gid = getgrnam(self.core.config["permission"]["group"])[2] - - chown(location, uid, gid) - except Exception, e: - self.core.log.warning(_("Setting User and Group failed: %s") % str(e)) - - # convert back to unicode - location = fs_decode(location) - name = safe_filename(self.pyfile.name) - - filename = join(location, name) - - self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) - - try: - newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, - chunks=self.getChunkCount(), resume=self.resumeDownload, - progressNotify=self.pyfile.setProgress, disposition=disposition) - finally: - self.pyfile.size = self.req.size - - if disposition and newname and newname != name: #triple check, just to be sure - self.core.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) - self.pyfile.name = newname - filename = join(location, newname) - - fs_filename = fs_encode(filename) - - if self.core.config["permission"]["change_file"]: - chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.core.config["permission"]["user"])[2] - gid = getgrnam(self.core.config["permission"]["group"])[2] - - chown(fs_filename, uid, gid) - except Exception, e: - self.core.log.warning(_("Setting User and Group failed: %s") % str(e)) - - self.lastDownload = filename - return self.lastDownload - - def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): - """ checks the content of the last downloaded file, re match is saved to `lastCheck` - - :param rules: dict with names and rules to match (compiled regexp or strings) - :param api_size: expected file size - :param max_size: if the file is larger then it wont be checked - :param delete: delete if matched - :param read_size: amount of bytes to read from files larger then max_size - :return: dictionary key of the first rule that matched - """ - lastDownload = fs_encode(self.lastDownload) - if not exists(lastDownload): return None - - size = stat(lastDownload) - size = size.st_size - - if api_size and api_size <= size: return None - elif size > max_size and not read_size: return None - self.core.log.debug("Download Check triggered") - f = open(lastDownload, "rb") - content = f.read(read_size if read_size else -1) - f.close() - #produces encoding errors, better log to other file in the future? - #self.core.log.debug("Content: %s" % content) - for name, rule in rules.iteritems(): - if type(rule) in (str, unicode): - if rule in content: - if delete: - remove(lastDownload) - return name - elif hasattr(rule, "search"): - m = rule.search(content) - if m: - if delete: - remove(lastDownload) - self.lastCheck = m - return name - - - def getPassword(self): - """ get the password the user provided in the package""" - password = self.pyfile.package().password - if not password: return "" - return password - - - def checkForSameFiles(self, starting=False): - """ checks if same file was/is downloaded within same package - - :param starting: indicates that the current download is going to start - :raises Skip: - """ - - pack = self.pyfile.package() - - for pyfile in self.core.files.cache.values(): - if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: - if pyfile.status in (0, 12): #finished or downloading - self.skip(pyfile.pluginname) - elif pyfile.status in ( - 5, 7) and starting: #a download is waiting/starting and was appenrently started before - self.skip(pyfile.pluginname) - - download_folder = self.core.config['general']['download_folder'] - location = fs_join(download_folder, pack.folder, self.pyfile.name) - - if starting and self.core.config['download']['skip_existing'] and exists(location): - size = os.stat(location).st_size - if size >= self.pyfile.size: - self.skip("File exists") - - pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) - if pyfile: - if exists(location): - self.skip(pyfile[0]) - - self.core.log.debug("File %s not skipped, because it does not exists" % self.pyfile.name) - - def clean(self): - """ clean everything and remove references """ - if hasattr(self, "pyfile"): - del self.pyfile - if hasattr(self, "req"): - self.req.close() - del self.req - if hasattr(self, "thread"): - del self.thread - if hasattr(self, "html"): - del self.html + return res \ No newline at end of file diff --git a/module/plugins/internal/ReCaptcha.py b/module/plugins/internal/ReCaptcha.py index a9d0f3752..40faff5f0 100644 --- a/module/plugins/internal/ReCaptcha.py +++ b/module/plugins/internal/ReCaptcha.py @@ -51,7 +51,7 @@ class ReCaptcha(Captcha): def _challenge_v1(self, key): - html = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", + html = self.plugin.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}) try: challenge = re.search("challenge : '(.+?)',", html).group(1) @@ -66,8 +66,8 @@ class ReCaptcha(Captcha): def result(self, server, challenge, key): - self.plugin.req.load("http://www.google.com/recaptcha/api/js/recaptcha.js") - html = self.plugin.req.load("http://www.google.com/recaptcha/api/reload", + self.plugin.load("http://www.google.com/recaptcha/api/js/recaptcha.js") + html = self.plugin.load("http://www.google.com/recaptcha/api/reload", get={'c' : challenge, 'k' : key, 'reason': "i", @@ -92,7 +92,7 @@ class ReCaptcha(Captcha): def _collectApiInfo(self): - html = self.plugin.req.load("http://www.google.com/recaptcha/api.js") + html = self.plugin.load("http://www.google.com/recaptcha/api.js") a = re.search(r'po.src = \'(.*?)\';', html).group(1) vers = a.split("/")[5] @@ -102,7 +102,7 @@ class ReCaptcha(Captcha): self.logDebug("API language: %s" % language) - html = self.plugin.req.load("https://apis.google.com/js/api.js") + html = self.plugin.load("https://apis.google.com/js/api.js") b = re.search(r'"h":"(.*?)","', html).group(1) jsh = b.decode('unicode-escape') @@ -112,7 +112,7 @@ class ReCaptcha(Captcha): def _prepareTimeAndRpc(self): - self.plugin.req.load("http://www.google.com/recaptcha/api2/demo") + self.plugin.load("http://www.google.com/recaptcha/api2/demo") millis = int(round(time.time() * 1000)) @@ -139,7 +139,7 @@ class ReCaptcha(Captcha): vers, language, jsh = self._collectApiInfo() millis, rpc = self._prepareTimeAndRpc() - html = self.plugin.req.load("https://www.google.com/recaptcha/api2/anchor", + html = self.plugin.load("https://www.google.com/recaptcha/api2/anchor", get={'k' : key, 'hl' : language, 'v' : vers, @@ -152,14 +152,15 @@ class ReCaptcha(Captcha): token1 = re.search(r'id="recaptcha-token" value="(.*?)">', html) self.logDebug("Token #1: %s" % token1.group(1)) - html = self.plugin.req.load("https://www.google.com/recaptcha/api2/frame", - get={'c' : token1.group(1), - 'hl' : language, - 'v' : vers, - 'bg' : botguardstring, - 'k' : key, - 'usegapi': "1", - 'jsh' : jsh}).decode('unicode-escape') + html = self.plugin.load("https://www.google.com/recaptcha/api2/frame", + get={'c' : token1.group(1), + 'hl' : language, + 'v' : vers, + 'bg' : botguardstring, + 'k' : key, + 'usegapi': "1", + 'jsh' : jsh}, + decode="unicode-escape") token2 = re.search(r'"finput","(.*?)",', html) self.logDebug("Token #2: %s" % token2.group(1)) @@ -179,7 +180,7 @@ class ReCaptcha(Captcha): timeToSolve = int(round(time.time() * 1000)) - millis_captcha_loading timeToSolveMore = timeToSolve + int(float("0." + str(random.randint(1, 99999999))) * 500) - html = self.plugin.req.load("https://www.google.com/recaptcha/api2/userverify", + html = self.plugin.load("https://www.google.com/recaptcha/api2/userverify", post={'k' : key, 'c' : token3.group(1), 'response': response, diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 3e8b89f79..db253d950 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -54,7 +54,7 @@ class SimpleCrypter(Crypter, SimpleHoster): #@TODO: Remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") - account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name) + account = self.core.accountManager.getAccountPlugin(account_name) if account and account.canUse(): self.user, data = account.selectAccount() @@ -68,9 +68,9 @@ class SimpleCrypter(Crypter, SimpleHoster): self.pyfile.error = "" #@TODO: Remove in 0.4.10 self.info = {} - self.html = "" - self.link = "" #@TODO: Move to Hoster in 0.4.10 - self.links = [] #@TODO: Move to Hoster in 0.4.10 + self.html = "" #@TODO: Recheck in 0.4.10 + self.link = "" #@TODO: Recheck in 0.4.10 + self.links = [] if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) @@ -87,11 +87,11 @@ class SimpleCrypter(Crypter, SimpleHoster): def handleDirect(self, pyfile): - for i in xrange(10): #@TODO: Use `pycurl.MAXREDIRS` value in 0.4.10 + for i in xrange(self.getConfig("maxredirs", plugin="UserAgentSwitcher")): redirect = self.link or pyfile.url self.logDebug("Redirect #%d to: %s" % (i, redirect)) - header = self.load(redirect, just_header=True, decode=True) + header = self.load(redirect, just_header=True) if 'location' in header and header['location']: self.link = header['location'] else: @@ -107,13 +107,13 @@ class SimpleCrypter(Crypter, SimpleHoster): self.handleDirect(pyfile) if self.link: - self.urls = self.fixurls([self.link]) + self.urls = [self.link] else: self.preload() self.checkInfo() - self.links = self.fixurls(self.getLinks()) or list() + self.links = self.getLinks() or list() if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): self.handlePages(pyfile) @@ -123,9 +123,6 @@ class SimpleCrypter(Crypter, SimpleHoster): if self.links: self.packages = [(self.info['name'], self.links, self.info['folder'])] - elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 - self.fail(_("No link grabbed")) - def checkNameSize(self, getinfo=True): if not self.info or getinfo: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index cec5f55f1..979da24a9 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -11,85 +11,17 @@ import urllib import urlparse from module.PyFile import statusMap as _statusMap -from module.network.CookieJar import CookieJar from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL from module.plugins.internal.Hoster import Hoster -from module.plugins.internal.Plugin import Fail, Retry -from module.utils import fixup, fs_encode, html_unescape, parseFileSize +from module.plugins.internal.Plugin import Fail, Retry, replace_patterns, set_cookies +from module.utils import fixup, fs_encode, parseFileSize #@TODO: Adapt and move to PyFile in 0.4.10 statusMap = dict((v, k) for k, v in _statusMap.iteritems()) -#@TODO: Remove in 0.4.10 -def _wait(self, seconds, reconnect): - if seconds: - self.setWait(int(seconds) + 1) - - if reconnect is not None: - self.wantReconnect = reconnect - - super(SimpleHoster, self).wait() - - -def replace_patterns(string, ruleslist): - for r in ruleslist: - rf, rt = r - string = re.sub(rf, rt, string) - return string - - -def set_cookies(cj, cookies): - for cookie in cookies: - if isinstance(cookie, tuple) and len(cookie) == 3: - domain, name, value = cookie - cj.setCookie(domain, name, value) - - -def parseHtmlTagAttrValue(attr_name, tag): - m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^>\s\"'][^>\s]*)\1" % attr_name, tag, re.I) - return m.group(2) if m else None - - -def parseHtmlForm(attr_str, html, input_names={}): - for form in re.finditer(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, - html, re.S | re.I): - inputs = {} - action = parseHtmlTagAttrValue("action", form.group('TAG')) - - for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?= Date: Tue, 16 Jun 2015 18:05:15 +0200 Subject: [Hoster] Import fixup --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 9db22d1d7..834a200ac 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -12,7 +12,7 @@ if os.name != "nt": import grp import pwd -from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip +from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip, parseHtmlForm from module.utils import fs_decode, fs_encode, save_join as fs_join -- cgit v1.2.3 From d99d6eddb6af637580bb6fc72013f913077525d6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 17 Jun 2015 11:23:08 +0200 Subject: Spare fixes --- module/plugins/internal/Captcha.py | 10 ++--- module/plugins/internal/Extractor.py | 27 ++++++++----- module/plugins/internal/Hoster.py | 78 +++++++++++++++++++++--------------- module/plugins/internal/OCR.py | 43 +++++++++++--------- 4 files changed, 90 insertions(+), 68 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 8dbc33da2..4629c9522 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -3,7 +3,6 @@ from module.plugins.internal.Plugin import Plugin -#@TODO: Extend (new) Plugin class; remove all `html` args class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" @@ -11,16 +10,15 @@ class Captcha(Plugin): __description__ = """Base captcha service plugin""" __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] - - - key = None #: last key detected + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] def __init__(self, plugin): - self.plugin = plugin super(Captcha, self).__init__(plugin.core) + self.plugin = plugin + self.key = None #: last key detected + #@TODO: Recheck in 0.4.10 def retrieve_key(self, html): diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 1a98060d9..2aa26e64a 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -4,6 +4,7 @@ import os import re from module.PyFile import PyFile +from module.plugins.internal.Plugin import Plugin class ArchiveError(Exception): @@ -18,8 +19,9 @@ class PasswordError(Exception): pass -class Extractor: +class Extractor(Plugin): __name__ = "Extractor" + __type__ = "extractor" __version__ = "0.24" __description__ = """Base extractor plugin""" @@ -46,7 +48,8 @@ class Extractor: @classmethod def isUsable(cls): - """ Check if system statisfy dependencies + """ + Check if system statisfy dependencies :return: boolean """ return None @@ -54,7 +57,8 @@ class Extractor: @classmethod def getTargets(cls, files_ids): - """ Filter suited targets from list of filename id tuple list + """ + Filter suited targets from list of filename id tuple list :param files_ids: List of filepathes :return: List of targets, id tuple list """ @@ -78,7 +82,7 @@ class Extractor: delete='No', keepbroken=False, fid=None): - """ Initialize extractor for specific file """ + """Initialize extractor for specific file""" self.manager = manager self.filename = filename self.out = out @@ -95,12 +99,13 @@ class Extractor: def init(self): - """ Initialize additional data structures """ + """Initialize additional data structures""" pass def check(self): - """Quick Check by listing content of archive. + """ + Quick Check by listing content of archive. Raises error if password is needed, integrity is questionable or else. :raises PasswordError @@ -109,8 +114,10 @@ class Extractor: """ raise NotImplementedError + def verify(self): - """Testing with Extractors buildt-in method + """ + Testing with Extractors buildt-in method Raises error if password is needed, integrity is questionable or else. :raises PasswordError @@ -125,7 +132,8 @@ class Extractor: def extract(self, password=None): - """Extract the archive. Raise specific errors in case of failure. + """ + Extract the archive. Raise specific errors in case of failure. :param progress: Progress function, call this to update status :param password password to use @@ -138,7 +146,8 @@ class Extractor: def getDeleteFiles(self): - """Return list of files to delete, do *not* delete them here. + """ + Return list of files to delete, do *not* delete them here. :return: List with paths of files to delete """ diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 834a200ac..40430c38d 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -178,11 +178,11 @@ class Hoster(Plugin): :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ - wait_time = int(seconds) + 1 - wait_until = time.time() + wait_time + wait_time = max(int(seconds), 1) + wait_until = time.time() + wait_time + 1 self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), - "Wait: %d seconds" % wait_time) + "Wait: %d(+1) seconds" % wait_time) self.pyfile.waitUntil = wait_until @@ -190,13 +190,13 @@ class Hoster(Plugin): self.setReconnect(reconnect) - def wait(self, seconds=None, reconnect=None): + def wait(self, seconds=0, reconnect=None): """ Waits the time previously set """ pyfile = self.pyfile - if seconds is not None: + if seconds > 0: self.setWait(seconds) if reconnect is not None: @@ -284,7 +284,7 @@ class Hoster(Plugin): self.retries[id] = 0 if 0 < max_tries <= self.retries[id]: - self.fail(reason or _("Max retries reached"), "retry") + self.fail(reason or _("Max retries reached"), _("retry")) self.wait(wait_time, False) @@ -480,7 +480,7 @@ class Hoster(Plugin): return self.lastDownload - def checkDownload(self, rules, delete=True, file_size=None, size_tolerance=1000, read_size=100000): + def checkDownload(self, rules, delete=True, file_size=0, size_tolerance=1000, read_size=100000): """ Checks the content of the last downloaded file, re match is saved to `lastCheck` @@ -491,40 +491,52 @@ class Hoster(Plugin): :param read_size: amount of bytes to read from files :return: dictionary key of the first rule that matched """ + do_delete = False lastDownload = fs_encode(self.lastDownload) if not self.lastDownload or not os.path.exists(lastDownload): self.lastDownload = "" self.fail(self.pyfile.error or _("No file downloaded")) - download_size = os.stat(lastDownload).st_size + try: + download_size = os.stat(lastDownload).st_size + + if download_size < 1: + do_delete = True + self.fail(_("Empty file")) + + elif file_size > 0: + diff = abs(file_size - download_size) + + if diff > size_tolerance: + do_delete = True + self.fail(_("File size mismatch")) + + elif diff != 0: + self.logWarning(_("File size is not equal to expected size")) - if download_size < 1 or (file_size and abs(file_size - download_size) > size_tolerance): - if delete: + self.logDebug("Download Check triggered") + + with open(lastDownload, "rb") as f: + content = f.read(read_size) + + # produces encoding errors, better log to other file in the future? + # self.logDebug("Content: %s" % content) + for name, rule in rules.iteritems(): + if isinstance(rule, basestring): + if rule in content: + do_delete = True + return name + + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + do_delete = True + self.lastCheck = m + return name + finally: + if delete and do_delete: os.remove(lastDownload) - self.fail(_("Empty file")) - - self.logDebug("Download Check triggered") - - with open(lastDownload, "rb") as f: - content = f.read(read_size) - - # produces encoding errors, better log to other file in the future? - # self.logDebug("Content: %s" % content) - for name, rule in rules.iteritems(): - if isinstance(rule, basestring): - if rule in content: - if delete: - os.remove(lastDownload) - return name - - elif hasattr(rule, "search"): - m = rule.search(content) - if m: - if delete: - os.remove(lastDownload) - self.lastCheck = m - return name def directLink(self, url, follow_location=None): diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 1782e17f0..2d41ab39e 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -11,12 +11,13 @@ except ImportError: import logging import os import subprocess -#import tempfile +# import tempfile +from module.plugins.internal.Plugin import Plugin from module.utils import save_join as fs_join -class OCR(object): +class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" __version__ = "0.11" @@ -37,7 +38,7 @@ class OCR(object): def deactivate(self): - """delete all tmp images""" + """Delete all tmp images""" pass @@ -48,21 +49,21 @@ class OCR(object): def run(self, command): """Run a command""" - popen = subprocess.Popen(command, bufsize = -1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + popen = subprocess.Popen(command, bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) popen.wait() - output = popen.stdout.read() +" | "+ popen.stderr.read() + output = popen.stdout.read() + " | " + popen.stderr.read() popen.stdout.close() popen.stderr.close() self.logger.debug("Tesseract ReturnCode %s Output: %s" % (popen.returncode, output)) - def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True): - #tmpTif = tempfile.NamedTemporaryFile(suffix=".tif") + def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True, pagesegmode=None): + # tmpTif = tempfile.NamedTemporaryFile(suffix=".tif") try: tmpTif = open(fs_join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") tmpTif.close() - #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") + # tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") tmpTxt = open(fs_join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") tmpTxt.close() @@ -78,10 +79,13 @@ class OCR(object): else: tessparams = ["tesseract"] - tessparams.extend( [os.path.abspath(tmpTif.name), os.path.abspath(tmpTxt.name).replace(".txt", "")] ) + tessparams.extend([os.path.abspath(tmpTif.name), os.path.abspath(tmpTxt.name).replace(".txt", "")]) + + if pagesegmode: + tessparams.extend(["-psm", str(pagesegmode)]) if subset and (digits or lowercase or uppercase): - #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") + # tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") with open(fs_join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") as tmpSub: tmpSub.write("tessedit_char_whitelist ") @@ -151,11 +155,11 @@ class OCR(object): count = 0 try: - if pixels[x-1, y-1] != 255: + if pixels[x - 1, y - 1] != 255: count += 1 - if pixels[x-1, y] != 255: + if pixels[x - 1, y] != 255: count += 1 - if pixels[x-1, y + 1] != 255: + if pixels[x - 1, y + 1] != 255: count += 1 if pixels[x, y + 1] != 255: count += 1 @@ -163,19 +167,19 @@ class OCR(object): count += 1 if pixels[x + 1, y] != 255: count += 1 - if pixels[x + 1, y-1] != 255: + if pixels[x + 1, y - 1] != 255: count += 1 - if pixels[x, y-1] != 255: + if pixels[x, y - 1] != 255: count += 1 except Exception: pass - # not enough neighbors are dark pixels so mark this pixel - # to be changed to white + # not enough neighbors are dark pixels so mark this pixel + # to be changed to white if count < allowed: pixels[x, y] = 1 - # second pass: this time set all 1's to 255 (white) + # second pass: this time set all 1's to 255 (white) for x in xrange(w): for y in xrange(h): if pixels[x, y] == 1: @@ -185,7 +189,7 @@ class OCR(object): def derotate_by_average(self): - """rotate by checking each angle and guess most suitable""" + """Rotate by checking each angle and guess most suitable""" w, h = self.image.size pixels = self.pixels @@ -211,7 +215,6 @@ class OCR(object): if pixels[x, y] == 0: pixels[x, y] = 255 - count = {} for x in xrange(w): -- cgit v1.2.3 From 164512b6a74c94a731fcee7435dce1ccfa2f71e7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 17 Jun 2015 18:29:50 +0200 Subject: Spare code cosmetics --- module/plugins/internal/Account.py | 17 ++++++++++++----- module/plugins/internal/Container.py | 12 +++++++----- module/plugins/internal/Crypter.py | 17 ++++++++++------- module/plugins/internal/Extractor.py | 12 +++++++++--- module/plugins/internal/Hook.py | 29 +++++++++++++++++++++-------- module/plugins/internal/Hoster.py | 1 - module/plugins/internal/MultiHook.py | 8 ++++++-- module/plugins/internal/OCR.py | 14 +++++++++----- module/plugins/internal/Plugin.py | 32 ++++++++++++++++++++++++-------- module/plugins/internal/SimpleCrypter.py | 1 - module/plugins/internal/SimpleHoster.py | 1 - module/plugins/internal/XFSHoster.py | 4 +++- 12 files changed, 101 insertions(+), 47 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 0e2996aaa..0595da20b 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -106,8 +106,9 @@ class Account(Plugin): def updateAccounts(self, user, password=None, options={}): - """Updates account and return true if anything changed""" - + """ + Updates account and return true if anything changed + """ if user in self.accounts: self.accounts[user]['valid'] = True #: do not remove or accounts will not login if password: @@ -227,7 +228,9 @@ class Account(Plugin): def selectAccount(self): - """Returns an valid account name and data""" + """ + Returns an valid account name and data + """ usable = [] for user, data in self.accounts.iteritems(): if not data['valid']: @@ -290,14 +293,18 @@ class Account(Plugin): def scheduleRefresh(self, user, time=0, force=True): - """Add task to refresh account info to sheduler""" + """ + Add task to refresh account info to sheduler + """ self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) @lock def checkLogin(self, user): - """Checks if user is still logged in""" + """ + Checks if user is still logged in + """ if user in self.timestamps: if self.login_timeout > 0 and self.timestamps[user] + self.login_timeout * 60 < time.time(): self.logDebug("Reached login timeout for %s" % user) diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 097b04ac7..fc1a93606 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -23,8 +23,9 @@ class Container(Crypter): def preprocessing(self, thread): - """Prepare""" - + """ + Prepare + """ self.setup() self.thread = thread @@ -37,9 +38,10 @@ class Container(Crypter): def _load2disk(self): - """Loads container to disk if its stored remotely and overwrite url, - or check existent on several places at disk""" - + """ + Loads container to disk if its stored remotely and overwrite url, + or check existent on several places at disk + """ if self.pyfile.url.startswith("http"): self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] content = self.load(self.pyfile.url) diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index a713c7da6..e97f22316 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -34,8 +34,9 @@ class Crypter(Hoster): def process(self, pyfile): - """Main method""" - + """ + Main method + """ self.decrypt(pyfile) if self.urls: @@ -52,20 +53,22 @@ class Crypter(Hoster): def _generate_packages(self): - """Generate new packages from self.urls""" - + """ + Generate new packages from self.urls + """ packages = [(name, links, None) for name, links in self.core.api.generatePackages(self.urls).iteritems()] self.packages.extend(packages) def _create_packages(self): - """Create new packages from self.packages""" - + """ + Create new packages from self.packages + """ package_folder = self.pyfile.package().folder package_password = self.pyfile.package().password package_queue = self.pyfile.package().queue - folder_per_package = self.core.config.get('general', 'folder_per_package') + folder_per_package = self.core.config.get("general", "folder_per_package") use_subfolder = self.getConfig('use_subfolder', folder_per_package) subfolder_per_package = self.getConfig('subfolder_per_package', True) diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 2aa26e64a..f73388d8c 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -82,7 +82,9 @@ class Extractor(Plugin): delete='No', keepbroken=False, fid=None): - """Initialize extractor for specific file""" + """ + Initialize extractor for specific file + """ self.manager = manager self.filename = filename self.out = out @@ -99,7 +101,9 @@ class Extractor(Plugin): def init(self): - """Initialize additional data structures""" + """ + Initialize additional data structures + """ pass @@ -155,5 +159,7 @@ class Extractor(Plugin): def list(self, password=None): - """Populate self.files at some point while extracting""" + """ + Populate self.files at some point while extracting + """ return self.files diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 01ffbc5f2..3a0431feb 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -6,8 +6,9 @@ from module.plugins.internal.Plugin import Plugin class Expose(object): - """Used for decoration to declare rpc services""" - + """ + Used for decoration to declare rpc services + """ def __new__(cls, f, *args, **kwargs): hookManager.addRPC(f.__module__, f.func_name, f.func_doc) return f @@ -113,12 +114,16 @@ class Hook(Plugin): def setup(self): - """More init stuff if needed""" + """ + More init stuff if needed + """ pass def is_activated(self): - """Checks if hook is activated""" + """ + Checks if hook is activated + """ return self.getConfig("activated") @@ -128,7 +133,9 @@ class Hook(Plugin): def deactivate(self): - """Called when hook was deactivated""" + """ + Called when hook was deactivated + """ pass @@ -138,7 +145,9 @@ class Hook(Plugin): def activate(self): - """Called when hook was activated""" + """ + Called when hook was activated + """ pass @@ -148,7 +157,9 @@ class Hook(Plugin): def exit(self): - """Called by core.shutdown just before pyLoad exit""" + """ + Called by core.shutdown just before pyLoad exit + """ pass @@ -212,7 +223,9 @@ class Hook(Plugin): def captcha_task(self, task): - """New captcha task for the plugin, it MUST set the handler and timeout or will be ignored""" + """ + New captcha task for the plugin, it MUST set the handler and timeout or will be ignored + """ pass diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 40430c38d..32c3312d0 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -321,7 +321,6 @@ class Hoster(Plugin): :return: result of decrypting """ - img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time.time())[-6:].replace(".", "") diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 0bcab5915..739b225b4 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -153,7 +153,9 @@ class MultiHook(Hook): def periodical(self): - """Reload plugin list periodically""" + """ + Reload plugin list periodically + """ self.loadAccount() if self.getConfig('reload', True): @@ -244,7 +246,9 @@ class MultiHook(Hook): def deactivate(self): - """Remove override for all plugins. Scheduler job is removed by hookmanager""" + """ + Remove override for all plugins. Scheduler job is removed by hookmanager + """ for plugin in self.supported: self.unloadPlugin(plugin) diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 2d41ab39e..5fe6f2532 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -38,7 +38,9 @@ class OCR(Plugin): def deactivate(self): - """Delete all tmp images""" + """ + Delete all tmp images + """ pass @@ -47,8 +49,9 @@ class OCR(Plugin): def run(self, command): - """Run a command""" - + """ + Run a command + """ popen = subprocess.Popen(command, bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) popen.wait() output = popen.stdout.read() + " | " + popen.stderr.read() @@ -189,8 +192,9 @@ class OCR(Plugin): def derotate_by_average(self): - """Rotate by checking each angle and guess most suitable""" - + """ + Rotate by checking each angle and guess most suitable + """ w, h = self.image.size pixels = self.pixels diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 0d4c3b165..68169e077 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -139,7 +139,9 @@ class Plugin(object): #: Deprecated method def setConf(self, *args, **kwargs): - """See `setConfig`""" + """ + See `setConfig` + """ return self.setConfig(*args, **kwargs) @@ -160,39 +162,53 @@ class Plugin(object): #: Deprecated method def getConf(self, *args, **kwargs): - """See `getConfig`""" + """ + See `getConfig` + """ return self.getConfig(*args, **kwargs) def store(self, key, value): - """Saves a value persistently to the database""" + """ + Saves a value persistently to the database + """ self.core.db.setStorage(self.__name__, key, value) #: Deprecated method def setStorage(self, *args, **kwargs): - """Same as `setStorage`""" + """ + Same as `setStorage` + """ return self.store(*args, **kwargs) def retrieve(self, key, default=None): - """Retrieves saved value or dict of all saved entries if key is None""" + """ + Retrieves saved value or dict of all saved entries if key is None + """ return self.core.db.getStorage(self.__name__, key) or default #: Deprecated method def getStorage(self, *args, **kwargs): - """Same as `getStorage`""" + """ + Same as `getStorage` + """ return self.retrieve(*args, **kwargs) def delStorage(self, key): - """Delete entry in db""" + """ + Delete entry in db + """ self.core.db.delStorage(self.__name__, key) def fail(self, reason): - """Fail and give reason""" + """ + Fail and give reason + """ raise Fail(fs_encode(reason)) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index db253d950..7685be55a 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -50,7 +50,6 @@ class SimpleCrypter(Crypter, SimpleHoster): def loadPage(self, page_n): return the html of the page number page_n """ - #@TODO: Remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 979da24a9..067b97747 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -143,7 +143,6 @@ class SimpleHoster(Hoster): LINK_PREMIUM_PATTERN: (optional) group(1) should be the direct link for premium download example: LINK_PREMIUM_PATTERN = r'
Date: Wed, 17 Jun 2015 18:59:20 +0200 Subject: Spare code cosmetics (2) --- module/plugins/internal/Account.py | 4 ++-- module/plugins/internal/AdYouLike.py | 22 +++++++++++----------- module/plugins/internal/Crypter.py | 2 +- module/plugins/internal/Hook.py | 4 ++-- module/plugins/internal/Hoster.py | 6 +++--- module/plugins/internal/MultiHook.py | 6 +++--- module/plugins/internal/OCR.py | 14 +++++++------- module/plugins/internal/Plugin.py | 6 +++--- module/plugins/internal/SevenZip.py | 6 +++--- module/plugins/internal/UnRar.py | 28 ++++++++++++++-------------- 10 files changed, 49 insertions(+), 49 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 0595da20b..d2efe7996 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -57,7 +57,7 @@ class Account(Plugin): @lock def _login(self, user, data): - # set timestamp for login + #: set timestamp for login self.timestamps[user] = time.time() req = self.getAccountRequest(user) @@ -192,7 +192,7 @@ class Account(Plugin): """ return {"validuntil" : None, #: -1 for unlimited "login" : name, - # "password" : self.accounts[name]['password'], #: commented due security reason + #: "password" : self.accounts[name]['password'], #: commented due security reason "options" : self.accounts[name]['options'], "valid" : self.accounts[name]['valid'], "trafficleft": None, #: in bytes, -1 for unlimited diff --git a/module/plugins/internal/AdYouLike.py b/module/plugins/internal/AdYouLike.py index b7324ef8e..f623ed268 100644 --- a/module/plugins/internal/AdYouLike.py +++ b/module/plugins/internal/AdYouLike.py @@ -37,8 +37,8 @@ class AdYouLike(Captcha): def challenge(self, key=None, html=None): ayl, callback = key or self.retrieve_key(html) - # {"adyoulike":{"key":"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, - # "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} + #: {"adyoulike":{"key":"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, + #: "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} ayl = json_loads(ayl) html = self.plugin.load("http://api-ayl.appspot.com/challenge", @@ -57,15 +57,15 @@ class AdYouLike(Captcha): def result(self, server, challenge): - # Adyoulike.g._jsonp_5579316662423138 - # ({"translations":{"fr":{"instructions_visual":"Recopiez « Soonnight » ci-dessous :"}}, - # "site_under":true,"clickable":true,"pixels":{"VIDEO_050":[],"DISPLAY":[],"VIDEO_000":[],"VIDEO_100":[], - # "VIDEO_025":[],"VIDEO_075":[]},"medium_type":"image/adyoulike", - # "iframes":{"big":""},"shares":{},"id":256, - # "token":"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1","formats":{"small":{"y":300,"x":0,"w":300,"h":60}, - # "big":{"y":0,"x":0,"w":300,"h":250},"hover":{"y":440,"x":0,"w":300,"h":60}}, - # "tid":"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) + #: Adyoulike.g._jsonp_5579316662423138 + #: ({"translations":{"fr":{"instructions_visual":"Recopiez « Soonnight » ci-dessous :"}}, + #: "site_under":true,"clickable":true,"pixels":{"VIDEO_050":[],"DISPLAY":[],"VIDEO_000":[],"VIDEO_100":[], + #: "VIDEO_025":[],"VIDEO_075":[]},"medium_type":"image/adyoulike", + #: "iframes":{"big":""},"shares":{},"id":256, + #: "token":"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1","formats":{"small":{"y":300,"x":0,"w":300,"h":60}, + #: "big":{"y":0,"x":0,"w":300,"h":250},"hover":{"y":440,"x":0,"w":300,"h":60}}, + #: "tid":"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) if isinstance(server, basestring): server = json_loads(server) diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index e97f22316..d8cda17d4 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -82,7 +82,7 @@ class Crypter(Hoster): if package_password: self.core.api.setPackageData(pid, {"password": package_password}) - # Workaround to do not break API addPackage method + #: Workaround to do not break API addPackage method setFolder = lambda x: self.core.api.setPackageData(pid, {"folder": x or ""}) if use_subfolder: diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 3a0431feb..5959089b5 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -46,7 +46,7 @@ class Hook(Plugin): #: automatically register event listeners for functions, attribute will be deleted dont use it yourself self.event_map = {} - # Deprecated alternative to event_map + #: Deprecated alternative to event_map #: List of events the plugin can handle, name the functions exactly like eventname. self.event_list = [] #@NOTE: dont make duplicate entries in event_map @@ -68,7 +68,7 @@ class Hook(Plugin): else: self.manager.addEvent(event, getattr(self, funcs)) - # delete for various reasons + #: delete for various reasons self.event_map = None if self.event_list: diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 32c3312d0..e1a13af01 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -435,7 +435,7 @@ class Hoster(Plugin): except Exception, e: self.fail(e) - # convert back to unicode + #: convert back to unicode location = fs_decode(location) name = safe_filename(self.pyfile.name) @@ -519,8 +519,8 @@ class Hoster(Plugin): with open(lastDownload, "rb") as f: content = f.read(read_size) - # produces encoding errors, better log to other file in the future? - # self.logDebug("Content: %s" % content) + #: produces encoding errors, better log to other file in the future? + #: self.logDebug("Content: %s" % content) for name, rule in rules.iteritems(): if isinstance(rule, basestring): if rule in content: diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 739b225b4..22b0fd7c6 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -207,7 +207,7 @@ class MultiHook(Hook): self.logError(_("No %s loaded") % self.plugintype) return - # inject plugin plugin + #: inject plugin plugin self.logDebug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) for plugin in self.supported: @@ -223,7 +223,7 @@ class MultiHook(Hook): self.logDebug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) - # create new regexp + #: create new regexp regexp = r'.*(?P%s).*' % "|".join(x.replace('.', '\.') for x in plugins) if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and "://" in self.pluginclass.__pattern__: regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) @@ -252,7 +252,7 @@ class MultiHook(Hook): for plugin in self.supported: self.unloadPlugin(plugin) - # reset pattern + #: reset pattern hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 5fe6f2532..0191a4938 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -61,12 +61,12 @@ class OCR(Plugin): def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True, pagesegmode=None): - # tmpTif = tempfile.NamedTemporaryFile(suffix=".tif") + #: tmpTif = tempfile.NamedTemporaryFile(suffix=".tif") try: tmpTif = open(fs_join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") tmpTif.close() - # tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") + #: tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") tmpTxt = open(fs_join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") tmpTxt.close() @@ -88,7 +88,7 @@ class OCR(Plugin): tessparams.extend(["-psm", str(pagesegmode)]) if subset and (digits or lowercase or uppercase): - # tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") + #: tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") with open(fs_join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") as tmpSub: tmpSub.write("tessedit_char_whitelist ") @@ -154,7 +154,7 @@ class OCR(Plugin): for y in xrange(h): if pixels[x, y] == 255: continue - # No point in processing white pixels since we only want to remove black pixel + #: No point in processing white pixels since we only want to remove black pixel count = 0 try: @@ -177,12 +177,12 @@ class OCR(Plugin): except Exception: pass - # not enough neighbors are dark pixels so mark this pixel - # to be changed to white + #: not enough neighbors are dark pixels so mark this pixel + #: to be changed to white if count < allowed: pixels[x, y] = 1 - # second pass: this time set all 1's to 255 (white) + #: second pass: this time set all 1's to 255 (white) for x in xrange(w): for y in xrange(h): if pixels[x, y] == 1: diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 68169e077..1f86214c4 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -45,7 +45,7 @@ def parseHtmlForm(attr_str, html, input_names={}): inputs[name] = value if input_names: - # check input attributes + #: check input attributes for key, val in input_names.iteritems(): if key in inputs: if isinstance(val, basestring) and inputs[key] == val: @@ -60,7 +60,7 @@ def parseHtmlForm(attr_str, html, input_names={}): else: return action, inputs #: passed attribute check else: - # no attribute check + #: no attribute check return action, inputs return {}, None #: no matching form found @@ -281,7 +281,7 @@ class Plugin(object): self.logError(e) if just_header: - # parse header + #: parse header header = {"code": req.code} for line in res.splitlines(): line = line.strip() diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index cad4cfa1c..7a79c278c 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -53,7 +53,7 @@ class SevenZip(UnRar): def verify(self, password): - # 7z can't distinguish crc and pw error in test + #: 7z can't distinguish crc and pw error in test p = self.call_cmd("l", "-slt", fs_encode(self.filename)) out, err = p.communicate() @@ -72,7 +72,7 @@ class SevenZip(UnRar): p = self.call_cmd("l", "-slt", fs_encode(self.filename)) out, err = p.communicate() - # check if output or error macthes the 'wrong password'-Regexp + #: check if output or error macthes the 'wrong password'-Regexp if self.re_wrongpwd.search(out): raise PasswordError @@ -91,7 +91,7 @@ class SevenZip(UnRar): renice(p.pid, self.renice) - # communicate and retrieve stderr + #: communicate and retrieve stderr self._progress(p) err = p.stderr.read().strip() diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 85e939b6a..25a8a52de 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -105,7 +105,7 @@ class UnRar(Extractor): if self.re_wrongcrc.search(err): raise CRCError(err) - # output only used to check if passworded files are present + #: output only used to check if passworded files are present for attr in self.re_filelist.findall(out): if attr[0].startswith("*"): raise PasswordError @@ -114,7 +114,7 @@ class UnRar(Extractor): def repair(self): p = self.call_cmd("rc", fs_encode(self.filename)) - # communicate and retrieve stderr + #: communicate and retrieve stderr self._progress(p) err = p.stderr.read().strip() if err or p.returncode: @@ -126,17 +126,17 @@ class UnRar(Extractor): s = "" while True: c = process.stdout.read(1) - # quit loop on eof + #: quit loop on eof if not c: break - # reading a percentage sign -> set progress and restart + #: reading a percentage sign -> set progress and restart if c == '%': self.notifyProgress(int(s)) s = "" - # not reading a digit -> therefore restart + #: not reading a digit -> therefore restart elif c not in digits: s = "" - # add digit to progressstring + #: add digit to progressstring else: s += c @@ -148,7 +148,7 @@ class UnRar(Extractor): renice(p.pid, self.renice) - # communicate and retrieve stderr + #: communicate and retrieve stderr self._progress(p) err = p.stderr.read().strip() @@ -171,10 +171,10 @@ class UnRar(Extractor): def getDeleteFiles(self): dir, name = os.path.split(self.filename) - # actually extracted file + #: actually extracted file files = [self.filename] - # eventually Multipart Files + #: eventually Multipart Files files.extend(fs_join(dir, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir)) if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file)) @@ -195,7 +195,7 @@ class UnRar(Extractor): result = set() if not self.fullpath and self.VERSION.startswith('5'): - # NOTE: Unrar 5 always list full path + #@NOTE: Unrar 5 always list full path for f in fs_decode(out).splitlines(): f = fs_join(self.out, os.path.basename(f.strip())) if os.path.isfile(f): @@ -211,7 +211,7 @@ class UnRar(Extractor): def call_cmd(self, command, *xargs, **kwargs): args = [] - # overwrite flag + #: overwrite flag if self.overwrite: args.append("-o+") else: @@ -222,10 +222,10 @@ class UnRar(Extractor): for word in self.excludefiles: args.append("-x'%s'" % word.strip()) - # assume yes on all queries + #: assume yes on all queries args.append("-y") - # set a password + #: set a password if "password" in kwargs and kwargs['password']: args.append("-p%s" % kwargs['password']) else: @@ -234,7 +234,7 @@ class UnRar(Extractor): if self.keepbroken: args.append("-kb") - # NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue + #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) self.manager.logDebug(" ".join(call)) -- cgit v1.2.3 From 9305859b64a2f0aef3f27fb7e5b75caa0cb836a6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 17 Jun 2015 19:24:49 +0200 Subject: Spare code cosmetics (3) --- module/plugins/internal/SevenZip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index 7a79c278c..147ac6bd4 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -139,8 +139,8 @@ class SevenZip(UnRar): args.append("-y") #set a password - if "password" in kwargs and kwargs["password"]: - args.append("-p%s" % kwargs["password"]) + if "password" in kwargs and kwargs['password']: + args.append("-p%s" % kwargs['password']) else: args.append("-p-") -- cgit v1.2.3 From f32fcfcff81b67c8102df3822434f4c2551d7ae6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 5 Jul 2015 13:13:05 +0200 Subject: Fix long int conversion issue --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index e1a13af01..92cb476ea 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -182,7 +182,7 @@ class Hoster(Plugin): wait_until = time.time() + wait_time + 1 self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), - "Wait: %d(+1) seconds" % wait_time) + "Wait: %d+1 seconds" % wait_time) self.pyfile.waitUntil = wait_until @@ -641,7 +641,7 @@ class Hoster(Plugin): return True else: size = self.pyfile.size / 1024 - self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) + self.logInfo(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic)) return size <= traffic -- cgit v1.2.3 From fe70307e0b6916cc5a4273c73cdd355dee02546a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 5 Jul 2015 13:14:40 +0200 Subject: [Plugin] Fix decoding in load method --- module/plugins/internal/Plugin.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 1f86214c4..9b110280e 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -2,6 +2,7 @@ from __future__ import with_statement +import inspect import os import re import urllib @@ -260,14 +261,9 @@ class Plugin(object): res = req.load(url, get, post, ref, cookies, just_header, True, bool(decode)) if decode: - res = html_unescape(res) - - if isinstance(decode, basestring): - res = res.decode(decode) + res = html_unescape(res).decode(decode if isinstance(decode, basestring) else 'utf8') if self.core.debug: - import inspect - frame = inspect.currentframe() framefile = fs_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) try: -- cgit v1.2.3 From b1759bc440cd6013837697eb8de540914f693ffd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 7 Jul 2015 01:23:55 +0200 Subject: No camelCase style anymore --- module/plugins/internal/Account.py | 108 ++++++++++-------- module/plugins/internal/AdYouLike.py | 10 +- module/plugins/internal/AdsCaptcha.py | 10 +- module/plugins/internal/Captcha.py | 2 +- module/plugins/internal/Container.py | 2 +- module/plugins/internal/Crypter.py | 12 +- module/plugins/internal/DeadCrypter.py | 6 +- module/plugins/internal/DeadHoster.py | 6 +- module/plugins/internal/Extractor.py | 14 +-- module/plugins/internal/Hook.py | 60 +++++----- module/plugins/internal/Hoster.py | 138 +++++++++++------------ module/plugins/internal/MultiHook.py | 68 +++++------ module/plugins/internal/MultiHoster.py | 58 +++++----- module/plugins/internal/OCR.py | 4 +- module/plugins/internal/Plugin.py | 58 ++++++---- module/plugins/internal/ReCaptcha.py | 40 +++---- module/plugins/internal/SevenZip.py | 8 +- module/plugins/internal/SimpleCrypter.py | 42 +++---- module/plugins/internal/SimpleHoster.py | 188 +++++++++++++++---------------- module/plugins/internal/SolveMedia.py | 18 +-- module/plugins/internal/UnRar.py | 12 +- module/plugins/internal/UnZip.py | 4 +- module/plugins/internal/XFSAccount.py | 28 ++--- module/plugins/internal/XFSCrypter.py | 2 +- module/plugins/internal/XFSHoster.py | 70 ++++++------ 25 files changed, 498 insertions(+), 470 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index d2efe7996..9e0efaee6 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -16,7 +16,7 @@ class WrongPassword(Exception): class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.03" + __version__ = "0.04" __description__ = """Base account plugin""" __license__ = "GPLv3" @@ -37,7 +37,7 @@ class Account(Plugin): self.init() - self.setAccounts(accounts) + self.set_accounts(accounts) def init(self): @@ -60,18 +60,18 @@ class Account(Plugin): #: set timestamp for login self.timestamps[user] = time.time() - req = self.getAccountRequest(user) + req = self.get_account_request(user) try: self.login(user, data, req) except WrongPassword: - self.logWarning( + self.log_warning( _("Could not login with account %(user)s | %(msg)s") % {"user": user, "msg": _("Wrong Password")}) success = data['valid'] = False except Exception, e: - self.logWarning( + self.log_warning( _("Could not login with account %(user)s | %(msg)s") % {"user": user, "msg": e}) success = data['valid'] = False @@ -88,7 +88,7 @@ class Account(Plugin): def relogin(self, user): - req = self.getAccountRequest(user) + req = self.get_account_request(user) if req: req.cj.clear() req.close() @@ -98,14 +98,14 @@ class Account(Plugin): return self._login(user, self.accounts[user]) - def setAccounts(self, accounts): + def set_accounts(self, accounts): self.accounts = accounts for user, data in self.accounts.iteritems(): self._login(user, data) self.infos[user] = {} - def updateAccounts(self, user, password=None, options={}): + def update_accounts(self, user, password=None, options={}): """ Updates account and return true if anything changed """ @@ -125,7 +125,12 @@ class Account(Plugin): return True - def removeAccount(self, user): + #: Deprecated method, use `update_accounts` instead + def updateAccounts(self, *args, **kwargs): + return self.update_accounts(*args, **kwargs) + + + def remove_account(self, user): if user in self.accounts: del self.accounts[user] if user in self.infos: @@ -134,28 +139,33 @@ class Account(Plugin): del self.timestamps[user] + #: Deprecated method, use `remove_account` instead + def removeAccount(self, *args, **kwargs): + return self.remove_account(*args, **kwargs) + + @lock - def getAccountInfo(self, name, force=False): + def get_account_info(self, name, force=False): """ Retrieve account infos for an user, do **not** overwrite this method!\\ - just use it to retrieve infos in hoster plugins. see `loadAccountInfo` + just use it to retrieve infos in hoster plugins. see `load_account_info` :param name: username :param force: reloads cached account information :return: dictionary with information """ - data = Account.loadAccountInfo(self, name) + data = Account.load_account_info(self, name) if force or name not in self.infos: - self.logDebug("Get Account Info for %s" % name) - req = self.getAccountRequest(name) + self.log_debug("Get Account Info for %s" % name) + req = self.get_account_request(name) try: - infos = self.loadAccountInfo(name, req) + infos = self.load_account_info(name, req) if not type(infos) == dict: raise Exception("Wrong return format") except Exception, e: - infos = super(self.__class__, self).loadAccountInfo(name, req) + infos = super(self.__class__, self).load_account_info(name, req) infos['error'] = str(e) if self.core.debug: @@ -164,24 +174,24 @@ class Account(Plugin): if req: req.close() - self.logDebug("Account Info: %s" % infos) + self.log_debug("Account Info: %s" % infos) infos['timestamp'] = time.time() self.infos[name] = infos elif "timestamp" in self.infos[name] and self.infos[name]['timestamp'] + self.info_threshold * 60 < time.time(): - self.logDebug("Reached timeout for account data") - self.scheduleRefresh(name) + self.log_debug("Reached timeout for account data") + self.schedule_refresh(name) data.update(self.infos[name]) return data - def isPremium(self, user): - info = self.getAccountInfo(user) + def is_premium(self, user): + info = self.get_account_info(user) return info['premium'] - def loadAccountInfo(self, name, req=None): + def load_account_info(self, name, req=None): """ This should be overwritten in account plugin,\ and retrieving account information for user @@ -202,20 +212,25 @@ class Account(Plugin): "type" : self.__name__} - def getAllAccounts(self, force=False): - return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] + def get_all_accounts(self, force=False): + return [self.get_account_info(user, force) for user, data in self.accounts.iteritems()] - def getAccountRequest(self, user=None): + #: Deprecated method, use `get_all_accounts` instead + def getAllAccounts(self, *args, **kwargs): + return self.get_all_accounts(*args, **kwargs) + + + def get_account_request(self, user=None): if not user: - user, data = self.selectAccount() + user, data = self.select_account() return self.core.requestFactory.getRequest(self.__name__, user) - def getAccountCookies(self, user=None): + def get_account_cookies(self, user=None): if not user: - user, data = self.selectAccount() + user, data = self.select_account() if not user: return None @@ -223,11 +238,11 @@ class Account(Plugin): return cj - def getAccountData(self, user): + def get_account_data(self, user): return self.accounts[user] - def selectAccount(self): + def select_account(self): """ Returns an valid account name and data """ @@ -244,7 +259,7 @@ class Account(Plugin): if not compare_time(start.split(":"), end.split(":")): continue except Exception: - self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + self.log_warning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) if user in self.infos: if "validuntil" in self.infos[user]: @@ -262,52 +277,57 @@ class Account(Plugin): return random.choice(usable) - def canUse(self): - return self.selectAccount() != (None, None) + def can_use(self): + return self.select_account() != (None, None) - def parseTraffic(self, value, unit=None): #: return kilobytes + def parse_traffic(self, value, unit=None): #: return kilobytes if not unit and not isinstance(value, basestring): unit = "KB" return parseFileSize(value, unit) - def wrongPassword(self): + def wrong_password(self): raise WrongPassword def empty(self, user): if user in self.infos: - self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) + self.log_warning(_("Account %s has not enough traffic, checking again in 30min") % user) self.infos[user].update({"trafficleft": 0}) - self.scheduleRefresh(user, 30 * 60) + self.schedule_refresh(user, 30 * 60) def expired(self, user): if user in self.infos: - self.logWarning(_("Account %s is expired, checking again in 1h") % user) + self.log_warning(_("Account %s is expired, checking again in 1h") % user) self.infos[user].update({"validuntil": time.time() - 1}) - self.scheduleRefresh(user, 60 * 60) + self.schedule_refresh(user, 60 * 60) - def scheduleRefresh(self, user, time=0, force=True): + def schedule_refresh(self, user, time=0, force=True): """ Add task to refresh account info to sheduler """ - self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) - self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) + self.log_debug("Scheduled Account refresh for %s in %s seconds." % (user, time)) + self.core.scheduler.addJob(time, self.get_account_info, [user, force]) + + + #: Deprecated method, use `schedule_refresh` instead + def scheduleRefresh(self, *args, **kwargs): + return self.schedule_refresh(*args, **kwargs) @lock - def checkLogin(self, user): + def check_login(self, user): """ Checks if user is still logged in """ if user in self.timestamps: if self.login_timeout > 0 and self.timestamps[user] + self.login_timeout * 60 < time.time(): - self.logDebug("Reached login timeout for %s" % user) + self.log_debug("Reached login timeout for %s" % user) return self.relogin(user) else: return True diff --git a/module/plugins/internal/AdYouLike.py b/module/plugins/internal/AdYouLike.py index f623ed268..07e7e4d17 100644 --- a/module/plugins/internal/AdYouLike.py +++ b/module/plugins/internal/AdYouLike.py @@ -9,7 +9,7 @@ from module.plugins.internal.Captcha import Captcha class AdYouLike(Captcha): __name__ = "AdYouLike" __type__ = "captcha" - __version__ = "0.06" + __version__ = "0.07" __description__ = """AdYouLike captcha service plugin""" __license__ = "GPLv3" @@ -27,10 +27,10 @@ class AdYouLike(Captcha): n = re.search(self.CALLBACK_PATTERN, html) if m and n: self.key = (m.group(1).strip(), n.group(1).strip()) - self.logDebug("Ayl: %s | Callback: %s" % self.key) + self.log_debug("Ayl: %s | Callback: %s" % self.key) return self.key #: key is the tuple(ayl, callback) else: - self.logWarning("Ayl or callback pattern not found") + self.log_warning("Ayl or callback pattern not found") return None @@ -51,7 +51,7 @@ class AdYouLike(Captcha): except AttributeError: self.fail(_("AdYouLike challenge pattern not found")) - self.logDebug("Challenge: %s" % challenge) + self.log_debug("Challenge: %s" % challenge) return self.result(ayl, challenge), challenge @@ -86,6 +86,6 @@ class AdYouLike(Captcha): '_ayl_token_challenge': challenge['token'], '_ayl_response' : response} - self.logDebug("Result: %s" % result) + self.log_debug("Result: %s" % result) return result diff --git a/module/plugins/internal/AdsCaptcha.py b/module/plugins/internal/AdsCaptcha.py index e058352dd..b45a6dfda 100644 --- a/module/plugins/internal/AdsCaptcha.py +++ b/module/plugins/internal/AdsCaptcha.py @@ -9,7 +9,7 @@ from module.plugins.internal.Captcha import Captcha class AdsCaptcha(Captcha): __name__ = "AdsCaptcha" __type__ = "captcha" - __version__ = "0.09" + __version__ = "0.10" __description__ = """AdsCaptcha captcha service plugin""" __license__ = "GPLv3" @@ -27,10 +27,10 @@ class AdsCaptcha(Captcha): n = re.search(self.CAPTCHAID_PATTERN, html) if m and n: self.key = (m.group(1).strip(), n.group(1).strip()) #: key is the tuple(PublicKey, CaptchaId) - self.logDebug("Key: %s | ID: %s" % self.key) + self.log_debug("Key: %s | ID: %s" % self.key) return self.key else: - self.logWarning("Key or id pattern not found") + self.log_warning("Key or id pattern not found") return None @@ -47,7 +47,7 @@ class AdsCaptcha(Captcha): except AttributeError: self.fail(_("AdsCaptcha challenge pattern not found")) - self.logDebug("Challenge: %s" % challenge) + self.log_debug("Challenge: %s" % challenge) return self.result(server, challenge), challenge @@ -58,6 +58,6 @@ class AdsCaptcha(Captcha): cookies=True, imgtype="jpg") - self.logDebug("Result: %s" % result) + self.log_debug("Result: %s" % result) return result diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 4629c9522..50ebea89c 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -6,7 +6,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.30" + __version__ = "0.31" __description__ = """Base captcha service plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index fc1a93606..91b5a6f76 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -12,7 +12,7 @@ from module.utils import save_join as fs_join class Container(Crypter): __name__ = "Container" __type__ = "container" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index d8cda17d4..f6d3528fd 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -9,7 +9,7 @@ from module.utils import save_path as safe_filename class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config.get("general", "folder_per_package") @@ -69,11 +69,11 @@ class Crypter(Hoster): package_queue = self.pyfile.package().queue folder_per_package = self.core.config.get("general", "folder_per_package") - use_subfolder = self.getConfig('use_subfolder', folder_per_package) - subfolder_per_package = self.getConfig('subfolder_per_package', True) + use_subfolder = self.get_config('use_subfolder', folder_per_package) + subfolder_per_package = self.get_config('subfolder_per_package', True) for name, links, folder in self.packages: - self.logDebug("Parsed package: %s" % name, + self.log_debug("Parsed package: %s" % name, "%d links" % len(links), "Saved to folder: %s" % folder if folder else "Saved to download folder") @@ -88,14 +88,14 @@ class Crypter(Hoster): if use_subfolder: if not subfolder_per_package: setFolder(package_folder) - self.logDebug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) + self.log_debug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) elif not folder_per_package or name != folder: if not folder: folder = urlparse.urlparse(name).path.split("/")[-1] setFolder(safe_filename(folder)) - self.logDebug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) + self.log_debug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) elif folder_per_package: setFolder(None) diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index ef0d12b91..d79551b52 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -7,7 +7,7 @@ from module.plugins.internal.SimpleCrypter import create_getInfo class DeadCrypter(Crypter): __name__ = "DeadCrypter" __type__ = "crypter" - __version__ = "0.06" + __version__ = "0.07" __pattern__ = r'^unmatchable$' @@ -17,8 +17,8 @@ class DeadCrypter(Crypter): @classmethod - def apiInfo(cls, *args, **kwargs): - api = super(DeadCrypter, cls).apiInfo(*args, **kwargs) + def api_info(cls, *args, **kwargs): + api = super(DeadCrypter, cls).api_info(*args, **kwargs) api['status'] = 1 return api diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index accb15a78..86f4381a3 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -7,7 +7,7 @@ from module.plugins.internal.SimpleHoster import create_getInfo class DeadHoster(Hoster): __name__ = "DeadHoster" __type__ = "hoster" - __version__ = "0.16" + __version__ = "0.17" __pattern__ = r'^unmatchable$' @@ -17,8 +17,8 @@ class DeadHoster(Hoster): @classmethod - def apiInfo(cls, *args, **kwargs): - api = super(DeadHoster, cls).apiInfo(*args, **kwargs) + def api_info(cls, *args, **kwargs): + api = super(DeadHoster, cls).api_info(*args, **kwargs) api['status'] = 1 return api diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index f73388d8c..60d8b3b3b 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.24" + __version__ = "0.25" __description__ = """Base extractor plugin""" __license__ = "GPLv3" @@ -36,18 +36,18 @@ class Extractor(Plugin): @classmethod - def isArchive(cls, filename): + def is_archive(cls, filename): name = os.path.basename(filename).lower() return any(name.endswith(ext) for ext in cls.EXTENSIONS) @classmethod - def isMultipart(cls, filename): + def is_multipart(cls, filename): return False @classmethod - def isUsable(cls): + def is_usable(cls): """ Check if system statisfy dependencies :return: boolean @@ -56,7 +56,7 @@ class Extractor(Plugin): @classmethod - def getTargets(cls, files_ids): + def get_targets(cls, files_ids): """ Filter suited targets from list of filename id tuple list :param files_ids: List of filepathes @@ -97,7 +97,7 @@ class Extractor(Plugin): self.files = [] #: Store extracted files here pyfile = self.manager.core.files.getFile(fid) if fid else None - self.notifyProgress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None + self.notify_progress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None def init(self): @@ -149,7 +149,7 @@ class Extractor(Plugin): raise NotImplementedError - def getDeleteFiles(self): + def get_delete_files(self): """ Return list of files to delete, do *not* delete them here. diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 5959089b5..7ef8f0189 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -25,7 +25,7 @@ def threaded(fn): class Hook(Plugin): __name__ = "Hook" __type__ = "hook" - __version__ = "0.08" + __version__ = "0.09" __config__ = [] #: [("name", "type", "desc", "default")] __threaded__ = [] #@TODO: Remove in 0.4.10 @@ -72,7 +72,7 @@ class Hook(Plugin): self.event_map = None if self.event_list: - self.logDebug("Deprecated method `event_list`, use `event_map` instead") + self.log_debug("Deprecated method `event_list`, use `event_map` instead") for f in self.event_list: self.manager.addEvent(f, getattr(self, f)) @@ -85,8 +85,8 @@ class Hook(Plugin): #: Deprecated method, use `init_periodical` instead - def initPeriodical(self): - return self.init_periodical() + def initPeriodical(self, *args, **kwargs): + return self.init_periodical(*args, **kwargs) def _periodical(self, threaded): @@ -98,7 +98,7 @@ class Hook(Plugin): self.periodical() except Exception, e: - self.logError(_("Error executing hook: %s") % e) + self.log_error(_("Error executing hook: %s") % e) if self.core.debug: traceback.print_exc() @@ -124,12 +124,12 @@ class Hook(Plugin): """ Checks if hook is activated """ - return self.getConfig("activated") + return self.get_config("activated") #: Deprecated method, use `is_activated` instead - def isActivated(self): - return self.is_activated() + def isActivated(self, *args, **kwargs): + return self.is_activated(*args, **kwargs) def deactivate(self): @@ -140,8 +140,8 @@ class Hook(Plugin): #: Deprecated method, use `deactivate` instead - def unload(self): - return self.deactivate() + def unload(self, *args, **kwargs): + return self.deactivate(*args, **kwargs) def activate(self): @@ -152,8 +152,8 @@ class Hook(Plugin): #: Deprecated method, use `activate` instead - def coreReady(self): - return self.activate() + def coreReady(self, *args, **kwargs): + return self.activate(*args, **kwargs) def exit(self): @@ -164,8 +164,8 @@ class Hook(Plugin): #: Deprecated method, use `exit` instead - def coreExiting(self): - return self.exit() + def coreExiting(self, *args, **kwargs): + return self.exit(*args, **kwargs) def download_preparing(self, pyfile): @@ -173,8 +173,8 @@ class Hook(Plugin): #: Deprecated method, use `download_preparing` instead - def downloadPreparing(self, pyfile): - return self.download_preparing(pyfile) + def downloadPreparing(self, *args, **kwargs): + return self.download_preparing(*args, **kwargs) def download_finished(self, pyfile): @@ -182,8 +182,8 @@ class Hook(Plugin): #: Deprecated method, use `download_finished` instead - def downloadFinished(self, pyfile): - return self.download_finished(pyfile) + def downloadFinished(self, *args, **kwargs): + return self.download_finished(*args, **kwargs) def download_failed(self, pyfile): @@ -191,8 +191,8 @@ class Hook(Plugin): #: Deprecated method, use `download_failed` instead - def downloadFailed(self, pyfile): - return self.download_failed(pyfile) + def downloadFailed(self, *args, **kwargs): + return self.download_failed(*args, **kwargs) def package_finished(self, pypack): @@ -200,8 +200,8 @@ class Hook(Plugin): #: Deprecated method, use `package_finished` instead - def packageFinished(self, pypack): - return self.package_finished(pypack) + def packageFinished(self, *args, **kwargs): + return self.package_finished(*args, **kwargs) def before_reconnect(self, ip): @@ -209,8 +209,8 @@ class Hook(Plugin): #: Deprecated method, use `before_reconnect` instead - def beforeReconnecting(self, ip): - return self.before_reconnect(ip) + def beforeReconnecting(self, *args, **kwargs): + return self.before_reconnect(*args, **kwargs) def after_reconnect(self, ip, oldip): @@ -230,8 +230,8 @@ class Hook(Plugin): #: Deprecated method, use `captcha_task` instead - def newCaptchaTask(self, task): - return self.captcha_task(task) + def newCaptchaTask(self, *args, **kwargs): + return self.captcha_task(*args, **kwargs) def captcha_correct(self, task): @@ -239,8 +239,8 @@ class Hook(Plugin): #: Deprecated method, use `captcha_correct` instead - def captchaCorrect(self, task): - return self.captcha_correct(task) + def captchaCorrect(self, *args, **kwargs): + return self.captcha_correct(*args, **kwargs) def captcha_invalid(self, task): @@ -248,5 +248,5 @@ class Hook(Plugin): #: Deprecated method, use `captcha_invalid` instead - def captchaInvalid(self, task): - return self.captcha_invalid(task) + def captchaInvalid(self, *args, **kwargs): + return self.captcha_invalid(*args, **kwargs) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 92cb476ea..09672f6f0 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -16,15 +16,15 @@ from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry from module.utils import fs_decode, fs_encode, save_join as fs_join -def getInfo(urls): - #result = [ .. (name, size, status, url) .. ] +def get_info(urls): + # result = [ .. (name, size, status, url) .. ] pass class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.03" + __version__ = "0.04" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] @@ -41,18 +41,18 @@ class Hoster(Plugin): super(Hoster, self).__init__(pyfile.m.core) #: engage wan reconnection - self.wantReconnect = False + self.want_reconnect = False #: enable simultaneous processing of multiple downloads - self.multiDL = True - self.limitDL = 0 + self.multi_dl = True + self.limit_dl = 0 #: chunk limit - self.chunkLimit = 1 - self.resumeDownload = False + self.chunk_limit = 1 + self.resume_download = False #: time.time() + wait in seconds - self.waitUntil = 0 + self.wait_until = 0 self.waiting = False #: captcha reader instance @@ -75,10 +75,10 @@ class Hoster(Plugin): #: Browser instance, see `network.Browser` self.req = self.account.getAccountRequest(self.user) - self.chunkLimit = -1 #: chunk limit, -1 for unlimited + self.chunk_limit = -1 #: chunk limit, -1 for unlimited #: enables resume (will be ignored if server dont accept chunks) - self.resumeDownload = True + self.resume_download = True #: premium status self.premium = self.account.isPremium(self.user) @@ -91,16 +91,16 @@ class Hoster(Plugin): self.thread = None #: holds thread in future #: location where the last call to download was saved - self.lastDownload = "" + self.last_download = "" #: re match of the last call to `checkDownload` - self.lastCheck = None + self.last_check = None #: js engine, see `JsEngine` self.js = self.core.js #: captcha task - self.cTask = None + self.c_task = None #: some plugins store html code here self.html = None @@ -150,13 +150,13 @@ class Hoster(Plugin): raise NotImplementedError - def getChunkCount(self): - if self.chunkLimit <= 0: + def get_chunk_count(self): + if self.chunk_limit <= 0: return self.core.config.get("download", "chunks") - return min(self.core.config.get("download", "chunks"), self.chunkLimit) + return min(self.core.config.get("download", "chunks"), self.chunk_limit) - def resetAccount(self): + def reset_account(self): """ Don't use account and retry download """ @@ -165,13 +165,13 @@ class Hoster(Plugin): self.retry() - def setReconnect(self, reconnect): + def set_reconnect(self, reconnect): reconnect = bool(reconnect) - self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) - self.wantReconnect = reconnect + self.log_debug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.want_reconnect)) + self.want_reconnect = reconnect - def setWait(self, seconds, reconnect=None): + def set_wait(self, seconds, reconnect=None): """ Set a specific wait time later used with `wait` @@ -181,13 +181,13 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), + self.log_debug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), "Wait: %d+1 seconds" % wait_time) self.pyfile.waitUntil = wait_until if reconnect is not None: - self.setReconnect(reconnect) + self.set_reconnect(reconnect) def wait(self, seconds=0, reconnect=None): @@ -197,21 +197,21 @@ class Hoster(Plugin): pyfile = self.pyfile if seconds > 0: - self.setWait(seconds) + self.set_wait(seconds) if reconnect is not None: - self.setReconnect(reconnect) + self.set_reconnect(reconnect) self.waiting = True status = pyfile.status pyfile.setStatus("waiting") - self.logInfo(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), - _("Reconnect: %s") % self.wantReconnect) + self.log_info(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), + _("Reconnect: %s") % self.want_reconnect) if self.account: - self.logDebug("Ignore reconnection due account logged") + self.log_debug("Ignore reconnection due account logged") while pyfile.waitUntil > time.time(): if pyfile.abort: @@ -227,7 +227,7 @@ class Hoster(Plugin): if self.thread.m.reconnecting.isSet(): self.waiting = False - self.wantReconnect = False + self.want_reconnect = False raise Reconnect time.sleep(1) @@ -262,7 +262,7 @@ class Hoster(Plugin): raise Fail("offline") - def tempOffline(self, reason=""): + def temp_offline(self, reason=""): """ Fail and indicates file ist temporary offline, the core may take consequences """ @@ -292,19 +292,19 @@ class Hoster(Plugin): raise Retry(reason) - def invalidCaptcha(self): - self.logError(_("Invalid captcha")) - if self.cTask: - self.cTask.invalid() + def invalid_captcha(self): + self.log_error(_("Invalid captcha")) + if self.c_task: + self.c_task.invalid() - def correctCaptcha(self): - self.logInfo(_("Correct captcha")) - if self.cTask: - self.cTask.correct() + def correct_captcha(self): + self.log_info(_("Correct captcha")) + if self.c_task: + self.c_task.correct() - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, + def decrypt_captcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', result_type='textual'): """ Loads a captcha and decrypts it with ocr, plugin, user input @@ -345,7 +345,7 @@ class Hoster(Plugin): else: captchaManager = self.core.captchaManager task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) - self.cTask = task + self.c_task = task captchaManager.handleCaptcha(task) while task.isWaiting(): @@ -364,7 +364,7 @@ class Hoster(Plugin): self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) result = task.result - self.logDebug("Received captcha result: %s" % result) + self.log_debug("Received captcha result: %s" % result) if not self.core.debug: try: @@ -409,10 +409,10 @@ class Hoster(Plugin): self.fail(_("No url given")) if self.core.debug: - self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + self.log_debug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) - self.correctCaptcha() - self.checkForSameFiles() + self.correct_captcha() + self.check_for_same_files() self.pyfile.setStatus("downloading") @@ -445,7 +445,7 @@ class Hoster(Plugin): try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, - chunks=self.getChunkCount(), resume=self.resumeDownload, + chunks=self.get_chunk_count(), resume=self.resume_download, progressNotify=self.pyfile.setProgress, disposition=disposition) finally: self.pyfile.size = self.req.size @@ -454,7 +454,7 @@ class Hoster(Plugin): newname = urlparse.urlparse(newname).path.split('/')[-1] if disposition and newname != name: - self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) + self.log_info(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) self.pyfile.name = newname filename = os.path.join(location, newname) @@ -464,7 +464,7 @@ class Hoster(Plugin): try: os.chmod(fs_filename, int(self.core.config.get("permission", "file"), 8)) except Exception, e: - self.logWarning(_("Setting file mode failed"), e) + self.log_warning(_("Setting file mode failed"), e) if self.core.config.get("permission", "change_dl") and os.name != "nt": try: @@ -473,13 +473,13 @@ class Hoster(Plugin): os.chown(fs_filename, uid, gid) except Exception, e: - self.logWarning(_("Setting User and Group failed"), e) + self.log_warning(_("Setting User and Group failed"), e) - self.lastDownload = filename - return self.lastDownload + self.last_download = filename + return self.last_download - def checkDownload(self, rules, delete=True, file_size=0, size_tolerance=1000, read_size=100000): + def check_download(self, rules, delete=True, file_size=0, size_tolerance=1000, read_size=100000): """ Checks the content of the last downloaded file, re match is saved to `lastCheck` @@ -491,10 +491,10 @@ class Hoster(Plugin): :return: dictionary key of the first rule that matched """ do_delete = False - lastDownload = fs_encode(self.lastDownload) + lastDownload = fs_encode(self.last_download) - if not self.lastDownload or not os.path.exists(lastDownload): - self.lastDownload = "" + if not self.last_download or not os.path.exists(lastDownload): + self.last_download = "" self.fail(self.pyfile.error or _("No file downloaded")) try: @@ -512,15 +512,15 @@ class Hoster(Plugin): self.fail(_("File size mismatch")) elif diff != 0: - self.logWarning(_("File size is not equal to expected size")) + self.log_warning(_("File size is not equal to expected size")) - self.logDebug("Download Check triggered") + self.log_debug("Download Check triggered") with open(lastDownload, "rb") as f: content = f.read(read_size) #: produces encoding errors, better log to other file in the future? - #: self.logDebug("Content: %s" % content) + #: self.log_debug("Content: %s" % content) for name, rule in rules.iteritems(): if isinstance(rule, basestring): if rule in content: @@ -531,14 +531,14 @@ class Hoster(Plugin): m = rule.search(content) if m: do_delete = True - self.lastCheck = m + self.last_check = m return name finally: if delete and do_delete: os.remove(lastDownload) - def directLink(self, url, follow_location=None): + def direct_link(self, url, follow_location=None): link = "" if follow_location is None: @@ -548,11 +548,11 @@ class Hoster(Plugin): redirect = max(follow_location, 1) else: - redirect = self.getConfig("maxredirs", plugin="UserAgentSwitcher") + redirect = self.get_config("maxredirs", plugin="UserAgentSwitcher") for i in xrange(redirect): try: - self.logDebug("Redirect #%d to: %s" % (i, url)) + self.log_debug("Redirect #%d to: %s" % (i, url)) header = self.load(url, just_header=True) except Exception: #: Bad bad bad... rewrite this part in 0.4.10 @@ -618,18 +618,18 @@ class Hoster(Plugin): else: try: - self.logError(_("Too many redirects")) + self.log_error(_("Too many redirects")) except Exception: pass return link - def parseHtmlForm(self, attr_str="", input_names={}): + def parse_html_form(self, attr_str="", input_names={}): return parseHtmlForm(attr_str, self.html, input_names) - def checkTrafficLeft(self): + def check_traffic_left(self): if not self.account: return True @@ -641,18 +641,18 @@ class Hoster(Plugin): return True else: size = self.pyfile.size / 1024 - self.logInfo(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic)) + self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic)) return size <= traffic - def getPassword(self): + def get_password(self): """ Get the password the user provided in the package """ return self.pyfile.package().password or "" - def checkForSameFiles(self, starting=False): + def check_for_same_files(self, starting=False): """ Checks if same file was/is downloaded within same package @@ -681,7 +681,7 @@ class Hoster(Plugin): if os.path.exists(location): self.skip(pyfile[0]) - self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name) + self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name) def clean(self): diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 22b0fd7c6..623836cdc 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -11,7 +11,7 @@ from module.utils import decode, remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.49" + __version__ = "0.50" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), @@ -65,10 +65,10 @@ class MultiHook(Hook): self.pluginname = None self.plugintype = None - self.initPlugin() + self.init_plugin() - def initPlugin(self): + def init_plugin(self): self.pluginname = self.__name__.rsplit("Hook", 1)[0] plugin, self.plugintype = self.core.pluginManager.findPlugin(self.pluginname) @@ -76,47 +76,47 @@ class MultiHook(Hook): self.pluginmodule = self.core.pluginManager.loadModule(self.plugintype, self.pluginname) self.pluginclass = getattr(self.pluginmodule, self.pluginname) else: - self.logWarning("Hook plugin will be deactivated due missing plugin reference") - self.setConfig('activated', False) + self.log_warning("Hook plugin will be deactivated due missing plugin reference") + self.set_config('activated', False) - def loadAccount(self): + def load_account(self): self.account = self.core.accountManager.getAccountPlugin(self.pluginname) if self.account and not self.account.canUse(): self.account = None if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: - self.logWarning("Hook plugin will be deactivated due missing account reference") - self.setConfig('activated', False) + self.log_warning("Hook plugin will be deactivated due missing account reference") + self.set_config('activated', False) def activate(self): self.init_periodical(threaded=True) - def pluginsCached(self): + def plugins_cached(self): if self.plugins: return self.plugins for _i in xrange(2): try: - pluginset = self._pluginSet(self.getHosters()) + pluginset = self._plugin_set(self.get_hosters()) break except Exception, e: - self.logDebug(e, "Waiting 1 minute and retry") + self.log_debug(e, "Waiting 1 minute and retry") time.sleep(60) else: - self.logWarning(_("Fallback to default reload interval due plugin parse error")) + self.log_warning(_("Fallback to default reload interval due plugin parse error")) self.interval = self.MIN_RELOAD_INTERVAL return list() try: - configmode = self.getConfig('pluginmode', 'all') + configmode = self.get_config('pluginmode', 'all') if configmode in ("listed", "unlisted"): - pluginlist = self.getConfig('pluginlist', '').replace('|', ',').replace(';', ',').split(',') - configset = self._pluginSet(pluginlist) + pluginlist = self.get_config('pluginlist', '').replace('|', ',').replace(';', ',').split(',') + configset = self._plugin_set(pluginlist) if configmode == "listed": pluginset &= configset @@ -124,14 +124,14 @@ class MultiHook(Hook): pluginset -= configset except Exception, e: - self.logError(e) + self.log_error(e) self.plugins = list(pluginset) return self.plugins - def _pluginSet(self, plugins): + def _plugin_set(self, plugins): regexp = re.compile(r'^[\w\-.^_]{3,63}\.[a-zA-Z]{2,}$', re.U) plugins = [decode(p.strip()).lower() for p in plugins if regexp.match(p.strip())] @@ -143,7 +143,7 @@ class MultiHook(Hook): return set(plugins) - def getHosters(self): + def get_hosters(self): """ Load list of supported hoster @@ -156,15 +156,15 @@ class MultiHook(Hook): """ Reload plugin list periodically """ - self.loadAccount() + self.load_account() - if self.getConfig('reload', True): - self.interval = max(self.getConfig('reloadinterval', 12) * 60 * 60, self.MIN_RELOAD_INTERVAL) + if self.get_config('reload', True): + self.interval = max(self.get_config('reloadinterval', 12) * 60 * 60, self.MIN_RELOAD_INTERVAL) else: self.core.scheduler.removeJob(self.cb) self.cb = None - self.logInfo(_("Reloading supported %s list") % self.plugintype) + self.log_info(_("Reloading supported %s list") % self.plugintype) old_supported = self.supported @@ -172,17 +172,17 @@ class MultiHook(Hook): self.new_supported = [] self.plugins = [] - self.overridePlugins() + self.override_plugins() old_supported = [plugin for plugin in old_supported if plugin not in self.supported] if old_supported: - self.logDebug("Unload: %s" % ", ".join(old_supported)) + self.log_debug("Unload: %s" % ", ".join(old_supported)) for plugin in old_supported: - self.unloadPlugin(plugin) + self.unload_plugin(plugin) - def overridePlugins(self): + def override_plugins(self): excludedList = [] if self.plugintype == "hoster": @@ -192,7 +192,7 @@ class MultiHook(Hook): pluginMap = {} accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.core.pluginManager.crypterPlugins.iterkeys()] - for plugin in self.pluginsCached(): + for plugin in self.plugins_cached(): name = remove_chars(plugin, "-.") if name in accountList: @@ -204,11 +204,11 @@ class MultiHook(Hook): self.new_supported.append(plugin) if not self.supported and not self.new_supported: - self.logError(_("No %s loaded") % self.plugintype) + self.log_error(_("No %s loaded") % self.plugintype) return #: inject plugin plugin - self.logDebug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) + self.log_debug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) for plugin in self.supported: hdict = self.core.pluginManager.plugins[self.plugintype][plugin] @@ -216,26 +216,26 @@ class MultiHook(Hook): hdict['new_name'] = self.pluginname if excludedList: - self.logInfo(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList)))) + self.log_info(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList)))) if self.new_supported: plugins = sorted(self.new_supported) - self.logDebug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) + self.log_debug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) #: create new regexp regexp = r'.*(?P%s).*' % "|".join(x.replace('.', '\.') for x in plugins) if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and "://" in self.pluginclass.__pattern__: regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) - self.logDebug("Regexp: %s" % regexp) + self.log_debug("Regexp: %s" % regexp) hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] hdict['pattern'] = regexp hdict['re'] = re.compile(regexp) - def unloadPlugin(self, plugin): + def unload_plugin(self, plugin): hdict = self.core.pluginManager.plugins[self.plugintype][plugin] if "module" in hdict: hdict.pop('module', None) @@ -250,7 +250,7 @@ class MultiHook(Hook): Remove override for all plugins. Scheduler job is removed by hookmanager """ for plugin in self.supported: - self.unloadPlugin(plugin) + self.unload_plugin(plugin) #: reset pattern hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 2b3eb8941..8360d871e 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.42" + __version__ = "0.43" __pattern__ = r'^unmatchable$' __config__ = [("use_premium" , "bool", "Use premium account if available" , True), @@ -24,19 +24,19 @@ class MultiHoster(SimpleHoster): def setup(self): - self.chunkLimit = 1 - self.multiDL = bool(self.account) - self.resumeDownload = self.premium + self.chunk_limit = 1 + self.multi_dl = bool(self.account) + self.resume_download = self.premium def prepare(self): self.info = {} self.html = "" self.link = "" #@TODO: Move to Hoster in 0.4.10 - self.directDL = False #@TODO: Move to Hoster in 0.4.10 + self.direct_d_l = False #@TODO: Move to Hoster in 0.4.10 - if not self.getConfig('use_premium', True): - self.retryFree() + if not self.get_config('use_premium', True): + self.retry_free() if self.LOGIN_ACCOUNT and not self.account: self.fail(_("Required account not found")) @@ -47,9 +47,9 @@ class MultiHoster(SimpleHoster): set_cookies(self.req.cj, self.COOKIES) if self.DIRECT_LINK is None: - self.directDL = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) + self.direct_d_l = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) else: - self.directDL = self.DIRECT_LINK + self.direct_d_l = self.DIRECT_LINK self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) @@ -58,36 +58,36 @@ class MultiHoster(SimpleHoster): try: self.prepare() - if self.directDL: - self.checkInfo() - self.logDebug("Looking for direct download link...") - self.handleDirect(pyfile) + if self.direct_d_l: + self.check_info() + self.log_debug("Looking for direct download link...") + self.handle_direct(pyfile) - if not self.link and not self.lastDownload: + if not self.link and not self.last_download: self.preload() - self.checkErrors() - self.checkStatus(getinfo=False) + self.check_errors() + self.check_status(getinfo=False) - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as premium download") - self.handlePremium(pyfile) + if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): + self.log_debug("Handled as premium download") + self.handle_premium(pyfile) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as free download") - self.handleFree(pyfile) + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): + self.log_debug("Handled as free download") + self.handle_free(pyfile) self.download(self.link, ref=False, disposition=True) - self.checkFile() + self.check_file() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 err = str(e) #@TODO: Recheck in 0.4.10 if self.premium: - self.logWarning(_("Premium download failed")) - self.retryFree() + self.log_warning(_("Premium download failed")) + self.retry_free() - elif self.getConfig("revertfailed", True) \ + elif self.get_config("revertfailed", True) \ and "new_module" in self.core.pluginManager.hosterPlugins[self.__name__]: hdict = self.core.pluginManager.hosterPlugins[self.__name__] @@ -107,11 +107,11 @@ class MultiHoster(SimpleHoster): raise Fail(err) - def handlePremium(self, pyfile): - return self.handleFree(pyfile) + def handle_premium(self, pyfile): + return self.handle_free(pyfile) - def handleFree(self, pyfile): + def handle_free(self, pyfile): if self.premium: raise NotImplementedError else: diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 0191a4938..880f8b570 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -20,7 +20,7 @@ from module.utils import save_join as fs_join class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" - __version__ = "0.11" + __version__ = "0.12" __description__ = """OCR base plugin""" __license__ = "GPLv3" @@ -71,7 +71,7 @@ class OCR(Plugin): tmpTxt.close() except IOError, e: - self.logError(e) + self.log_error(e) return self.logger.debug("save tiff") diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 9b110280e..0b08858f3 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -25,12 +25,12 @@ def set_cookies(cj, cookies): cj.setCookie(domain, name, value) -def parseHtmlTagAttrValue(attr_name, tag): +def parse_html_tag_attr_value(attr_name, tag): m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^>\s\"'][^>\s]*)\1" % attr_name, tag, re.I) return m.group(2) if m else None -def parseHtmlForm(attr_str, html, input_names={}): +def parse_html_form(attr_str, html, input_names={}): for form in re.finditer(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, html, re.S | re.I): inputs = {} @@ -78,7 +78,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.11" + __version__ = "0.12" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] @@ -106,28 +106,28 @@ class Plugin(object): 'msg' : msg or _(level.upper() + " MARK")}) - def logDebug(self, *args): + def log_debug(self, *args): if self.core.debug: return self._log("debug", args) - def logInfo(self, *args): + def log_info(self, *args): return self._log("info", args) - def logWarning(self, *args): + def log_warning(self, *args): return self._log("warning", args) - def logError(self, *args): + def log_error(self, *args): return self._log("error", args) - def logCritical(self, *args): + def log_critical(self, *args): return self._log("critical", args) - def setConfig(self, option, value): + def set_config(self, option, value): """ Set config value for current plugin @@ -138,15 +138,15 @@ class Plugin(object): self.core.config.setPlugin(self.__name__, option, value) - #: Deprecated method + #: Deprecated method, use `set_config` instead def setConf(self, *args, **kwargs): """ - See `setConfig` + See `set_config` """ - return self.setConfig(*args, **kwargs) + return self.set_config(*args, **kwargs) - def getConfig(self, option, default="", plugin=None): + def get_config(self, option, default="", plugin=None): """ Returns config value for current plugin @@ -157,16 +157,16 @@ class Plugin(object): return self.core.config.getPlugin(plugin or self.__name__, option) except KeyError: - self.logWarning(_("Config option or plugin not found")) + self.log_warning(_("Config option or plugin not found")) return default - #: Deprecated method + #: Deprecated method, use `get_config` instead def getConf(self, *args, **kwargs): """ - See `getConfig` + See `get_config` """ - return self.getConfig(*args, **kwargs) + return self.get_config(*args, **kwargs) def store(self, key, value): @@ -176,10 +176,10 @@ class Plugin(object): self.core.db.setStorage(self.__name__, key, value) - #: Deprecated method + #: Deprecated method, use `store` instead def setStorage(self, *args, **kwargs): """ - Same as `setStorage` + Same as `store` """ return self.store(*args, **kwargs) @@ -191,21 +191,29 @@ class Plugin(object): return self.core.db.getStorage(self.__name__, key) or default - #: Deprecated method + #: Deprecated method, use `retrieve` instead def getStorage(self, *args, **kwargs): """ - Same as `getStorage` + Same as `retrieve` """ return self.retrieve(*args, **kwargs) - def delStorage(self, key): + def delete(self, key): """ Delete entry in db """ self.core.db.delStorage(self.__name__, key) + #: Deprecated method, use `delete` instead + def delStorage(self, *args, **kwargs): + """ + Same as `delete` + """ + return self.delete(*args, **kwargs) + + def fail(self, reason): """ Fail and give reason @@ -250,7 +258,7 @@ class Plugin(object): self.fail(_("No url given")) if self.core.debug: - self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + self.log_debug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) if req is None: if hasattr(self, "req"): @@ -274,7 +282,7 @@ class Plugin(object): del frame #: delete the frame or it wont be cleaned f.write(res.encode('utf8')) except IOError, e: - self.logError(e) + self.log_error(e) if just_header: #: parse header @@ -297,4 +305,4 @@ class Plugin(object): header[key] = value res = header - return res \ No newline at end of file + return res diff --git a/module/plugins/internal/ReCaptcha.py b/module/plugins/internal/ReCaptcha.py index 40faff5f0..79bda9051 100644 --- a/module/plugins/internal/ReCaptcha.py +++ b/module/plugins/internal/ReCaptcha.py @@ -13,7 +13,7 @@ from module.plugins.internal.Captcha import Captcha class ReCaptcha(Captcha): __name__ = "ReCaptcha" __type__ = "captcha" - __version__ = "0.17" + __version__ = "0.18" __description__ = """ReCaptcha captcha service plugin""" __license__ = "GPLv3" @@ -32,10 +32,10 @@ class ReCaptcha(Captcha): m = re.search(self.KEY_V2_PATTERN, html) or re.search(self.KEY_V1_PATTERN, html) if m: self.key = m.group(1).strip() - self.logDebug("Key: %s" % self.key) + self.log_debug("Key: %s" % self.key) return self.key else: - self.logWarning("Key pattern not found") + self.log_warning("Key pattern not found") return None @@ -60,7 +60,7 @@ class ReCaptcha(Captcha): except AttributeError: self.fail(_("ReCaptcha challenge pattern not found")) - self.logDebug("Challenge: %s" % challenge) + self.log_debug("Challenge: %s" % challenge) return self.result(server, challenge, key) @@ -79,50 +79,50 @@ class ReCaptcha(Captcha): except AttributeError: self.fail(_("ReCaptcha second challenge pattern not found")) - self.logDebug("Second challenge: %s" % challenge) + self.log_debug("Second challenge: %s" % challenge) result = self.plugin.decryptCaptcha("%simage" % server, get={'c': challenge}, cookies=True, forceUser=True, imgtype="jpg") - self.logDebug("Result: %s" % result) + self.log_debug("Result: %s" % result) return result, challenge - def _collectApiInfo(self): + def _collect_api_info(self): html = self.plugin.load("http://www.google.com/recaptcha/api.js") a = re.search(r'po.src = \'(.*?)\';', html).group(1) vers = a.split("/")[5] - self.logDebug("API version: %s" % vers) + self.log_debug("API version: %s" % vers) language = a.split("__")[1].split(".")[0] - self.logDebug("API language: %s" % language) + self.log_debug("API language: %s" % language) html = self.plugin.load("https://apis.google.com/js/api.js") b = re.search(r'"h":"(.*?)","', html).group(1) jsh = b.decode('unicode-escape') - self.logDebug("API jsh-string: %s" % jsh) + self.log_debug("API jsh-string: %s" % jsh) return vers, language, jsh - def _prepareTimeAndRpc(self): + def _prepare_time_and_rpc(self): self.plugin.load("http://www.google.com/recaptcha/api2/demo") millis = int(round(time.time() * 1000)) - self.logDebug("Time: %s" % millis) + self.log_debug("Time: %s" % millis) rand = random.randint(1, 99999999) a = "0.%s" % str(rand * 2147483647) rpc = int(100000000 * float(a)) - self.logDebug("Rpc-token: %s" % rpc) + self.log_debug("Rpc-token: %s" % rpc) return millis, rpc @@ -136,8 +136,8 @@ class ReCaptcha(Captcha): parent = "" botguardstring = "!A" - vers, language, jsh = self._collectApiInfo() - millis, rpc = self._prepareTimeAndRpc() + vers, language, jsh = self._collect_api_info() + millis, rpc = self._prepare_time_and_rpc() html = self.plugin.load("https://www.google.com/recaptcha/api2/anchor", get={'k' : key, @@ -150,7 +150,7 @@ class ReCaptcha(Captcha): 'rpctoken': rpc}) token1 = re.search(r'id="recaptcha-token" value="(.*?)">', html) - self.logDebug("Token #1: %s" % token1.group(1)) + self.log_debug("Token #1: %s" % token1.group(1)) html = self.plugin.load("https://www.google.com/recaptcha/api2/frame", get={'c' : token1.group(1), @@ -163,10 +163,10 @@ class ReCaptcha(Captcha): decode="unicode-escape") token2 = re.search(r'"finput","(.*?)",', html) - self.logDebug("Token #2: %s" % token2.group(1)) + self.log_debug("Token #2: %s" % token2.group(1)) token3 = re.search(r'"rresp","(.*?)",', html) - self.logDebug("Token #3: %s" % token3.group(1)) + self.log_debug("Token #3: %s" % token3.group(1)) millis_captcha_loading = int(round(time.time() * 1000)) captcha_response = self.plugin.decryptCaptcha("https://www.google.com/recaptcha/api2/payload", @@ -175,7 +175,7 @@ class ReCaptcha(Captcha): forceUser=True) response = b64encode('{"response":"%s"}' % captcha_response) - self.logDebug("Result: %s" % response) + self.log_debug("Result: %s" % response) timeToSolve = int(round(time.time() * 1000)) - millis_captcha_loading timeToSolveMore = timeToSolve + int(float("0." + str(random.randint(1, 99999999))) * 500) @@ -189,7 +189,7 @@ class ReCaptcha(Captcha): 'bg' : botguardstring}) token4 = re.search(r'"uvresp","(.*?)",', html) - self.logDebug("Token #4: %s" % token4.group(1)) + self.log_debug("Token #4: %s" % token4.group(1)) result = token4.group(1) diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index 147ac6bd4..86ff5156e 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -10,7 +10,7 @@ from module.utils import fs_encode, save_join as fs_join class SevenZip(UnRar): __name__ = "SevenZip" - __version__ = "0.11" + __version__ = "0.12" __description__ = """7-Zip extractor plugin""" __license__ = "GPLv3" @@ -37,7 +37,7 @@ class SevenZip(UnRar): @classmethod - def isUsable(cls): + def is_usable(cls): if os.name == "nt": cls.CMD = os.path.join(pypath, "7z.exe") p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -134,11 +134,11 @@ class SevenZip(UnRar): def call_cmd(self, command, *xargs, **kwargs): args = [] - #overwrite flag + # overwrite flag if self.overwrite: args.append("-y") - #set a password + # set a password if "password" in kwargs and kwargs['password']: args.append("-p%s" % kwargs['password']) else: diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 7685be55a..fe47178e3 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.55" + __version__ = "0.56" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] @@ -47,7 +47,7 @@ class SimpleCrypter(Crypter, SimpleHoster): and its loadPage method: - def loadPage(self, page_n): + def load_page(self, page_n): return the html of the page number page_n """ #@TODO: Remove in 0.4.10 @@ -85,10 +85,10 @@ class SimpleCrypter(Crypter, SimpleHoster): self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - def handleDirect(self, pyfile): - for i in xrange(self.getConfig("maxredirs", plugin="UserAgentSwitcher")): + def handle_direct(self, pyfile): + for i in xrange(self.get_config("maxredirs", plugin="UserAgentSwitcher")): redirect = self.link or pyfile.url - self.logDebug("Redirect #%d to: %s" % (i, redirect)) + self.log_debug("Redirect #%d to: %s" % (i, redirect)) header = self.load(redirect, just_header=True) if 'location' in header and header['location']: @@ -96,38 +96,38 @@ class SimpleCrypter(Crypter, SimpleHoster): else: break else: - self.logError(_("Too many redirects")) + self.log_error(_("Too many redirects")) def decrypt(self, pyfile): self.prepare() - self.logDebug("Looking for link redirect...") - self.handleDirect(pyfile) + self.log_debug("Looking for link redirect...") + self.handle_direct(pyfile) if self.link: self.urls = [self.link] else: self.preload() - self.checkInfo() + self.check_info() - self.links = self.getLinks() or list() + self.links = self.get_links() or list() if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): - self.handlePages(pyfile) + self.handle_pages(pyfile) - self.logDebug("Package has %d links" % len(self.links)) + self.log_debug("Package has %d links" % len(self.links)) if self.links: self.packages = [(self.info['name'], self.links, self.info['folder'])] - def checkNameSize(self, getinfo=True): + def check_name_size(self, getinfo=True): if not self.info or getinfo: - self.logDebug("File info (BEFORE): %s" % self.info) - self.info.update(self.getInfo(self.pyfile.url, self.html)) - self.logDebug("File info (AFTER): %s" % self.info) + self.log_debug("File info (BEFORE): %s" % self.info) + self.info.update(self.get_info(self.pyfile.url, self.html)) + self.log_debug("File info (AFTER): %s" % self.info) try: url = self.info['url'].strip() @@ -144,11 +144,11 @@ class SimpleCrypter(Crypter, SimpleHoster): except Exception: pass - self.logDebug("File name: %s" % self.pyfile.name, + self.log_debug("File name: %s" % self.pyfile.name, "File folder: %s" % self.pyfile.name) - def getLinks(self): + def get_links(self): """ Returns the links extracted from self.html You should override this only if it's impossible to extract links using only the LINK_PATTERN. @@ -156,12 +156,12 @@ class SimpleCrypter(Crypter, SimpleHoster): return re.findall(self.LINK_PATTERN, self.html) - def handlePages(self, pyfile): + def handle_pages(self, pyfile): try: pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) except Exception: pages = 1 for p in xrange(2, pages + 1): - self.html = self.loadPage(p) - self.links += self.getLinks() + self.html = self.load_page(p) + self.links += self.get_links() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 067b97747..2e31c1176 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -25,7 +25,7 @@ statusMap = dict((v, k) for k, v in _statusMap.iteritems()) #@TODO: Remove in 0.4.10 def parseFileInfo(plugin, url="", html=""): if hasattr(plugin, "getInfo"): - info = plugin.getInfo(url, html) + info = plugin.get_info(url, html) res = info['name'], info['size'], info['status'], info['url'] else: url = urllib.unquote(url) @@ -42,7 +42,7 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 def create_getInfo(plugin): - def getInfo(urls): + def get_info(urls): for url in urls: if hasattr(plugin, "URL_REPLACEMENTS"): url = replace_patterns(url, plugin.URL_REPLACEMENTS) @@ -55,7 +55,7 @@ def timestamp(): return int(time.time() * 1000) -def secondsToMidnight(gmt=0): +def seconds_to_midnight(gmt=0): now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) if now.hour is 0 and now.minute < 10: @@ -76,7 +76,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.69" + __version__ = "1.70" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -132,7 +132,7 @@ class SimpleHoster(Hoster): example: ERROR_PATTERN = r'' - Instead overriding handleFree and handlePremium methods you may define the following patterns for basic link handling: + Instead overriding handle_free and handle_premium methods you may define the following patterns for basic link handling: LINK_PATTERN: (optional) group(1) should be the direct link for free and premium download example: LINK_PATTERN = r'
', " ", errmsg) - self.logWarning(self.info['error']) + self.log_warning(self.info['error']) if re.search('da(il)?y|today', errmsg, re.I): - wait_time = secondsToMidnight(gmt=2) + wait_time = seconds_to_midnight(gmt=2) else: wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1, "": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) - self.wantReconnect = wait_time > 300 + self.want_reconnect = wait_time > 300 self.retry(1, wait_time, _("Download limit exceeded")) if hasattr(self, 'HAPPY_HOUR_PATTERN') and re.search(self.HAPPY_HOUR_PATTERN, self.html): - self.multiDL = True + self.multi_dl = True if hasattr(self, 'ERROR_PATTERN'): m = re.search(self.ERROR_PATTERN, self.html) @@ -438,30 +438,30 @@ class SimpleHoster(Hoster): errmsg = m.group(0).strip() self.info['error'] = re.sub(r'<.*?>', " ", errmsg) - self.logWarning(self.info['error']) + self.log_warning(self.info['error']) if re.search('limit|wait|slot', errmsg, re.I): if re.search("da(il)?y|today", errmsg): - wait_time = secondsToMidnight(gmt=2) + wait_time = seconds_to_midnight(gmt=2) else: wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1, "": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) - self.wantReconnect = wait_time > 300 + self.want_reconnect = wait_time > 300 self.retry(1, wait_time, _("Download limit exceeded")) elif re.search('country|ip|region|nation', errmsg, re.I): self.fail(_("Connection from your current IP address is not allowed")) elif re.search('captcha|code', errmsg, re.I): - self.invalidCaptcha() + self.invalid_captcha() self.retry(10, reason=_("Wrong captcha")) elif re.search('countdown|expired', errmsg, re.I): self.retry(10, 60, _("Link expired")) elif re.search('maintenance|maintainance|temp', errmsg, re.I): - self.tempOffline() + self.temp_offline() elif re.search('up to|size', errmsg, re.I): self.fail(_("File too large for free download")) @@ -478,7 +478,7 @@ class SimpleHoster(Hoster): self.fail(_("File can be downloaded by premium users only")) else: - self.wantReconnect = True + self.want_reconnect = True self.retry(wait_time=60, reason=errmsg) elif hasattr(self, 'WAIT_PATTERN'): @@ -496,12 +496,12 @@ class SimpleHoster(Hoster): self.info.pop('error', None) - def checkStatus(self, getinfo=True): + def check_status(self, getinfo=True): if not self.info or getinfo: - self.logDebug("Update file info...") - self.logDebug("Previous file info: %s" % self.info) - self.info.update(self.getInfo(self.pyfile.url, self.html)) - self.logDebug("Current file info: %s" % self.info) + self.log_debug("Update file info...") + self.log_debug("Previous file info: %s" % self.info) + self.info.update(self.get_info(self.pyfile.url, self.html)) + self.log_debug("Current file info: %s" % self.info) try: status = self.info['status'] @@ -510,21 +510,21 @@ class SimpleHoster(Hoster): self.offline() elif status is 6: - self.tempOffline() + self.temp_offline() elif status is 8: self.fail(self.info['error'] if 'error' in self.info else _("Failed")) finally: - self.logDebug("File status: %s" % statusMap[status]) + self.log_debug("File status: %s" % statusMap[status]) - def checkNameSize(self, getinfo=True): + def check_name_size(self, getinfo=True): if not self.info or getinfo: - self.logDebug("Update file info...") - self.logDebug("Previous file info: %s" % self.info) - self.info.update(self.getInfo(self.pyfile.url, self.html)) - self.logDebug("Current file info: %s" % self.info) + self.log_debug("Update file info...") + self.log_debug("Previous file info: %s" % self.info) + self.info.update(self.get_info(self.pyfile.url, self.html)) + self.log_debug("Current file info: %s" % self.info) try: url = self.info['url'].strip() @@ -543,44 +543,44 @@ class SimpleHoster(Hoster): except Exception: pass - self.logDebug("File name: %s" % self.pyfile.name, + self.log_debug("File name: %s" % self.pyfile.name, "File size: %s byte" % self.pyfile.size if self.pyfile.size > 0 else "File size: Unknown") - def checkInfo(self): - self.checkNameSize() + def check_info(self): + self.check_name_size() if self.html: - self.checkErrors() - self.checkNameSize() + self.check_errors() + self.check_name_size() - self.checkStatus(getinfo=False) + self.check_status(getinfo=False) - #: Deprecated + #: Deprecated method def getFileInfo(self): self.info = {} - self.checkInfo() + self.check_info() return self.info - def handleDirect(self, pyfile): - link = self.directLink(pyfile.url, self.resumeDownload) + def handle_direct(self, pyfile): + link = self.direct_link(pyfile.url, self.resume_download) if link: - self.logInfo(_("Direct download link detected")) + self.log_info(_("Direct download link detected")) self.link = link else: - self.logDebug("Direct download link not found") + self.log_debug("Direct download link not found") - def handleMulti(self, pyfile): #: Multi-hoster handler + def handle_multi(self, pyfile): #: Multi-hoster handler pass - def handleFree(self, pyfile): + def handle_free(self, pyfile): if not hasattr(self, 'LINK_FREE_PATTERN'): - self.logError(_("Free download not implemented")) + self.log_error(_("Free download not implemented")) m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: @@ -589,11 +589,11 @@ class SimpleHoster(Hoster): self.link = m.group(1) - def handlePremium(self, pyfile): + def handle_premium(self, pyfile): if not hasattr(self, 'LINK_PREMIUM_PATTERN'): - self.logError(_("Premium download not implemented")) - self.logDebug("Handled as free download") - self.handleFree(pyfile) + self.log_error(_("Premium download not implemented")) + self.log_debug("Handled as free download") + self.handle_free(pyfile) m = re.search(self.LINK_PREMIUM_PATTERN, self.html) if m is None: @@ -602,7 +602,7 @@ class SimpleHoster(Hoster): self.link = m.group(1) - def retryFree(self): + def retry_free(self): if not self.premium: return self.premium = False diff --git a/module/plugins/internal/SolveMedia.py b/module/plugins/internal/SolveMedia.py index dde6223ed..927d4e536 100644 --- a/module/plugins/internal/SolveMedia.py +++ b/module/plugins/internal/SolveMedia.py @@ -9,7 +9,7 @@ from module.plugins.internal.Captcha import Captcha class SolveMedia(Captcha): __name__ = "SolveMedia" __type__ = "captcha" - __version__ = "0.14" + __version__ = "0.15" __description__ = """SolveMedia captcha service plugin""" __license__ = "GPLv3" @@ -25,10 +25,10 @@ class SolveMedia(Captcha): m = re.search(self.KEY_PATTERN, html) if m: self.key = m.group(1).strip() - self.logDebug("Key: %s" % self.key) + self.log_debug("Key: %s" % self.key) return self.key else: - self.logWarning("Key pattern not found") + self.log_warning("Key pattern not found") return None @@ -43,7 +43,7 @@ class SolveMedia(Captcha): magic = re.search(r'name="magic" value="(.+?)"', html).group(1) except AttributeError: - self.logWarning("Magic pattern not found") + self.log_warning("Magic pattern not found") magic = None try: @@ -54,13 +54,13 @@ class SolveMedia(Captcha): self.fail(_("SolveMedia challenge pattern not found")) else: - self.logDebug("Challenge: %s" % challenge) + self.log_debug("Challenge: %s" % challenge) try: result = self.result("http://api.solvemedia.com/papi/media", challenge) except Fail, e: - self.logWarning(e) + self.log_warning(e) self.plugin.invalidCaptcha() result = None @@ -81,8 +81,8 @@ class SolveMedia(Captcha): else: if "error" in html: - self.logWarning("Captcha code was invalid") - self.logDebug("Retry #%d" % i) + self.log_warning("Captcha code was invalid") + self.log_debug("Retry #%d" % i) html = self.plugin.load(redirect) else: break @@ -99,6 +99,6 @@ class SolveMedia(Captcha): cookies=True, imgtype="gif") - self.logDebug("Result: %s" % result) + self.log_debug("Result: %s" % result) return result diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 25a8a52de..4494b98c6 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.20" + __version__ = "1.21" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -48,7 +48,7 @@ class UnRar(Extractor): @classmethod - def isUsable(cls): + def is_usable(cls): if os.name == "nt": try: cls.CMD = os.path.join(pypath, "RAR.exe") @@ -79,7 +79,7 @@ class UnRar(Extractor): @classmethod - def isMultipart(cls, filename): + def is_multipart(cls, filename): return True if cls.re_multipart.search(filename) else False @@ -131,7 +131,7 @@ class UnRar(Extractor): break #: reading a percentage sign -> set progress and restart if c == '%': - self.notifyProgress(int(s)) + self.notify_progress(int(s)) s = "" #: not reading a digit -> therefore restart elif c not in digits: @@ -168,14 +168,14 @@ class UnRar(Extractor): self.files = self.list(password) - def getDeleteFiles(self): + def get_delete_files(self): dir, name = os.path.split(self.filename) #: actually extracted file files = [self.filename] #: eventually Multipart Files - files.extend(fs_join(dir, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir)) + files.extend(fs_join(dir, os.path.basename(file)) for file in filter(self.is_multipart, os.listdir(dir)) if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file)) return files diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 4c18a0e35..c2eb522a8 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -12,7 +12,7 @@ from module.utils import fs_encode class UnZip(Extractor): __name__ = "UnZip" - __version__ = "1.12" + __version__ = "1.13" __description__ = """Zip extractor plugin""" __license__ = "GPLv3" @@ -24,7 +24,7 @@ class UnZip(Extractor): @classmethod - def isUsable(cls): + def is_usable(cls): return sys.version_info[:2] >= (2, 6) diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index c26a91775..fb2302f4c 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -11,7 +11,7 @@ from module.plugins.internal.Plugin import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.38" + __version__ = "0.39" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -40,7 +40,7 @@ class XFSAccount(Account): def init(self): if not self.HOSTER_DOMAIN: - self.logError(_("Missing HOSTER_DOMAIN")) + self.log_error(_("Missing HOSTER_DOMAIN")) self.COOKIES = False else: @@ -52,7 +52,7 @@ class XFSAccount(Account): set_cookies(req.cj, self.COOKIES) - def loadAccountInfo(self, user, req): + def load_account_info(self, user, req): validuntil = None trafficleft = None leechtraffic = None @@ -71,16 +71,16 @@ class XFSAccount(Account): m = re.search(self.VALID_UNTIL_PATTERN, html) if m: expiredate = m.group(1).strip() - self.logDebug("Expire date: " + expiredate) + self.log_debug("Expire date: " + expiredate) try: validuntil = time.mktime(time.strptime(expiredate, "%d %B %Y")) except Exception, e: - self.logError(e) + self.log_error(e) else: - self.logDebug("Valid until: %s" % validuntil) + self.log_debug("Valid until: %s" % validuntil) if validuntil > time.mktime(time.gmtime()): premium = True @@ -89,7 +89,7 @@ class XFSAccount(Account): premium = False validuntil = None #: registered account type (not premium) else: - self.logDebug("VALID_UNTIL_PATTERN not found") + self.log_debug("VALID_UNTIL_PATTERN not found") m = re.search(self.TRAFFIC_LEFT_PATTERN, html) if m: @@ -109,12 +109,12 @@ class XFSAccount(Account): else: unit = "" - trafficleft = self.parseTraffic(size + unit) + trafficleft = self.parse_traffic(size + unit) except Exception, e: - self.logError(e) + self.log_error(e) else: - self.logDebug("TRAFFIC_LEFT_PATTERN not found") + self.log_debug("TRAFFIC_LEFT_PATTERN not found") leech = [m.groupdict() for m in re.finditer(self.LEECH_TRAFFIC_PATTERN, html)] if leech: @@ -136,12 +136,12 @@ class XFSAccount(Account): else: unit = "" - leechtraffic += self.parseTraffic(size + unit) + leechtraffic += self.parse_traffic(size + unit) except Exception, e: - self.logError(e) + self.log_error(e) else: - self.logDebug("LEECH_TRAFFIC_PATTERN not found") + self.log_debug("LEECH_TRAFFIC_PATTERN not found") return {'validuntil' : validuntil, 'trafficleft' : trafficleft, @@ -173,4 +173,4 @@ class XFSAccount(Account): html = self.load(url, post=inputs, req=req) if re.search(self.LOGIN_FAIL_PATTERN, html): - self.wrongPassword() + self.wrong_password() diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 0f5bfd5d7..c371e97f5 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.10" + __version__ = "0.11" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 0f4f90128..68cf25f14 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -6,14 +6,14 @@ import re from module.plugins.internal.ReCaptcha import ReCaptcha from module.plugins.internal.SolveMedia import SolveMedia -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, secondsToMidnight +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, seconds_to_midnight from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.53" + __version__ = "0.54" __pattern__ = r'^unmatchable$' @@ -51,8 +51,8 @@ class XFSHoster(SimpleHoster): def setup(self): - self.chunkLimit = -1 if self.premium else 1 - self.resumeDownload = self.multiDL = self.premium + self.chunk_limit = -1 if self.premium else 1 + self.resume_download = self.multi_dl = self.premium def prepare(self): @@ -80,20 +80,20 @@ class XFSHoster(SimpleHoster): super(XFSHoster, self).prepare() if self.DIRECT_LINK is None: - self.directDL = self.premium + self.direct_d_l = self.premium - def handleFree(self, pyfile): + def handle_free(self, pyfile): for i in xrange(1, 6): - self.logDebug("Getting download link #%d" % i) + self.log_debug("Getting download link #%d" % i) - self.checkErrors() + self.check_errors() m = re.search(self.LINK_PATTERN, self.html, re.S) if m: break - data = self.getPostParameters() + data = self.get_post_parameters() self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 0) @@ -109,24 +109,24 @@ class XFSHoster(SimpleHoster): if m: break else: - self.logError(data['op'] if 'op' in data else _("UNKNOWN")) + self.log_error(data['op'] if 'op' in data else _("UNKNOWN")) return "" self.link = m.group(1) - def handlePremium(self, pyfile): - return self.handleFree(pyfile) + def handle_premium(self, pyfile): + return self.handle_free(pyfile) - def handleMulti(self, pyfile): + def handle_multi(self, pyfile): if not self.account: self.fail(_("Only registered or premium users can use url leech feature")) - #only tested with easybytez.com + # only tested with easybytez.com self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) - action, inputs = self.parseHtmlForm() + action, inputs = self.parse_html_form() upload_id = "%012d" % int(random.random() * 10 ** 12) action += upload_id + "&js_on=1&utype=prem&upload_type=url" @@ -135,19 +135,19 @@ class XFSHoster(SimpleHoster): inputs['url_mass'] = pyfile.url inputs['up1oad_type'] = 'url' - self.logDebug(action, inputs) + self.log_debug(action, inputs) self.req.setOption("timeout", 600) #: wait for file to upload to easybytez.com self.html = self.load(action, post=inputs) - self.checkErrors() + self.check_errors() - action, inputs = self.parseHtmlForm('F1') + action, inputs = self.parse_html_form('F1') if not inputs: self.retry(reason=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) - self.logDebug(inputs) + self.log_debug(inputs) stmsg = inputs['st'] @@ -158,12 +158,12 @@ class XFSHoster(SimpleHoster): self.retry(20, 3 * 60, _("Can not leech file")) elif 'today' in stmsg: - self.retry(wait_time=secondsToMidnight(gmt=2), reason=_("You've used all Leech traffic today")) + self.retry(wait_time=seconds_to_midnight(gmt=2), reason=_("You've used all Leech traffic today")) else: self.fail(stmsg) - #get easybytez.com link for uploaded file + # get easybytez.com link for uploaded file m = re.search(self.LINK_LEECH_PATTERN, self.html) if m is None: self.error(_("LINK_LEECH_PATTERN not found")) @@ -174,22 +174,22 @@ class XFSHoster(SimpleHoster): self.link = header['location'] - def getPostParameters(self): + def get_post_parameters(self): if self.FORM_PATTERN or self.FORM_INPUTS_MAP: - action, inputs = self.parseHtmlForm(self.FORM_PATTERN or "", self.FORM_INPUTS_MAP or {}) + action, inputs = self.parse_html_form(self.FORM_PATTERN or "", self.FORM_INPUTS_MAP or {}) else: - action, inputs = self.parseHtmlForm(input_names={'op': re.compile(r'^download')}) + action, inputs = self.parse_html_form(input_names={'op': re.compile(r'^download')}) if not inputs: - action, inputs = self.parseHtmlForm('F1') + action, inputs = self.parse_html_form('F1') if not inputs: self.retry(reason=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) - self.logDebug(inputs) + self.log_debug(inputs) if 'op' in inputs: if "password" in inputs: - password = self.getPassword() + password = self.get_password() if password: inputs['password'] = password else: @@ -199,9 +199,9 @@ class XFSHoster(SimpleHoster): m = re.search(self.WAIT_PATTERN, self.html) if m: wait_time = int(m.group(1)) - self.setWait(wait_time, False) + self.set_wait(wait_time, False) - self.handleCaptcha(inputs) + self.handle_captcha(inputs) self.wait() else: inputs['referer'] = self.pyfile.url @@ -216,11 +216,11 @@ class XFSHoster(SimpleHoster): return inputs - def handleCaptcha(self, inputs): + def handle_captcha(self, inputs): m = re.search(self.CAPTCHA_PATTERN, self.html) if m: captcha_url = m.group(1) - inputs['code'] = self.decryptCaptcha(captcha_url) + inputs['code'] = self.decrypt_captcha(captcha_url) return m = re.search(self.CAPTCHA_BLOCK_PATTERN, self.html, re.S) @@ -228,11 +228,11 @@ class XFSHoster(SimpleHoster): captcha_div = m.group(1) numerals = re.findall(r'(\d)', html_unescape(captcha_div)) - self.logDebug(captcha_div) + self.log_debug(captcha_div) inputs['code'] = "".join(a[1] for a in sorted(numerals, key=lambda num: int(num[0]))) - self.logDebug("Captcha code: %s" % inputs['code'], numerals) + self.log_debug("Captcha code: %s" % inputs['code'], numerals) return recaptcha = ReCaptcha(self) @@ -243,7 +243,7 @@ class XFSHoster(SimpleHoster): captcha_key = recaptcha.detect_key() else: - self.logDebug("ReCaptcha key: %s" % captcha_key) + self.log_debug("ReCaptcha key: %s" % captcha_key) if captcha_key: inputs['recaptcha_response_field'], inputs['recaptcha_challenge_field'] = recaptcha.challenge(captcha_key) @@ -257,7 +257,7 @@ class XFSHoster(SimpleHoster): captcha_key = solvemedia.detect_key() else: - self.logDebug("SolveMedia key: %s" % captcha_key) + self.log_debug("SolveMedia key: %s" % captcha_key) if captcha_key: inputs['adcopy_response'], inputs['adcopy_challenge'] = solvemedia.challenge(captcha_key) -- cgit v1.2.3 From d2e2b127651a5a44b56337eb6d9ca246c97a208a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 17 Jul 2015 03:03:26 +0200 Subject: Spare fixes and code cosmetics --- module/plugins/internal/Account.py | 25 ++++++++++++++++--------- module/plugins/internal/Hoster.py | 16 ++++++++-------- module/plugins/internal/MultiHook.py | 2 +- module/plugins/internal/MultiHoster.py | 8 ++++---- module/plugins/internal/Plugin.py | 4 ++-- module/plugins/internal/SevenZip.py | 2 +- module/plugins/internal/SimpleCrypter.py | 8 ++++---- module/plugins/internal/SimpleHoster.py | 11 ++++++----- module/plugins/internal/UnRar.py | 4 ++-- module/plugins/internal/XFSAccount.py | 4 ++-- module/plugins/internal/XFSHoster.py | 4 ++-- 11 files changed, 48 insertions(+), 40 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 9e0efaee6..8fd16bba9 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -60,9 +60,9 @@ class Account(Plugin): #: set timestamp for login self.timestamps[user] = time.time() - req = self.get_account_request(user) + self.req = self.get_account_request(user) try: - self.login(user, data, req) + self.login(user, data, self.req) except WrongPassword: self.log_warning( @@ -82,8 +82,10 @@ class Account(Plugin): success = True finally: - if req: - req.close() + if hasattr(self, "req"): + if self.req: + self.req.close() + del self.req return success @@ -158,26 +160,31 @@ class Account(Plugin): if force or name not in self.infos: self.log_debug("Get Account Info for %s" % name) - req = self.get_account_request(name) + self.req = self.get_account_request(name) try: - infos = self.load_account_info(name, req) + infos = self.load_account_info(name, self.req) if not type(infos) == dict: raise Exception("Wrong return format") + except Exception, e: - infos = super(self.__class__, self).load_account_info(name, req) + infos = super(self.__class__, self).load_account_info(name, self.req) infos['error'] = str(e) if self.core.debug: traceback.print_exc() - if req: - req.close() + finally: + if hasattr(self, "req"): + if self.req: + self.req.close() + del self.req self.log_debug("Account Info: %s" % infos) infos['timestamp'] = time.time() self.infos[name] = infos + elif "timestamp" in self.infos[name] and self.infos[name]['timestamp'] + self.info_threshold * 60 < time.time(): self.log_debug("Reached timeout for account data") self.schedule_refresh(name) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 09672f6f0..722fae2e0 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -12,7 +12,7 @@ if os.name != "nt": import grp import pwd -from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip, parseHtmlForm +from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip, parse_html_form from module.utils import fs_decode, fs_encode, save_join as fs_join @@ -67,21 +67,21 @@ class Hoster(Plugin): #: username/login self.user = None - if self.account and not self.account.canUse(): + if self.account and not self.account.can_use(): self.account = None if self.account: - self.user, data = self.account.selectAccount() + self.user, data = self.account.select_account() #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest(self.user) + self.req = self.account.get_account_request(self.user) self.chunk_limit = -1 #: chunk limit, -1 for unlimited #: enables resume (will be ignored if server dont accept chunks) self.resume_download = True #: premium status - self.premium = self.account.isPremium(self.user) + self.premium = self.account.is_premium(self.user) else: self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) @@ -132,7 +132,7 @@ class Hoster(Plugin): self.thread = thread if self.account: - self.account.checkLogin(self.user) + self.account.check_login(self.user) else: self.req.clearCookies() @@ -626,14 +626,14 @@ class Hoster(Plugin): def parse_html_form(self, attr_str="", input_names={}): - return parseHtmlForm(attr_str, self.html, input_names) + return parse_html_form(attr_str, self.html, input_names) def check_traffic_left(self): if not self.account: return True - traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] + traffic = self.account.get_account_info(self.user, True)['trafficleft'] if traffic is None: return False diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 623836cdc..2e99afa1c 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -83,7 +83,7 @@ class MultiHook(Hook): def load_account(self): self.account = self.core.accountManager.getAccountPlugin(self.pluginname) - if self.account and not self.account.canUse(): + if self.account and not self.account.can_use(): self.account = None if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 8360d871e..363d46739 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -33,7 +33,7 @@ class MultiHoster(SimpleHoster): self.info = {} self.html = "" self.link = "" #@TODO: Move to Hoster in 0.4.10 - self.direct_d_l = False #@TODO: Move to Hoster in 0.4.10 + self.direct_dl = False #@TODO: Move to Hoster in 0.4.10 if not self.get_config('use_premium', True): self.retry_free() @@ -47,9 +47,9 @@ class MultiHoster(SimpleHoster): set_cookies(self.req.cj, self.COOKIES) if self.DIRECT_LINK is None: - self.direct_d_l = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) + self.direct_dl = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) else: - self.direct_d_l = self.DIRECT_LINK + self.direct_dl = self.DIRECT_LINK self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) @@ -58,7 +58,7 @@ class MultiHoster(SimpleHoster): try: self.prepare() - if self.direct_d_l: + if self.direct_dl: self.check_info() self.log_debug("Looking for direct download link...") self.handle_direct(pyfile) diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 0b08858f3..01294ea92 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -236,7 +236,7 @@ class Plugin(object): return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip() - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=True, req=None): + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=True, multipart=True, req=None): """ Load content at url and returns it @@ -266,7 +266,7 @@ class Plugin(object): else: req = self.core.requestFactory.getRequest(self.__name__) - res = req.load(url, get, post, ref, cookies, just_header, True, bool(decode)) + res = req.load(url, get, post, ref, cookies, just_header, multipart, bool(decode)) if decode: res = html_unescape(res).decode(decode if isinstance(decode, basestring) else 'utf8') diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index 86ff5156e..d067468dd 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -147,7 +147,7 @@ class SevenZip(UnRar): #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.manager.logDebug(" ".join(call)) + self.manager.log_debug(" ".join(call)) p = subprocess.Popen(call, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return p diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index fe47178e3..82f5ef332 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -55,10 +55,10 @@ class SimpleCrypter(Crypter, SimpleHoster): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") account = self.core.accountManager.getAccountPlugin(account_name) - if account and account.canUse(): - self.user, data = account.selectAccount() - self.req = account.getAccountRequest(self.user) - self.premium = account.isPremium(self.user) + if account and account.can_use(): + self.user, data = account.select_account() + self.req = account.get_account_request(self.user) + self.premium = account.is_premium(self.user) self.account = account diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2e31c1176..0e4b94534 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -266,7 +266,7 @@ class SimpleHoster(Hoster): self.info = {} self.html = "" #@TODO: Recheck in 0.4.10 self.link = "" #@TODO: Recheck in 0.4.10 - self.direct_d_l = False + self.direct_dl = False self.multihost = False if not self.get_config('use_premium', True): @@ -297,9 +297,9 @@ class SimpleHoster(Hoster): return if self.DIRECT_LINK is None: - self.direct_d_l = bool(self.account) + self.direct_dl = bool(self.account) else: - self.direct_d_l = self.DIRECT_LINK + self.direct_dl = self.DIRECT_LINK self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) @@ -316,7 +316,7 @@ class SimpleHoster(Hoster): self.prepare() self.check_info() - if self.direct_d_l: + if self.direct_dl: self.log_debug("Looking for direct download link...") self.handle_direct(pyfile) @@ -364,7 +364,8 @@ class SimpleHoster(Hoster): self.invalid_captcha() self.retry(10, reason=_("Wrong captcha")) - elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, file_size=self.info['size']): + elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, + file_size=self.info['size']): self.error(_("Empty file")) else: diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 4494b98c6..50c421e8d 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -191,7 +191,7 @@ class UnRar(Extractor): raise ArchiveError(_("Cannot open file")) if err.strip(): #: only log error at this point - self.manager.logError(err.strip()) + self.manager.log_error(err.strip()) result = set() if not self.fullpath and self.VERSION.startswith('5'): @@ -237,7 +237,7 @@ class UnRar(Extractor): #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.manager.logDebug(" ".join(call)) + self.manager.log_debug(" ".join(call)) p = subprocess.Popen(call, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return p diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index fb2302f4c..23759f6d3 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -5,7 +5,7 @@ import time import urlparse from module.plugins.internal.Account import Account -from module.plugins.internal.Plugin import parseHtmlForm, set_cookies +from module.plugins.internal.Plugin import parse_html_form, set_cookies class XFSAccount(Account): @@ -157,7 +157,7 @@ class XFSAccount(Account): self.LOGIN_URL = urlparse.urljoin(self.HOSTER_URL, "login.html") html = self.load(self.LOGIN_URL, req=req) - action, inputs = parseHtmlForm('name="FL"', html) + action, inputs = parse_html_form('name="FL"', html) if not inputs: inputs = {'op' : "login", 'redirect': self.HOSTER_URL} diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 68cf25f14..9f8728512 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -47,7 +47,7 @@ class XFSHoster(SimpleHoster): SOLVEMEDIA_PATTERN = None FORM_PATTERN = None - FORM_INPUTS_MAP = None #: dict passed as input_names to parseHtmlForm + FORM_INPUTS_MAP = None #: dict passed as input_names to parse_html_form def setup(self): @@ -80,7 +80,7 @@ class XFSHoster(SimpleHoster): super(XFSHoster, self).prepare() if self.DIRECT_LINK is None: - self.direct_d_l = self.premium + self.direct_dl = self.premium def handle_free(self, pyfile): -- cgit v1.2.3 From 1f5a55ae2133a782bdcca334ecbcdbde50dbcf99 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 17 Jul 2015 15:29:48 +0200 Subject: No more need to use the req argument when call load method --- module/plugins/internal/XFSAccount.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 23759f6d3..4005b8e21 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -64,7 +64,7 @@ class XFSAccount(Account): 'leechtraffic': leechtraffic, 'premium' : premium} - html = self.load(self.HOSTER_URL, get={'op': "my_account"}, req=req) + html = self.load(self.HOSTER_URL, get={'op': "my_account"}) premium = True if re.search(self.PREMIUM_PATTERN, html) else False @@ -155,7 +155,7 @@ class XFSAccount(Account): if not self.LOGIN_URL: self.LOGIN_URL = urlparse.urljoin(self.HOSTER_URL, "login.html") - html = self.load(self.LOGIN_URL, req=req) + html = self.load(self.LOGIN_URL) action, inputs = parse_html_form('name="FL"', html) if not inputs: @@ -170,7 +170,7 @@ class XFSAccount(Account): else: url = self.HOSTER_URL - html = self.load(url, post=inputs, req=req) + html = self.load(url, post=inputs) if re.search(self.LOGIN_FAIL_PATTERN, html): self.wrong_password() -- cgit v1.2.3 From ce3788f41d400e486c13126891641ad45740020c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Jul 2015 19:19:51 +0200 Subject: Reorder some functions --- module/plugins/internal/Hoster.py | 40 +++++++++++++++++-- module/plugins/internal/Plugin.py | 26 +++++++++++- module/plugins/internal/SimpleHoster.py | 70 +++------------------------------ 3 files changed, 68 insertions(+), 68 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 722fae2e0..9a9e61e46 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -6,25 +6,47 @@ import inspect import os import random import time +import urllib import urlparse if os.name != "nt": import grp import pwd -from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip, parse_html_form +from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip + chunks, replace_patterns, seconds_to_midnight, + set_cookies, parse_html_form, parse_html_tag_attr_value, + timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join -def get_info(urls): +#@TODO: Remove in 0.4.10 +def parse_fileInfo(klass, url="", html=""): + info = klass.get_info(url, html) + return info['name'], info['size'], info['status'], info['url'] + + +#@TODO: Remove in 0.4.10 +def getInfo(urls): # result = [ .. (name, size, status, url) .. ] pass +#@TODO: Remove in 0.4.10 +def create_getInfo(klass): + def get_info(urls): + for url in urls: + if hasattr(klass, "URL_REPLACEMENTS"): + url = replace_patterns(url, klass.URL_REPLACEMENTS) + yield parse_fileInfo(klass, url) + + return get_info + + class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] @@ -111,6 +133,18 @@ class Hoster(Plugin): self.init() + @classmethod + def get_info(cls, url="", html=""): + url = urllib.unquote(url) + url_p = urlparse.urlparse(url) + return {'name' : (url_p.path.split('/')[-1] + or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] + or url_p.netloc.split('.', 1)[0]), + 'size' : 0, + 'status': 3 if url else 8, + 'url' : url} + + def init(self): """ Initialize the plugin (in addition to `__init__`) diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 01294ea92..d1b8bb1cf 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -11,6 +11,29 @@ from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as from module.utils import fs_encode, fs_decode, html_unescape, save_join as fs_join +#@TODO: Move to utils in 0.4.10 +def timestamp(): + return int(time.time() * 1000) + + +def seconds_to_midnight(gmt=0): + now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) + + if now.hour is 0 and now.minute < 10: + midnight = now + else: + midnight = now + datetime.timedelta(days=1) + + td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now + + if hasattr(td, 'total_seconds'): + res = td.total_seconds() + else: #@NOTE: work-around for python 2.5 and 2.6 missing datetime.timedelta.total_seconds + res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 + + return int(res) + + def replace_patterns(string, ruleslist): for r in ruleslist: rf, rt = r @@ -67,6 +90,7 @@ def parse_html_form(attr_str, html, input_names={}): return {}, None #: no matching form found +#@TODO: Move to utils in 0.4.10 def chunks(iterable, size): it = iter(iterable) item = list(islice(it, size)) @@ -78,7 +102,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.12" + __version__ = "0.13" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 0e4b94534..772e6ea2f 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -13,70 +13,19 @@ import urlparse from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL -from module.plugins.internal.Hoster import Hoster +from module.plugins.internal.Hoster import Hoster, parse_fileInfo, create_getInfo from module.plugins.internal.Plugin import Fail, Retry, replace_patterns, set_cookies -from module.utils import fixup, fs_encode, parseFileSize +from module.utils import fixup, fs_encode, parseFileSize as parse_size #@TODO: Adapt and move to PyFile in 0.4.10 statusMap = dict((v, k) for k, v in _statusMap.iteritems()) -#@TODO: Remove in 0.4.10 -def parseFileInfo(plugin, url="", html=""): - if hasattr(plugin, "getInfo"): - info = plugin.get_info(url, html) - res = info['name'], info['size'], info['status'], info['url'] - else: - url = urllib.unquote(url) - url_p = urlparse.urlparse(url) - res = ((url_p.path.split('/')[-1] - or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] - or url_p.netloc.split('.', 1)[0]), - 0, - 3 if url else 8, - url) - - return res - - -#@TODO: Remove in 0.4.10 -def create_getInfo(plugin): - def get_info(urls): - for url in urls: - if hasattr(plugin, "URL_REPLACEMENTS"): - url = replace_patterns(url, plugin.URL_REPLACEMENTS) - yield parseFileInfo(plugin, url) - - return getInfo - - -def timestamp(): - return int(time.time() * 1000) - - -def seconds_to_midnight(gmt=0): - now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) - - if now.hour is 0 and now.minute < 10: - midnight = now - else: - midnight = now + datetime.timedelta(days=1) - - td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now - - if hasattr(td, 'total_seconds'): - res = td.total_seconds() - else: #@NOTE: work-around for python 2.5 and 2.6 missing datetime.timedelta.total_seconds - res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 - - return int(res) - - class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.70" + __version__ = "1.71" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -166,14 +115,7 @@ class SimpleHoster(Hoster): @classmethod def api_info(cls, url): - url = urllib.unquote(url) - url_p = urlparse.urlparse(url) - return {'name' : (url_p.path.split('/')[-1] - or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] - or url_p.netloc.split('.', 1)[0]), - 'size' : 0, - 'status': 3 if url else 8, - 'url' : url} + return super(SimpleHoster, self).get_info(url) @classmethod @@ -240,11 +182,11 @@ class SimpleHoster(Hoster): if 'S' in info['pattern']: size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info['pattern'] else info['pattern']['S'], cls.SIZE_REPLACEMENTS) - info['size'] = parseFileSize(size) + info['size'] = parse_size(size) elif isinstance(info['size'], basestring): unit = info['units'] if 'units' in info else None - info['size'] = parseFileSize(info['size'], unit) + info['size'] = parse_size(info['size'], unit) if 'H' in info['pattern']: hashtype = info['pattern']['T'] if 'T' in info['pattern'] else "hash" -- cgit v1.2.3 From 9e5d813d7721e351ac02ba72bdc473a7d77ba6b7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Jul 2015 20:04:36 +0200 Subject: Code cosmetics --- module/plugins/internal/Account.py | 6 +++--- module/plugins/internal/AdYouLike.py | 2 +- module/plugins/internal/AdsCaptcha.py | 2 +- module/plugins/internal/DeadCrypter.py | 13 ++++++------- module/plugins/internal/DeadHoster.py | 13 ++++++------- module/plugins/internal/Hoster.py | 6 +++--- module/plugins/internal/MultiHook.py | 4 ++-- module/plugins/internal/Plugin.py | 2 +- module/plugins/internal/ReCaptcha.py | 2 +- module/plugins/internal/SimpleHoster.py | 8 ++++---- module/plugins/internal/SolveMedia.py | 6 +++--- 11 files changed, 31 insertions(+), 33 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 8fd16bba9..b437b4a55 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -6,7 +6,7 @@ import time import traceback from module.plugins.internal.Plugin import Plugin -from module.utils import compare_time, lock, parseFileSize +from module.utils import compare_time, lock, parseFileSize as parse_size class WrongPassword(Exception): @@ -266,7 +266,7 @@ class Account(Plugin): if not compare_time(start.split(":"), end.split(":")): continue except Exception: - self.log_warning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + self.log_warning(_("Your Time %s has wrong format, use 1:22-3:44") % time_data) if user in self.infos: if "validuntil" in self.infos[user]: @@ -291,7 +291,7 @@ class Account(Plugin): def parse_traffic(self, value, unit=None): #: return kilobytes if not unit and not isinstance(value, basestring): unit = "KB" - return parseFileSize(value, unit) + return parse_size(value, unit) def wrong_password(self): diff --git a/module/plugins/internal/AdYouLike.py b/module/plugins/internal/AdYouLike.py index 07e7e4d17..9036b4632 100644 --- a/module/plugins/internal/AdYouLike.py +++ b/module/plugins/internal/AdYouLike.py @@ -30,7 +30,7 @@ class AdYouLike(Captcha): self.log_debug("Ayl: %s | Callback: %s" % self.key) return self.key #: key is the tuple(ayl, callback) else: - self.log_warning("Ayl or callback pattern not found") + self.log_warning(_("Ayl or callback pattern not found")) return None diff --git a/module/plugins/internal/AdsCaptcha.py b/module/plugins/internal/AdsCaptcha.py index b45a6dfda..788b5e71a 100644 --- a/module/plugins/internal/AdsCaptcha.py +++ b/module/plugins/internal/AdsCaptcha.py @@ -30,7 +30,7 @@ class AdsCaptcha(Captcha): self.log_debug("Key: %s | ID: %s" % self.key) return self.key else: - self.log_warning("Key or id pattern not found") + self.log_warning(_("Key or id pattern not found")) return None diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index d79551b52..15ba68b28 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.Crypter import Crypter -from module.plugins.internal.SimpleCrypter import create_getInfo +from module.plugins.internal.Crypter import Crypter, create_getInfo class DeadCrypter(Crypter): __name__ = "DeadCrypter" __type__ = "crypter" - __version__ = "0.07" + __version__ = "0.08" __pattern__ = r'^unmatchable$' @@ -17,10 +16,10 @@ class DeadCrypter(Crypter): @classmethod - def api_info(cls, *args, **kwargs): - api = super(DeadCrypter, cls).api_info(*args, **kwargs) - api['status'] = 1 - return api + def get_info(cls, *args, **kwargs): + info = super(DeadCrypter, cls).get_info(*args, **kwargs) + info['status'] = 1 + return info def setup(self): diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 86f4381a3..5050ba059 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.Hoster import Hoster -from module.plugins.internal.SimpleHoster import create_getInfo +from module.plugins.internal.Hoster import Hoster, create_getInfo class DeadHoster(Hoster): __name__ = "DeadHoster" __type__ = "hoster" - __version__ = "0.17" + __version__ = "0.18" __pattern__ = r'^unmatchable$' @@ -17,10 +16,10 @@ class DeadHoster(Hoster): @classmethod - def api_info(cls, *args, **kwargs): - api = super(DeadHoster, cls).api_info(*args, **kwargs) - api['status'] = 1 - return api + def get_info(cls, *args, **kwargs): + info = super(DeadHoster, cls).get_info(*args, **kwargs) + info['status'] = 1 + return info def setup(self): diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 9a9e61e46..15c46101b 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -216,7 +216,7 @@ class Hoster(Plugin): wait_until = time.time() + wait_time + 1 self.log_debug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), - "Wait: %d+1 seconds" % wait_time) + "Wait: %d (+1) seconds" % wait_time) self.pyfile.waitUntil = wait_until @@ -242,7 +242,7 @@ class Hoster(Plugin): pyfile.setStatus("waiting") self.log_info(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), - _("Reconnect: %s") % self.want_reconnect) + _("Reconnect: %s") % self.want_reconnect) if self.account: self.log_debug("Ignore reconnection due account logged") @@ -443,7 +443,7 @@ class Hoster(Plugin): self.fail(_("No url given")) if self.core.debug: - self.log_debug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + self.log_debug("Download url " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) self.correct_captcha() self.check_for_same_files() diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 2e99afa1c..7afe95705 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -76,7 +76,7 @@ class MultiHook(Hook): self.pluginmodule = self.core.pluginManager.loadModule(self.plugintype, self.pluginname) self.pluginclass = getattr(self.pluginmodule, self.pluginname) else: - self.log_warning("Hook plugin will be deactivated due missing plugin reference") + self.log_warning(_("Hook plugin will be deactivated due missing plugin reference")) self.set_config('activated', False) @@ -87,7 +87,7 @@ class MultiHook(Hook): self.account = None if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: - self.log_warning("Hook plugin will be deactivated due missing account reference") + self.log_warning(_("Hook plugin will be deactivated due missing account reference")) self.set_config('activated', False) diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index d1b8bb1cf..dc5995e65 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -282,7 +282,7 @@ class Plugin(object): self.fail(_("No url given")) if self.core.debug: - self.log_debug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + self.log_debug("Load url " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) if req is None: if hasattr(self, "req"): diff --git a/module/plugins/internal/ReCaptcha.py b/module/plugins/internal/ReCaptcha.py index 79bda9051..1caf6b7b0 100644 --- a/module/plugins/internal/ReCaptcha.py +++ b/module/plugins/internal/ReCaptcha.py @@ -35,7 +35,7 @@ class ReCaptcha(Captcha): self.log_debug("Key: %s" % self.key) return self.key else: - self.log_warning("Key pattern not found") + self.log_warning(_("Key pattern not found")) return None diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 772e6ea2f..93008f16d 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -12,8 +12,8 @@ import urlparse from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader -from module.network.RequestFactory import getURL -from module.plugins.internal.Hoster import Hoster, parse_fileInfo, create_getInfo +from module.network.RequestFactory import getURL as get_url +from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo from module.plugins.internal.Plugin import Fail, Retry, replace_patterns, set_cookies from module.utils import fixup, fs_encode, parseFileSize as parse_size @@ -136,7 +136,7 @@ class SimpleHoster(Hoster): elif info['status'] is 3: try: - html = getURL(url, cookies=cls.COOKIES, decode=cls.TEXT_ENCODING) + html = get_url(url, cookies=cls.COOKIES, decode=cls.TEXT_ENCODING) except BadHeader, e: info['error'] = "%d: %s" % (e.code, e.content) @@ -322,7 +322,7 @@ class SimpleHoster(Hoster): except Exception: pass - self.log_warning("Check result: " + errmsg, "Waiting 1 minute and retry") + self.log_warning(_("Check result: ") + errmsg, _("Waiting 1 minute and retry")) self.want_reconnect = True self.retry(wait_time=60, reason=errmsg) else: diff --git a/module/plugins/internal/SolveMedia.py b/module/plugins/internal/SolveMedia.py index 927d4e536..87537470f 100644 --- a/module/plugins/internal/SolveMedia.py +++ b/module/plugins/internal/SolveMedia.py @@ -28,7 +28,7 @@ class SolveMedia(Captcha): self.log_debug("Key: %s" % self.key) return self.key else: - self.log_warning("Key pattern not found") + self.log_warning(_("Key pattern not found") return None @@ -43,7 +43,7 @@ class SolveMedia(Captcha): magic = re.search(r'name="magic" value="(.+?)"', html).group(1) except AttributeError: - self.log_warning("Magic pattern not found") + self.log_warning(_("Magic pattern not found") magic = None try: @@ -81,7 +81,7 @@ class SolveMedia(Captcha): else: if "error" in html: - self.log_warning("Captcha code was invalid") + self.log_warning(_("Captcha code was invalid")) self.log_debug("Retry #%d" % i) html = self.plugin.load(redirect) else: -- cgit v1.2.3 From 1bc7db9098967a166b17df59065861fbe3cd41d8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Jul 2015 23:26:48 +0200 Subject: [Hoster] Fix the http request issue --- module/plugins/internal/Account.py | 5 +++-- module/plugins/internal/Hoster.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index b437b4a55..951d279be 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -16,7 +16,7 @@ class WrongPassword(Exception): class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.04" + __version__ = "0.05" __description__ = """Base account plugin""" __license__ = "GPLv3" @@ -92,8 +92,9 @@ class Account(Plugin): def relogin(self, user): req = self.get_account_request(user) if req: - req.cj.clear() + req.clearCookies() req.close() + if user in self.infos: del self.infos[user] #: delete old information diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 15c46101b..637d9f3b5 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -165,6 +165,8 @@ class Hoster(Plugin): """ self.thread = thread + self.req.renewHTTPRequest() + if self.account: self.account.check_login(self.user) else: -- cgit v1.2.3 From dad722ac7255640e7e0541c4094a4d2e4de79cd3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 00:05:58 +0200 Subject: Code cosmetics (2) --- module/plugins/internal/Account.py | 20 +++++++------- module/plugins/internal/AdYouLike.py | 2 +- module/plugins/internal/AdsCaptcha.py | 2 +- module/plugins/internal/Captcha.py | 2 +- module/plugins/internal/Crypter.py | 2 +- module/plugins/internal/Hook.py | 4 +-- module/plugins/internal/Hoster.py | 48 ++++++++++++++++----------------- module/plugins/internal/MultiHook.py | 6 ++--- module/plugins/internal/OCR.py | 12 ++++----- module/plugins/internal/Plugin.py | 16 +++++------ module/plugins/internal/SevenZip.py | 10 +++---- module/plugins/internal/SimpleHoster.py | 2 +- module/plugins/internal/UnRar.py | 28 +++++++++---------- module/plugins/internal/XFSAccount.py | 6 ++--- module/plugins/internal/XFSHoster.py | 8 +++--- 15 files changed, 84 insertions(+), 84 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 951d279be..534f5607d 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -28,12 +28,12 @@ class Account(Plugin): self.manager = manager self.accounts = {} - self.infos = {} #: cache for account information + self.infos = {} #: Cache for account information self.lock = threading.RLock() self.timestamps = {} - self.login_timeout = 10 * 60 #: after that time (in minutes) pyload will relogin the account - self.info_threshold = 10 * 60 #: after that time (in minutes) account data will be reloaded + self.login_timeout = 10 * 60 #: After that time (in minutes) pyload will relogin the account + self.info_threshold = 10 * 60 #: After that time (in minutes) account data will be reloaded self.init() @@ -57,7 +57,7 @@ class Account(Plugin): @lock def _login(self, user, data): - #: set timestamp for login + #: Set timestamp for login self.timestamps[user] = time.time() self.req = self.get_account_request(user) @@ -96,7 +96,7 @@ class Account(Plugin): req.close() if user in self.infos: - del self.infos[user] #: delete old information + del self.infos[user] #: Delete old information return self._login(user, self.accounts[user]) @@ -113,7 +113,7 @@ class Account(Plugin): Updates account and return true if anything changed """ if user in self.accounts: - self.accounts[user]['valid'] = True #: do not remove or accounts will not login + self.accounts[user]['valid'] = True #: Do not remove or accounts will not login if password: self.accounts[user]['password'] = password self.relogin(user) @@ -210,13 +210,13 @@ class Account(Plugin): """ return {"validuntil" : None, #: -1 for unlimited "login" : name, - #: "password" : self.accounts[name]['password'], #: commented due security reason + # "password" : self.accounts[name]['password'], #: Commented due security reason "options" : self.accounts[name]['options'], "valid" : self.accounts[name]['valid'], - "trafficleft": None, #: in bytes, -1 for unlimited + "trafficleft": None, #: In bytes, -1 for unlimited "maxtraffic" : None, "premium" : None, - "timestamp" : 0, #: time this info was retrieved + "timestamp" : 0, #: Time this info was retrieved "type" : self.__name__} @@ -289,7 +289,7 @@ class Account(Plugin): return self.select_account() != (None, None) - def parse_traffic(self, value, unit=None): #: return kilobytes + def parse_traffic(self, value, unit=None): #: Return kilobytes if not unit and not isinstance(value, basestring): unit = "KB" return parse_size(value, unit) diff --git a/module/plugins/internal/AdYouLike.py b/module/plugins/internal/AdYouLike.py index 9036b4632..64ce6e7fa 100644 --- a/module/plugins/internal/AdYouLike.py +++ b/module/plugins/internal/AdYouLike.py @@ -28,7 +28,7 @@ class AdYouLike(Captcha): if m and n: self.key = (m.group(1).strip(), n.group(1).strip()) self.log_debug("Ayl: %s | Callback: %s" % self.key) - return self.key #: key is the tuple(ayl, callback) + return self.key #: Key is the tuple(ayl, callback) else: self.log_warning(_("Ayl or callback pattern not found")) return None diff --git a/module/plugins/internal/AdsCaptcha.py b/module/plugins/internal/AdsCaptcha.py index 788b5e71a..5f4dbaa86 100644 --- a/module/plugins/internal/AdsCaptcha.py +++ b/module/plugins/internal/AdsCaptcha.py @@ -26,7 +26,7 @@ class AdsCaptcha(Captcha): m = re.search(self.PUBLICKEY_PATTERN, html) n = re.search(self.CAPTCHAID_PATTERN, html) if m and n: - self.key = (m.group(1).strip(), n.group(1).strip()) #: key is the tuple(PublicKey, CaptchaId) + self.key = (m.group(1).strip(), n.group(1).strip()) #: Key is the tuple(PublicKey, CaptchaId) self.log_debug("Key: %s | ID: %s" % self.key) return self.key else: diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 50ebea89c..68d5d3bb8 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -17,7 +17,7 @@ class Captcha(Plugin): super(Captcha, self).__init__(plugin.core) self.plugin = plugin - self.key = None #: last key detected + self.key = None #: Last key detected #@TODO: Recheck in 0.4.10 diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index f6d3528fd..dbecb0cf8 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -20,7 +20,7 @@ class Crypter(Hoster): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - html = None #: last html loaded #@TODO: Move to Hoster + html = None #: Last html loaded #@TODO: Move to Hoster def __init__(self, pyfile): diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 7ef8f0189..162c9ecbf 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -43,7 +43,7 @@ class Hook(Plugin): #: `HookManager` self.manager = manager - #: automatically register event listeners for functions, attribute will be deleted dont use it yourself + #: Automatically register event listeners for functions, attribute will be deleted dont use it yourself self.event_map = {} #: Deprecated alternative to event_map @@ -68,7 +68,7 @@ class Hook(Plugin): else: self.manager.addEvent(event, getattr(self, funcs)) - #: delete for various reasons + #: Delete for various reasons self.event_map = None if self.event_list: diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 637d9f3b5..c0e374a86 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -28,7 +28,7 @@ def parse_fileInfo(klass, url="", html=""): #@TODO: Remove in 0.4.10 def getInfo(urls): - # result = [ .. (name, size, status, url) .. ] + #: result = [ .. (name, size, status, url) .. ] pass @@ -62,14 +62,14 @@ class Hoster(Plugin): def __init__(self, pyfile): super(Hoster, self).__init__(pyfile.m.core) - #: engage wan reconnection + #: Engage wan reconnection self.want_reconnect = False - #: enable simultaneous processing of multiple downloads + #: Enable simultaneous processing of multiple downloads self.multi_dl = True self.limit_dl = 0 - #: chunk limit + #: Chunk limit self.chunk_limit = 1 self.resume_download = False @@ -77,13 +77,13 @@ class Hoster(Plugin): self.wait_until = 0 self.waiting = False - #: captcha reader instance + #: Captcha reader instance self.ocr = None - #: account handler instance, see :py:class:`Account` + #: Account handler instance, see :py:class:`Account` self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) - #: premium status + #: Premium status self.premium = False #: username/login @@ -97,37 +97,37 @@ class Hoster(Plugin): #: Browser instance, see `network.Browser` self.req = self.account.get_account_request(self.user) - self.chunk_limit = -1 #: chunk limit, -1 for unlimited + self.chunk_limit = -1 #: Chunk limit, -1 for unlimited - #: enables resume (will be ignored if server dont accept chunks) + #: Enables resume (will be ignored if server dont accept chunks) self.resume_download = True - #: premium status + #: Premium status self.premium = self.account.is_premium(self.user) else: self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) - #: associated pyfile instance, see `PyFile` + #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile - self.thread = None #: holds thread in future + self.thread = None #: Holds thread in future - #: location where the last call to download was saved + #: Location where the last call to download was saved self.last_download = "" - #: re match of the last call to `checkDownload` + #: Re match of the last call to `checkDownload` self.last_check = None - #: js engine, see `JsEngine` + #: Js engine, see `JsEngine` self.js = self.core.js - #: captcha task + #: Captcha task self.c_task = None - #: some plugins store html code here + #: Some plugins store html code here self.html = None - #: dict of the amount of retries already made + #: Dict of the amount of retries already made self.retries = {} self.init() @@ -392,7 +392,7 @@ class Hoster(Plugin): captchaManager.removeTask(task) - if task.error and has_plugin: #: ignore default error message since the user could use OCR + if task.error and has_plugin: #: Ignore default error message since the user could use OCR self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) elif task.error: self.fail(task.error) @@ -471,7 +471,7 @@ class Hoster(Plugin): except Exception, e: self.fail(e) - #: convert back to unicode + #: Convert back to unicode location = fs_decode(location) name = safe_filename(self.pyfile.name) @@ -555,8 +555,8 @@ class Hoster(Plugin): with open(lastDownload, "rb") as f: content = f.read(read_size) - #: produces encoding errors, better log to other file in the future? - #: self.log_debug("Content: %s" % content) + #: Produces encoding errors, better log to other file in the future? + # self.log_debug("Content: %s" % content) for name, rule in rules.iteritems(): if isinstance(rule, basestring): if rule in content: @@ -699,9 +699,9 @@ class Hoster(Plugin): for pyfile in self.core.files.cache.values(): if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: - if pyfile.status in (0, 12): #: finished or downloading + if pyfile.status in (0, 12): #: Finished or downloading self.skip(pyfile.pluginname) - elif pyfile.status in (5, 7) and starting: #: a download is waiting/starting and was appenrently started before + elif pyfile.status in (5, 7) and starting: #: A download is waiting/starting and was appenrently started before self.skip(pyfile.pluginname) download_folder = self.core.config.get("general", "download_folder") diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 7afe95705..4a43e89c7 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -207,7 +207,7 @@ class MultiHook(Hook): self.log_error(_("No %s loaded") % self.plugintype) return - #: inject plugin plugin + #: Inject plugin plugin self.log_debug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) for plugin in self.supported: @@ -223,7 +223,7 @@ class MultiHook(Hook): self.log_debug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) - #: create new regexp + #: Create new regexp regexp = r'.*(?P%s).*' % "|".join(x.replace('.', '\.') for x in plugins) if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and "://" in self.pluginclass.__pattern__: regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) @@ -252,7 +252,7 @@ class MultiHook(Hook): for plugin in self.supported: self.unload_plugin(plugin) - #: reset pattern + #: Reset pattern hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 880f8b570..9ddca0315 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -61,12 +61,12 @@ class OCR(Plugin): def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True, pagesegmode=None): - #: tmpTif = tempfile.NamedTemporaryFile(suffix=".tif") + # tmpTif = tempfile.NamedTemporaryFile(suffix=".tif") try: tmpTif = open(fs_join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") tmpTif.close() - #: tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") + # tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") tmpTxt = open(fs_join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") tmpTxt.close() @@ -88,7 +88,7 @@ class OCR(Plugin): tessparams.extend(["-psm", str(pagesegmode)]) if subset and (digits or lowercase or uppercase): - #: tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") + # tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") with open(fs_join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") as tmpSub: tmpSub.write("tessedit_char_whitelist ") @@ -177,12 +177,12 @@ class OCR(Plugin): except Exception: pass - #: not enough neighbors are dark pixels so mark this pixel - #: to be changed to white + #: Not enough neighbors are dark pixels so mark this pixel + #: To be changed to white if count < allowed: pixels[x, y] = 1 - #: second pass: this time set all 1's to 255 (white) + #: Second pass: this time set all 1's to 255 (white) for x in xrange(w): for y in xrange(h): if pixels[x, y] == 1: diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index dc5995e65..51f952b40 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -69,7 +69,7 @@ def parse_html_form(attr_str, html, input_names={}): inputs[name] = value if input_names: - #: check input attributes + #: Check input attributes for key, val in input_names.iteritems(): if key in inputs: if isinstance(val, basestring) and inputs[key] == val: @@ -78,16 +78,16 @@ def parse_html_form(attr_str, html, input_names={}): continue elif hasattr(val, "search") and re.match(val, inputs[key]): continue - break #: attibute value does not match + break #: Attibute value does not match else: - break #: attibute name does not match + break #: Attibute name does not match else: - return action, inputs #: passed attribute check + return action, inputs #: Passed attribute check else: - #: no attribute check + #: No attribute check return action, inputs - return {}, None #: no matching form found + return {}, None #: No matching form found #@TODO: Move to utils in 0.4.10 @@ -303,13 +303,13 @@ class Plugin(object): os.makedirs(os.path.join("tmp", self.__name__)) with open(framefile, "wb") as f: - del frame #: delete the frame or it wont be cleaned + del frame #: Delete the frame or it wont be cleaned f.write(res.encode('utf8')) except IOError, e: self.log_error(e) if just_header: - #: parse header + #: Parse header header = {"code": req.code} for line in res.splitlines(): line = line.strip() diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index d067468dd..c76ef43e2 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -72,7 +72,7 @@ class SevenZip(UnRar): p = self.call_cmd("l", "-slt", fs_encode(self.filename)) out, err = p.communicate() - #: check if output or error macthes the 'wrong password'-Regexp + #: Check if output or error macthes the 'wrong password'-Regexp if self.re_wrongpwd.search(out): raise PasswordError @@ -91,7 +91,7 @@ class SevenZip(UnRar): renice(p.pid, self.renice) - #: communicate and retrieve stderr + #: Communicate and retrieve stderr self._progress(p) err = p.stderr.read().strip() @@ -102,7 +102,7 @@ class SevenZip(UnRar): elif self.re_wrongcrc.search(err): raise CRCError(err) - else: #: raise error if anything is on stderr + else: #: Raise error if anything is on stderr raise ArchiveError(err) if p.returncode > 1: @@ -134,11 +134,11 @@ class SevenZip(UnRar): def call_cmd(self, command, *xargs, **kwargs): args = [] - # overwrite flag + #: Overwrite flag if self.overwrite: args.append("-y") - # set a password + #: Set a password if "password" in kwargs and kwargs['password']: args.append("-p%s" % kwargs['password']) else: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 93008f16d..08f13f900 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -124,7 +124,7 @@ class SimpleHoster(Hoster): online = True if info['status'] is 2 else False try: - info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here + info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: Pattern groups will be saved here except Exception: info['pattern'] = {} diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 50c421e8d..abd94b4df 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -68,7 +68,7 @@ class UnRar(Extractor): cls.__name__ = "RAR" cls.REPAIR = True - except OSError: #: fallback to unrar + except OSError: #: Fallback to unrar p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() @@ -105,7 +105,7 @@ class UnRar(Extractor): if self.re_wrongcrc.search(err): raise CRCError(err) - #: output only used to check if passworded files are present + #: Output only used to check if passworded files are present for attr in self.re_filelist.findall(out): if attr[0].startswith("*"): raise PasswordError @@ -114,7 +114,7 @@ class UnRar(Extractor): def repair(self): p = self.call_cmd("rc", fs_encode(self.filename)) - #: communicate and retrieve stderr + #: Communicate and retrieve stderr self._progress(p) err = p.stderr.read().strip() if err or p.returncode: @@ -126,17 +126,17 @@ class UnRar(Extractor): s = "" while True: c = process.stdout.read(1) - #: quit loop on eof + #: Quit loop on eof if not c: break - #: reading a percentage sign -> set progress and restart + #: Reading a percentage sign -> set progress and restart if c == '%': self.notify_progress(int(s)) s = "" - #: not reading a digit -> therefore restart + #: Not reading a digit -> therefore restart elif c not in digits: s = "" - #: add digit to progressstring + #: Add digit to progressstring else: s += c @@ -148,7 +148,7 @@ class UnRar(Extractor): renice(p.pid, self.renice) - #: communicate and retrieve stderr + #: Communicate and retrieve stderr self._progress(p) err = p.stderr.read().strip() @@ -159,7 +159,7 @@ class UnRar(Extractor): elif self.re_wrongcrc.search(err): raise CRCError(err) - else: #: raise error if anything is on stderr + else: #: Raise error if anything is on stderr raise ArchiveError(err) if p.returncode: @@ -171,7 +171,7 @@ class UnRar(Extractor): def get_delete_files(self): dir, name = os.path.split(self.filename) - #: actually extracted file + #: Actually extracted file files = [self.filename] #: eventually Multipart Files @@ -190,7 +190,7 @@ class UnRar(Extractor): if "Cannot open" in err: raise ArchiveError(_("Cannot open file")) - if err.strip(): #: only log error at this point + if err.strip(): #: Only log error at this point self.manager.log_error(err.strip()) result = set() @@ -211,7 +211,7 @@ class UnRar(Extractor): def call_cmd(self, command, *xargs, **kwargs): args = [] - #: overwrite flag + #: Overwrite flag if self.overwrite: args.append("-o+") else: @@ -222,10 +222,10 @@ class UnRar(Extractor): for word in self.excludefiles: args.append("-x'%s'" % word.strip()) - #: assume yes on all queries + #: Assume yes on all queries args.append("-y") - #: set a password + #: Set a password if "password" in kwargs and kwargs['password']: args.append("-p%s" % kwargs['password']) else: diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 4005b8e21..7ae6d816e 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -30,10 +30,10 @@ class XFSAccount(Account): VALID_UNTIL_PATTERN = r'Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})' TRAFFIC_LEFT_PATTERN = r'Traffic available today:.*?\s*(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' - TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found + TRAFFIC_LEFT_UNIT = "MB" #: Used only if no group was found LEECH_TRAFFIC_PATTERN = r'Leech Traffic left:.*?(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' - LEECH_TRAFFIC_UNIT = "MB" #: used only if no group was found + LEECH_TRAFFIC_UNIT = "MB" #: Used only if no group was found LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<' @@ -87,7 +87,7 @@ class XFSAccount(Account): trafficleft = -1 else: premium = False - validuntil = None #: registered account type (not premium) + validuntil = None #: Registered account type (not premium) else: self.log_debug("VALID_UNTIL_PATTERN not found") diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 9f8728512..d4810f8b7 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -47,7 +47,7 @@ class XFSHoster(SimpleHoster): SOLVEMEDIA_PATTERN = None FORM_PATTERN = None - FORM_INPUTS_MAP = None #: dict passed as input_names to parse_html_form + FORM_INPUTS_MAP = None #: Dict passed as input_names to parse_html_form def setup(self): @@ -123,7 +123,7 @@ class XFSHoster(SimpleHoster): if not self.account: self.fail(_("Only registered or premium users can use url leech feature")) - # only tested with easybytez.com + #: Only tested with easybytez.com self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) action, inputs = self.parse_html_form() @@ -137,7 +137,7 @@ class XFSHoster(SimpleHoster): self.log_debug(action, inputs) - self.req.setOption("timeout", 600) #: wait for file to upload to easybytez.com + self.req.setOption("timeout", 600) #: Wait for file to upload to easybytez.com self.html = self.load(action, post=inputs) @@ -163,7 +163,7 @@ class XFSHoster(SimpleHoster): else: self.fail(stmsg) - # get easybytez.com link for uploaded file + #: Get easybytez.com link for uploaded file m = re.search(self.LINK_LEECH_PATTERN, self.html) if m is None: self.error(_("LINK_LEECH_PATTERN not found")) -- cgit v1.2.3 From a0e2459f2d1506912ac4a5d0c330c8dae01a4768 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 00:28:18 +0200 Subject: Improve fixurl method --- module/plugins/internal/Hoster.py | 12 +++++------- module/plugins/internal/Plugin.py | 11 ++++++----- module/plugins/internal/SimpleHoster.py | 5 ++--- 3 files changed, 13 insertions(+), 15 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index c0e374a86..e7df336b6 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -6,7 +6,6 @@ import inspect import os import random import time -import urllib import urlparse if os.name != "nt": @@ -14,7 +13,7 @@ if os.name != "nt": import pwd from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip - chunks, replace_patterns, seconds_to_midnight, + chunks, fixurl as _fixurl, replace_patterns, seconds_to_midnight, set_cookies, parse_html_form, parse_html_tag_attr_value, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join @@ -135,7 +134,7 @@ class Hoster(Plugin): @classmethod def get_info(cls, url="", html=""): - url = urllib.unquote(url) + url = _fixurl(url) url_p = urlparse.urlparse(url) return {'name' : (url_p.path.split('/')[-1] or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] @@ -412,12 +411,11 @@ class Hoster(Plugin): def fixurl(self, url): - url_p = urlparse.urlparse(self.pyfile.url) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - - url = super(Hoster, self).fixurl(url) + url = _fixurl(url) if not urlparse.urlparse(url).scheme: + url_p = urlparse.urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) url = urlparse.urljoin(baseurl, url) return url diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 51f952b40..93dd2ecbe 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -11,6 +11,11 @@ from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as from module.utils import fs_encode, fs_decode, html_unescape, save_join as fs_join +#@TODO: Move to utils in 0.4.10 +def fixurl(url): + return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip() + + #@TODO: Move to utils in 0.4.10 def timestamp(): return int(time.time() * 1000) @@ -256,10 +261,6 @@ class Plugin(object): raise Fail(msg) - def fixurl(self, url): - return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip() - - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=True, multipart=True, req=None): """ Load content at url and returns it @@ -276,7 +277,7 @@ class Plugin(object): if hasattr(self, 'pyfile') and self.pyfile.abort: self.abort() - url = self.fixurl(url) + url = fixurl(url) if not url or not isinstance(url, basestring): self.fail(_("No url given")) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 08f13f900..5bf2f0417 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -7,14 +7,13 @@ import mimetypes import os import re import time -import urllib import urlparse from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, Retry, replace_patterns, set_cookies +from module.plugins.internal.Plugin import Fail, Retry, fixurl, replace_patterns, set_cookies from module.utils import fixup, fs_encode, parseFileSize as parse_size @@ -176,7 +175,7 @@ class SimpleHoster(Hoster): info['status'] = 2 if 'N' in info['pattern']: - info['name'] = replace_patterns(urllib.unquote(info['pattern']['N'].strip()), + info['name'] = replace_patterns(fixurl(info['pattern']['N']), cls.NAME_REPLACEMENTS) if 'S' in info['pattern']: -- cgit v1.2.3 From 502517f37c7540b0bddb092e69386d9d6f08800c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 09:42:34 +0200 Subject: Fix addons --- module/plugins/internal/Account.py | 3 + module/plugins/internal/Addon.py | 242 +++++++++++++++++++++++++++++++ module/plugins/internal/Captcha.py | 9 ++ module/plugins/internal/Extractor.py | 2 + module/plugins/internal/Hook.py | 238 +----------------------------- module/plugins/internal/MultiHook.py | 4 +- module/plugins/internal/MultiHoster.py | 5 +- module/plugins/internal/OCR.py | 8 + module/plugins/internal/Plugin.py | 4 +- module/plugins/internal/SimpleCrypter.py | 1 - module/plugins/internal/SimpleHoster.py | 3 +- 11 files changed, 274 insertions(+), 245 deletions(-) create mode 100644 module/plugins/internal/Addon.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 534f5607d..7bf8964ad 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -41,6 +41,9 @@ class Account(Plugin): def init(self): + """ + Initialize additional data structures + """ pass diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py new file mode 100644 index 000000000..f3da5272f --- /dev/null +++ b/module/plugins/internal/Addon.py @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- + +import traceback + +from module.plugins.internal.Plugin import Plugin + + +class Expose(object): + """ + Used for decoration to declare rpc services + """ + def __new__(cls, f, *args, **kwargs): + hookManager.addRPC(f.__module__, f.func_name, f.func_doc) + return f + + +def threaded(fn): + + def run(*args, **kwargs): + hookManager.startThread(fn, *args, **kwargs) + + return run + + +class Addon(Plugin): + __name__ = "Addon" + __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 + __version__ = "0.01" + + __config__ = [] #: [("name", "type", "desc", "default")] + __threaded__ = [] #@TODO: Remove in 0.4.10 + + __description__ = """Base addon plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def __init__(self, core, manager): + super(Addon, self).__init__(core) + + #: `HookManager` + self.manager = manager + + #: Automatically register event listeners for functions, attribute will be deleted dont use it yourself + self.event_map = {} + + #: Deprecated alternative to event_map + #: List of events the plugin can handle, name the functions exactly like eventname. + self.event_list = [] #@NOTE: dont make duplicate entries in event_map + + #: Callback of periodical job task, used by HookManager + self.cb = None + self.interval = 60 + + self.init() + self.init_events() + + + def init_events(self): + if self.event_map: + for event, funcs in self.event_map.iteritems(): + if type(funcs) in (list, tuple): + for f in funcs: + self.manager.addEvent(event, getattr(self, f)) + else: + self.manager.addEvent(event, getattr(self, funcs)) + + #: Delete for various reasons + self.event_map = None + + if self.event_list: + self.log_debug("Deprecated method `event_list`, use `event_map` instead") + + for f in self.event_list: + self.manager.addEvent(f, getattr(self, f)) + + self.event_list = None + + + def init_periodical(self, delay=0, threaded=False): + self.cb = self.core.scheduler.addJob(max(0, delay), self._periodical, [threaded], threaded=threaded) + + + #: Deprecated method, use `init_periodical` instead + def initPeriodical(self, *args, **kwargs): + return self.init_periodical(*args, **kwargs) + + + def _periodical(self, threaded): + if self.interval < 0: + self.cb = None + return + + try: + self.periodical() + + except Exception, e: + self.log_error(_("Error executing addon: %s") % e) + if self.core.debug: + traceback.print_exc() + + self.cb = self.core.scheduler.addJob(self.interval, self._periodical, [threaded], threaded=threaded) + + + def periodical(self): + pass + + + def __repr__(self): + return "" % self.__name__ + + + def is_activated(self): + """ + Checks if addon is activated + """ + return self.get_config("activated") + + + #: Deprecated method, use `is_activated` instead + def isActivated(self, *args, **kwargs): + return self.is_activated(*args, **kwargs) + + + def deactivate(self): + """ + Called when addon was deactivated + """ + pass + + + #: Deprecated method, use `deactivate` instead + def unload(self, *args, **kwargs): + return self.deactivate(*args, **kwargs) + + + def activate(self): + """ + Called when addon was activated + """ + pass + + + #: Deprecated method, use `activate` instead + def coreReady(self, *args, **kwargs): + return self.activate(*args, **kwargs) + + + def exit(self): + """ + Called by core.shutdown just before pyLoad exit + """ + pass + + + #: Deprecated method, use `exit` instead + def coreExiting(self, *args, **kwargs): + return self.exit(*args, **kwargs) + + + def download_preparing(self, pyfile): + pass + + + #: Deprecated method, use `download_preparing` instead + def downloadPreparing(self, *args, **kwargs): + return self.download_preparing(*args, **kwargs) + + + def download_finished(self, pyfile): + pass + + + #: Deprecated method, use `download_finished` instead + def downloadFinished(self, *args, **kwargs): + return self.download_finished(*args, **kwargs) + + + def download_failed(self, pyfile): + pass + + + #: Deprecated method, use `download_failed` instead + def downloadFailed(self, *args, **kwargs): + return self.download_failed(*args, **kwargs) + + + def package_finished(self, pypack): + pass + + + #: Deprecated method, use `package_finished` instead + def packageFinished(self, *args, **kwargs): + return self.package_finished(*args, **kwargs) + + + def before_reconnect(self, ip): + pass + + + #: Deprecated method, use `before_reconnect` instead + def beforeReconnecting(self, *args, **kwargs): + return self.before_reconnect(*args, **kwargs) + + + def after_reconnect(self, ip, oldip): + pass + + + #: Deprecated method, use `after_reconnect` instead + def afterReconnecting(self, ip): + return self.after_reconnect(ip, None) + + + def captcha_task(self, task): + """ + New captcha task for the plugin, it MUST set the handler and timeout or will be ignored + """ + pass + + + #: Deprecated method, use `captcha_task` instead + def newCaptchaTask(self, *args, **kwargs): + return self.captcha_task(*args, **kwargs) + + + def captcha_correct(self, task): + pass + + + #: Deprecated method, use `captcha_correct` instead + def captchaCorrect(self, *args, **kwargs): + return self.captcha_correct(*args, **kwargs) + + + def captcha_invalid(self, task): + pass + + + #: Deprecated method, use `captcha_invalid` instead + def captchaInvalid(self, *args, **kwargs): + return self.captcha_invalid(*args, **kwargs) diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 68d5d3bb8..1309b0d6b 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -19,6 +19,15 @@ class Captcha(Plugin): self.plugin = plugin self.key = None #: Last key detected + self.init() + + + def init(self): + """ + Initialize additional data structures + """ + pass + #@TODO: Recheck in 0.4.10 def retrieve_key(self, html): diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 60d8b3b3b..929ef5378 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -99,6 +99,8 @@ class Extractor(Plugin): pyfile = self.manager.core.files.getFile(fid) if fid else None self.notify_progress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None + self.init() + def init(self): """ diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 162c9ecbf..a17037a11 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -1,34 +1,14 @@ # -*- coding: utf-8 -*- -import traceback +from module.plugins.internal.Addon import Addon -from module.plugins.internal.Plugin import Plugin - -class Expose(object): - """ - Used for decoration to declare rpc services - """ - def __new__(cls, f, *args, **kwargs): - hookManager.addRPC(f.__module__, f.func_name, f.func_doc) - return f - - -def threaded(fn): - - def run(*args, **kwargs): - hookManager.startThread(fn, *args, **kwargs) - - return run - - -class Hook(Plugin): +class Hook(Addon): __name__ = "Hook" __type__ = "hook" - __version__ = "0.09" + __version__ = "0.10" __config__ = [] #: [("name", "type", "desc", "default")] - __threaded__ = [] #@TODO: Remove in 0.4.10 __description__ = """Base hook plugin""" __license__ = "GPLv3" @@ -38,215 +18,5 @@ class Hook(Plugin): def __init__(self, core, manager): - super(Hook, self).__init__(core) - - #: `HookManager` - self.manager = manager - - #: Automatically register event listeners for functions, attribute will be deleted dont use it yourself - self.event_map = {} - - #: Deprecated alternative to event_map - #: List of events the plugin can handle, name the functions exactly like eventname. - self.event_list = [] #@NOTE: dont make duplicate entries in event_map - - #: Callback of periodical job task, used by HookManager - self.cb = None - self.interval = 60 - - self.setup() - self.init_events() + super(Hook, self).__init__(core, manager) self.init_periodical(10) - - - def init_events(self): - if self.event_map: - for event, funcs in self.event_map.iteritems(): - if type(funcs) in (list, tuple): - for f in funcs: - self.manager.addEvent(event, getattr(self, f)) - else: - self.manager.addEvent(event, getattr(self, funcs)) - - #: Delete for various reasons - self.event_map = None - - if self.event_list: - self.log_debug("Deprecated method `event_list`, use `event_map` instead") - - for f in self.event_list: - self.manager.addEvent(f, getattr(self, f)) - - self.event_list = None - - - def init_periodical(self, delay=0, threaded=False): - self.cb = self.core.scheduler.addJob(max(0, delay), self._periodical, [threaded], threaded=threaded) - - - #: Deprecated method, use `init_periodical` instead - def initPeriodical(self, *args, **kwargs): - return self.init_periodical(*args, **kwargs) - - - def _periodical(self, threaded): - if self.interval < 0: - self.cb = None - return - - try: - self.periodical() - - except Exception, e: - self.log_error(_("Error executing hook: %s") % e) - if self.core.debug: - traceback.print_exc() - - self.cb = self.core.scheduler.addJob(self.interval, self._periodical, [threaded], threaded=threaded) - - - def periodical(self): - pass - - - def __repr__(self): - return "" % self.__name__ - - - def setup(self): - """ - More init stuff if needed - """ - pass - - - def is_activated(self): - """ - Checks if hook is activated - """ - return self.get_config("activated") - - - #: Deprecated method, use `is_activated` instead - def isActivated(self, *args, **kwargs): - return self.is_activated(*args, **kwargs) - - - def deactivate(self): - """ - Called when hook was deactivated - """ - pass - - - #: Deprecated method, use `deactivate` instead - def unload(self, *args, **kwargs): - return self.deactivate(*args, **kwargs) - - - def activate(self): - """ - Called when hook was activated - """ - pass - - - #: Deprecated method, use `activate` instead - def coreReady(self, *args, **kwargs): - return self.activate(*args, **kwargs) - - - def exit(self): - """ - Called by core.shutdown just before pyLoad exit - """ - pass - - - #: Deprecated method, use `exit` instead - def coreExiting(self, *args, **kwargs): - return self.exit(*args, **kwargs) - - - def download_preparing(self, pyfile): - pass - - - #: Deprecated method, use `download_preparing` instead - def downloadPreparing(self, *args, **kwargs): - return self.download_preparing(*args, **kwargs) - - - def download_finished(self, pyfile): - pass - - - #: Deprecated method, use `download_finished` instead - def downloadFinished(self, *args, **kwargs): - return self.download_finished(*args, **kwargs) - - - def download_failed(self, pyfile): - pass - - - #: Deprecated method, use `download_failed` instead - def downloadFailed(self, *args, **kwargs): - return self.download_failed(*args, **kwargs) - - - def package_finished(self, pypack): - pass - - - #: Deprecated method, use `package_finished` instead - def packageFinished(self, *args, **kwargs): - return self.package_finished(*args, **kwargs) - - - def before_reconnect(self, ip): - pass - - - #: Deprecated method, use `before_reconnect` instead - def beforeReconnecting(self, *args, **kwargs): - return self.before_reconnect(*args, **kwargs) - - - def after_reconnect(self, ip, oldip): - pass - - - #: Deprecated method, use `after_reconnect` instead - def afterReconnecting(self, ip): - return self.after_reconnect(ip, None) - - - def captcha_task(self, task): - """ - New captcha task for the plugin, it MUST set the handler and timeout or will be ignored - """ - pass - - - #: Deprecated method, use `captcha_task` instead - def newCaptchaTask(self, *args, **kwargs): - return self.captcha_task(*args, **kwargs) - - - def captcha_correct(self, task): - pass - - - #: Deprecated method, use `captcha_correct` instead - def captchaCorrect(self, *args, **kwargs): - return self.captcha_correct(*args, **kwargs) - - - def captcha_invalid(self, task): - pass - - - #: Deprecated method, use `captcha_invalid` instead - def captchaInvalid(self, *args, **kwargs): - return self.captcha_invalid(*args, **kwargs) diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 4a43e89c7..127d83ab5 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -11,7 +11,7 @@ from module.utils import decode, remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.50" + __version__ = "0.51" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), @@ -54,7 +54,7 @@ class MultiHook(Hook): (r'^0' , "zero" )] - def setup(self): + def init(self): self.plugins = [] self.supported = [] self.new_supported = [] diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 363d46739..7bba2f6d3 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -30,9 +30,8 @@ class MultiHoster(SimpleHoster): def prepare(self): - self.info = {} - self.html = "" - self.link = "" #@TODO: Move to Hoster in 0.4.10 + self.html = "" + self.link = "" #@TODO: Move to Hoster in 0.4.10 self.direct_dl = False #@TODO: Move to Hoster in 0.4.10 if not self.get_config('use_premium', True): diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 9ddca0315..387db1d13 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -29,6 +29,14 @@ class OCR(Plugin): def __init__(self): self.logger = logging.getLogger("log") + self.init() + + + def init(self): + """ + Initialize additional data structures + """ + pass def load_image(self, image): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 93dd2ecbe..d0f75a8a3 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -122,9 +122,7 @@ class Plugin(object): def __init__(self, core): self.core = core - - #: Provide information in dict here - self.info = {} + self.info = {} #: Provide information in dict here def _log(self, level, args): diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 82f5ef332..cd447db6b 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -66,7 +66,6 @@ class SimpleCrypter(Crypter, SimpleHoster): def prepare(self): self.pyfile.error = "" #@TODO: Remove in 0.4.10 - self.info = {} self.html = "" #@TODO: Recheck in 0.4.10 self.link = "" #@TODO: Recheck in 0.4.10 self.links = [] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 5bf2f0417..911a7d411 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -204,10 +204,9 @@ class SimpleHoster(Hoster): def prepare(self): self.pyfile.error = "" #@TODO: Remove in 0.4.10 - self.info = {} self.html = "" #@TODO: Recheck in 0.4.10 self.link = "" #@TODO: Recheck in 0.4.10 - self.direct_dl = False + self.direct_dl = False self.multihost = False if not self.get_config('use_premium', True): -- cgit v1.2.3 From ff9383bfe06d14d23bc0ed6af79aa8967965d078 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 10:59:52 +0200 Subject: Code cosmetics (3) --- module/plugins/internal/Account.py | 34 ++++++++++++++++----------------- module/plugins/internal/AdYouLike.py | 20 +++++++++---------- module/plugins/internal/Crypter.py | 8 ++++---- module/plugins/internal/Extractor.py | 2 +- module/plugins/internal/Hoster.py | 4 ++-- module/plugins/internal/OCR.py | 2 +- module/plugins/internal/Plugin.py | 10 +++++++++- module/plugins/internal/SimpleHoster.py | 6 +++--- 8 files changed, 47 insertions(+), 39 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 7bf8964ad..db0002d09 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -69,14 +69,14 @@ class Account(Plugin): except WrongPassword: self.log_warning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user, - "msg": _("Wrong Password")}) + _("Could not login with account %(user)s | %(msg)s") % {'user': user, + 'msg': _("Wrong Password")}) success = data['valid'] = False except Exception, e: self.log_warning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user, - "msg": e}) + _("Could not login with account %(user)s | %(msg)s") % {'user': user, + 'msg': e}) success = data['valid'] = False if self.core.debug: traceback.print_exc() @@ -126,7 +126,7 @@ class Account(Plugin): self.accounts[user]['options'].update(options) return self.accounts[user]['options'] != before else: - self.accounts[user] = {"password": password, "options": options, "valid": True} + self.accounts[user] = {'password': password, 'options': options, 'valid': True} self._login(user, self.accounts[user]) return True @@ -211,16 +211,16 @@ class Account(Plugin): :param req: `Request` instance :return: """ - return {"validuntil" : None, #: -1 for unlimited - "login" : name, - # "password" : self.accounts[name]['password'], #: Commented due security reason - "options" : self.accounts[name]['options'], - "valid" : self.accounts[name]['valid'], - "trafficleft": None, #: In bytes, -1 for unlimited - "maxtraffic" : None, - "premium" : None, - "timestamp" : 0, #: Time this info was retrieved - "type" : self.__name__} + return {'validuntil' : None, #: -1 for unlimited + 'login' : name, + # 'password' : self.accounts[name]['password'], #: Commented due security reason + 'options' : self.accounts[name]['options'], + 'valid' : self.accounts[name]['valid'], + 'trafficleft': None, #: In bytes, -1 for unlimited + 'maxtraffic' : None, + 'premium' : None, + 'timestamp' : 0, #: Time this info was retrieved + 'type' : self.__name__} def get_all_accounts(self, force=False): @@ -306,7 +306,7 @@ class Account(Plugin): if user in self.infos: self.log_warning(_("Account %s has not enough traffic, checking again in 30min") % user) - self.infos[user].update({"trafficleft": 0}) + self.infos[user].update({'trafficleft': 0}) self.schedule_refresh(user, 30 * 60) @@ -314,7 +314,7 @@ class Account(Plugin): if user in self.infos: self.log_warning(_("Account %s is expired, checking again in 1h") % user) - self.infos[user].update({"validuntil": time.time() - 1}) + self.infos[user].update({'validuntil': time.time() - 1}) self.schedule_refresh(user, 60 * 60) diff --git a/module/plugins/internal/AdYouLike.py b/module/plugins/internal/AdYouLike.py index 64ce6e7fa..10f281494 100644 --- a/module/plugins/internal/AdYouLike.py +++ b/module/plugins/internal/AdYouLike.py @@ -37,8 +37,8 @@ class AdYouLike(Captcha): def challenge(self, key=None, html=None): ayl, callback = key or self.retrieve_key(html) - #: {"adyoulike":{"key":"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, - #: "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} + #: {'adyoulike':{'key':"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, + #: 'all':{'element_id':"ayl_private_cap_92300",'lang':"fr",'env':"prod"}} ayl = json_loads(ayl) html = self.plugin.load("http://api-ayl.appspot.com/challenge", @@ -58,14 +58,14 @@ class AdYouLike(Captcha): def result(self, server, challenge): #: Adyoulike.g._jsonp_5579316662423138 - #: ({"translations":{"fr":{"instructions_visual":"Recopiez « Soonnight » ci-dessous :"}}, - #: "site_under":true,"clickable":true,"pixels":{"VIDEO_050":[],"DISPLAY":[],"VIDEO_000":[],"VIDEO_100":[], - #: "VIDEO_025":[],"VIDEO_075":[]},"medium_type":"image/adyoulike", - #: "iframes":{"big":""},"shares":{},"id":256, - #: "token":"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1","formats":{"small":{"y":300,"x":0,"w":300,"h":60}, - #: "big":{"y":0,"x":0,"w":300,"h":250},"hover":{"y":440,"x":0,"w":300,"h":60}}, - #: "tid":"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) + #: ({'translations':{'fr':{'instructions_visual':"Recopiez « Soonnight » ci-dessous :"}}, + #: 'site_under':true,'clickable':true,'pixels':{'VIDEO_050':[],'DISPLAY':[],'VIDEO_000':[],'VIDEO_100':[], + #: 'VIDEO_025':[],'VIDEO_075':[]},'medium_type':"image/adyoulike", + #: 'iframes':{'big':""},'shares':{},'id':256, + #: 'token':"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1",'formats':{'small':{'y':300,'x':0,'w':300,'h':60}, + #: 'big':{'y':0,'x':0,'w':300,'h':250},'hover':{'y':440,'x':0,'w':300,'h':60}}, + #: 'tid':"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) if isinstance(server, basestring): server = json_loads(server) diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index dbecb0cf8..9ad16ece8 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -80,22 +80,22 @@ class Crypter(Hoster): pid = self.core.api.addPackage(name, self.fixurl(links), package_queue) if package_password: - self.core.api.setPackageData(pid, {"password": package_password}) + self.core.api.setPackageData(pid, {'password': package_password}) #: Workaround to do not break API addPackage method - setFolder = lambda x: self.core.api.setPackageData(pid, {"folder": x or ""}) + setFolder = lambda x: self.core.api.setPackageData(pid, {'folder': x or ""}) if use_subfolder: if not subfolder_per_package: setFolder(package_folder) - self.log_debug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) + self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) elif not folder_per_package or name != folder: if not folder: folder = urlparse.urlparse(name).path.split("/")[-1] setFolder(safe_filename(folder)) - self.log_debug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) + self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) elif folder_per_package: setFolder(None) diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 929ef5378..39220508e 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -67,7 +67,7 @@ class Extractor(Plugin): for fname, id, fout in files_ids: if cls.isArchive(fname): - pname = re.sub(cls.re_multipart, '', fname) if cls.isMultipart(fname) else os.path.splitext(fname)[0] + pname = re.sub(cls.re_multipart, "", fname) if cls.isMultipart(fname) else os.path.splitext(fname)[0] if pname not in processed: processed.append(pname) targets.append((fname, id, fout)) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index e7df336b6..497df00f3 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -488,7 +488,7 @@ class Hoster(Plugin): newname = urlparse.urlparse(newname).path.split('/')[-1] if disposition and newname != name: - self.log_info(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) + self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) self.pyfile.name = newname filename = os.path.join(location, newname) @@ -595,7 +595,7 @@ class Hoster(Plugin): req.close() - header = {"code": req.code} + header = {'code': req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 387db1d13..a00f53947 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -42,7 +42,7 @@ class OCR(Plugin): def load_image(self, image): self.image = Image.open(image) self.pixels = self.image.load() - self.result_captcha = '' + self.result_captcha = "" def deactivate(self): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index d0f75a8a3..7dec6344e 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -123,6 +123,14 @@ class Plugin(object): def __init__(self, core): self.core = core self.info = {} #: Provide information in dict here + self.init() + + + def init(self): + """ + Initialize the plugin (in addition to `__init__`) + """ + pass def _log(self, level, args): @@ -309,7 +317,7 @@ class Plugin(object): if just_header: #: Parse header - header = {"code": req.code} + header = {'code': req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 911a7d411..40489d404 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -361,7 +361,7 @@ class SimpleHoster(Hoster): if re.search('da(il)?y|today', errmsg, re.I): wait_time = seconds_to_midnight(gmt=2) else: - wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1, "": 1}[u.lower()] for v, u in + wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) self.want_reconnect = wait_time > 300 @@ -385,7 +385,7 @@ class SimpleHoster(Hoster): if re.search("da(il)?y|today", errmsg): wait_time = seconds_to_midnight(gmt=2) else: - wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1, "": 1}[u.lower()] for v, u in + wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) self.want_reconnect = wait_time > 300 @@ -430,7 +430,7 @@ class SimpleHoster(Hoster): except Exception: waitmsg = m.group(0).strip() - wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1, "": 1}[u.lower()] for v, u in + wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec|)', waitmsg, re.I)) self.wait(wait_time, wait_time > 300) -- cgit v1.2.3 From 56389e28ba5d2f5658278bc7f486d73be747f135 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 11:44:49 +0200 Subject: Rename self.core to self.pyload (plugins only) --- module/plugins/internal/Account.py | 13 ++++--- module/plugins/internal/Addon.py | 9 +++-- module/plugins/internal/Captcha.py | 3 +- module/plugins/internal/Container.py | 2 +- module/plugins/internal/Crypter.py | 10 ++--- module/plugins/internal/Extractor.py | 6 ++- module/plugins/internal/Hoster.py | 55 +++++++++++++------------- module/plugins/internal/MultiHook.py | 22 +++++------ module/plugins/internal/MultiHoster.py | 4 +- module/plugins/internal/OCR.py | 3 ++ module/plugins/internal/Plugin.py | 66 +++++++------------------------- module/plugins/internal/SevenZip.py | 2 +- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 8 ++-- module/plugins/internal/UnRar.py | 4 +- module/plugins/internal/XFSCrypter.py | 2 +- module/plugins/internal/XFSHoster.py | 2 +- 17 files changed, 91 insertions(+), 122 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index db0002d09..72981a108 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -24,7 +24,8 @@ class Account(Plugin): def __init__(self, manager, accounts): - super(Account, self).__init__(manager.core) + self.pyload = manager.core + self.info = {} #: Provide information in dict here self.manager = manager self.accounts = {} @@ -78,7 +79,7 @@ class Account(Plugin): _("Could not login with account %(user)s | %(msg)s") % {'user': user, 'msg': e}) success = data['valid'] = False - if self.core.debug: + if self.pyload.debug: traceback.print_exc() else: @@ -175,7 +176,7 @@ class Account(Plugin): infos = super(self.__class__, self).load_account_info(name, self.req) infos['error'] = str(e) - if self.core.debug: + if self.pyload.debug: traceback.print_exc() finally: @@ -236,7 +237,7 @@ class Account(Plugin): if not user: user, data = self.select_account() - return self.core.requestFactory.getRequest(self.__name__, user) + return self.pyload.requestFactory.getRequest(self.__name__, user) def get_account_cookies(self, user=None): @@ -245,7 +246,7 @@ class Account(Plugin): if not user: return None - cj = self.core.requestFactory.getCookieJar(self.__name__, user) + cj = self.pyload.requestFactory.getCookieJar(self.__name__, user) return cj @@ -323,7 +324,7 @@ class Account(Plugin): Add task to refresh account info to sheduler """ self.log_debug("Scheduled Account refresh for %s in %s seconds." % (user, time)) - self.core.scheduler.addJob(time, self.get_account_info, [user, force]) + self.pyload.scheduler.addJob(time, self.get_account_info, [user, force]) #: Deprecated method, use `schedule_refresh` instead diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index f3da5272f..a6bf685b5 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -36,7 +36,8 @@ class Addon(Plugin): def __init__(self, core, manager): - super(Addon, self).__init__(core) + self.pyload = core + self.info = {} #: Provide information in dict here #: `HookManager` self.manager = manager @@ -78,7 +79,7 @@ class Addon(Plugin): def init_periodical(self, delay=0, threaded=False): - self.cb = self.core.scheduler.addJob(max(0, delay), self._periodical, [threaded], threaded=threaded) + self.cb = self.pyload.scheduler.addJob(max(0, delay), self._periodical, [threaded], threaded=threaded) #: Deprecated method, use `init_periodical` instead @@ -96,10 +97,10 @@ class Addon(Plugin): except Exception, e: self.log_error(_("Error executing addon: %s") % e) - if self.core.debug: + if self.pyload.debug: traceback.print_exc() - self.cb = self.core.scheduler.addJob(self.interval, self._periodical, [threaded], threaded=threaded) + self.cb = self.pyload.scheduler.addJob(self.interval, self._periodical, [threaded], threaded=threaded) def periodical(self): diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 1309b0d6b..6ca754a06 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -14,7 +14,8 @@ class Captcha(Plugin): def __init__(self, plugin): - super(Captcha, self).__init__(plugin.core) + self.pyload = plugin.core + self.info = {} #: Provide information in dict here self.plugin = plugin self.key = None #: Last key detected diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 91b5a6f76..2c092f98a 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -45,7 +45,7 @@ class Container(Crypter): if self.pyfile.url.startswith("http"): self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] content = self.load(self.pyfile.url) - self.pyfile.url = fs_join(self.core.config.get("general", "download_folder"), self.pyfile.name) + self.pyfile.url = fs_join(self.pyload.config.get("general", "download_folder"), self.pyfile.name) try: with open(self.pyfile.url, "wb") as f: f.write(content) diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 9ad16ece8..653469079 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -56,7 +56,7 @@ class Crypter(Hoster): """ Generate new packages from self.urls """ - packages = [(name, links, None) for name, links in self.core.api.generatePackages(self.urls).iteritems()] + packages = [(name, links, None) for name, links in self.pyload.api.generatePackages(self.urls).iteritems()] self.packages.extend(packages) @@ -68,7 +68,7 @@ class Crypter(Hoster): package_password = self.pyfile.package().password package_queue = self.pyfile.package().queue - folder_per_package = self.core.config.get("general", "folder_per_package") + folder_per_package = self.pyload.config.get("general", "folder_per_package") use_subfolder = self.get_config('use_subfolder', folder_per_package) subfolder_per_package = self.get_config('subfolder_per_package', True) @@ -77,13 +77,13 @@ class Crypter(Hoster): "%d links" % len(links), "Saved to folder: %s" % folder if folder else "Saved to download folder") - pid = self.core.api.addPackage(name, self.fixurl(links), package_queue) + pid = self.pyload.api.addPackage(name, self.fixurl(links), package_queue) if package_password: - self.core.api.setPackageData(pid, {'password': package_password}) + self.pyload.api.setPackageData(pid, {'password': package_password}) #: Workaround to do not break API addPackage method - setFolder = lambda x: self.core.api.setPackageData(pid, {'folder': x or ""}) + setFolder = lambda x: self.pyload.api.setPackageData(pid, {'folder': x or ""}) if use_subfolder: if not subfolder_per_package: diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 39220508e..97c2c63b1 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -85,7 +85,9 @@ class Extractor(Plugin): """ Initialize extractor for specific file """ - self.manager = manager + self.pyload = manager.core + self.info = {} #: Provide information in dict here + self.filename = filename self.out = out self.fullpath = fullpath @@ -96,7 +98,7 @@ class Extractor(Plugin): self.keepbroken = keepbroken self.files = [] #: Store extracted files here - pyfile = self.manager.core.files.getFile(fid) if fid else None + pyfile = self.pyload.files.getFile(fid) if fid else None self.notify_progress = lambda x: pyfile.setProgress(x) if pyfile else lambda x: None self.init() diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 497df00f3..f5566fb6e 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -59,7 +59,8 @@ class Hoster(Plugin): def __init__(self, pyfile): - super(Hoster, self).__init__(pyfile.m.core) + self.pyload = pyfile.m.core + self.info = {} #: Provide information in dict here #: Engage wan reconnection self.want_reconnect = False @@ -118,7 +119,7 @@ class Hoster(Plugin): self.last_check = None #: Js engine, see `JsEngine` - self.js = self.core.js + self.js = self.pyload.js #: Captcha task self.c_task = None @@ -187,8 +188,8 @@ class Hoster(Plugin): def get_chunk_count(self): if self.chunk_limit <= 0: - return self.core.config.get("download", "chunks") - return min(self.core.config.get("download", "chunks"), self.chunk_limit) + return self.pyload.config.get("download", "chunks") + return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) def reset_account(self): @@ -196,7 +197,7 @@ class Hoster(Plugin): Don't use account and retry download """ self.account = None - self.req = self.core.requestFactory.getRequest(self.__name__) + self.req = self.pyload.requestFactory.getRequest(self.__name__) self.retry() @@ -363,10 +364,10 @@ class Hoster(Plugin): with open(os.path.join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: tmpCaptcha.write(img) - has_plugin = self.__name__ in self.core.pluginManager.ocrPlugins + has_plugin = self.__name__ in self.pyload.pluginManager.ocrPlugins - if self.core.captcha: - Ocr = self.core.pluginManager.loadClass("ocr", self.__name__) + if self.pyload.captcha: + Ocr = self.pyload.pluginManager.loadClass("ocr", self.__name__) else: Ocr = None @@ -378,7 +379,7 @@ class Hoster(Plugin): ocr = Ocr() result = ocr.get_captcha(tmpCaptcha.name) else: - captchaManager = self.core.captchaManager + captchaManager = self.pyload.captchaManager task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) self.c_task = task captchaManager.handleCaptcha(task) @@ -401,7 +402,7 @@ class Hoster(Plugin): result = task.result self.log_debug("Received captcha result: %s" % result) - if not self.core.debug: + if not self.pyload.debug: try: os.remove(tmpCaptcha.name) except Exception: @@ -442,8 +443,8 @@ class Hoster(Plugin): if not url or not isinstance(url, basestring): self.fail(_("No url given")) - if self.core.debug: - self.log_debug("Download url " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + if self.pyload.debug: + self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) self.correct_captcha() self.check_for_same_files() @@ -453,17 +454,17 @@ class Hoster(Plugin): if disposition: self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name - download_folder = self.core.config.get("general", "download_folder") + download_folder = self.pyload.config.get("general", "download_folder") location = fs_join(download_folder, self.pyfile.package().folder) if not os.path.exists(location): try: - os.makedirs(location, int(self.core.config.get("permission", "folder"), 8)) + os.makedirs(location, int(self.pyload.config.get("permission", "folder"), 8)) - if self.core.config.get("permission", "change_dl") and os.name != "nt": - uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2] - gid = grp.getgrnam(self.core.config.get("permission", "group"))[2] + if self.pyload.config.get("permission", "change_dl") and os.name != "nt": + uid = pwd.getpwnam(self.pyload.config.get("permission", "user"))[2] + gid = grp.getgrnam(self.pyload.config.get("permission", "group"))[2] os.chown(location, uid, gid) except Exception, e: @@ -475,7 +476,7 @@ class Hoster(Plugin): filename = os.path.join(location, name) - self.core.addonManager.dispatchEvent("download-start", self.pyfile, url, filename) + self.pyload.addonManager.dispatchEvent("download-start", self.pyfile, url, filename) try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, @@ -494,16 +495,16 @@ class Hoster(Plugin): fs_filename = fs_encode(filename) - if self.core.config.get("permission", "change_file"): + if self.pyload.config.get("permission", "change_file"): try: - os.chmod(fs_filename, int(self.core.config.get("permission", "file"), 8)) + os.chmod(fs_filename, int(self.pyload.config.get("permission", "file"), 8)) except Exception, e: self.log_warning(_("Setting file mode failed"), e) - if self.core.config.get("permission", "change_dl") and os.name != "nt": + if self.pyload.config.get("permission", "change_dl") and os.name != "nt": try: - uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2] - gid = grp.getgrnam(self.core.config.get("permission", "group"))[2] + uid = pwd.getpwnam(self.pyload.config.get("permission", "user"))[2] + gid = grp.getgrnam(self.pyload.config.get("permission", "group"))[2] os.chown(fs_filename, uid, gid) except Exception, e: @@ -695,22 +696,22 @@ class Hoster(Plugin): """ pack = self.pyfile.package() - for pyfile in self.core.files.cache.values(): + for pyfile in self.pyload.files.cache.values(): if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: if pyfile.status in (0, 12): #: Finished or downloading self.skip(pyfile.pluginname) elif pyfile.status in (5, 7) and starting: #: A download is waiting/starting and was appenrently started before self.skip(pyfile.pluginname) - download_folder = self.core.config.get("general", "download_folder") + download_folder = self.pyload.config.get("general", "download_folder") location = fs_join(download_folder, pack.folder, self.pyfile.name) - if starting and self.core.config.get("download", "skip_existing") and os.path.exists(location): + if starting and self.pyload.config.get("download", "skip_existing") and os.path.exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: self.skip("File exists") - pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + pyfile = self.pyload.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: if os.path.exists(location): self.skip(pyfile[0]) diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 127d83ab5..4e6b0cf95 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -70,10 +70,10 @@ class MultiHook(Hook): def init_plugin(self): self.pluginname = self.__name__.rsplit("Hook", 1)[0] - plugin, self.plugintype = self.core.pluginManager.findPlugin(self.pluginname) + plugin, self.plugintype = self.pyload.pluginManager.findPlugin(self.pluginname) if plugin: - self.pluginmodule = self.core.pluginManager.loadModule(self.plugintype, self.pluginname) + self.pluginmodule = self.pyload.pluginManager.loadModule(self.plugintype, self.pluginname) self.pluginclass = getattr(self.pluginmodule, self.pluginname) else: self.log_warning(_("Hook plugin will be deactivated due missing plugin reference")) @@ -81,7 +81,7 @@ class MultiHook(Hook): def load_account(self): - self.account = self.core.accountManager.getAccountPlugin(self.pluginname) + self.account = self.pyload.accountManager.getAccountPlugin(self.pluginname) if self.account and not self.account.can_use(): self.account = None @@ -161,7 +161,7 @@ class MultiHook(Hook): if self.get_config('reload', True): self.interval = max(self.get_config('reloadinterval', 12) * 60 * 60, self.MIN_RELOAD_INTERVAL) else: - self.core.scheduler.removeJob(self.cb) + self.pyload.scheduler.removeJob(self.cb) self.cb = None self.log_info(_("Reloading supported %s list") % self.plugintype) @@ -186,11 +186,11 @@ class MultiHook(Hook): excludedList = [] if self.plugintype == "hoster": - pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) - accountList = [account.type.lower() for account in self.core.api.getAccounts(False) if account.valid and account.premium] + pluginMap = dict((name.lower(), name) for name in self.pyload.pluginManager.hosterPlugins.iterkeys()) + accountList = [account.type.lower() for account in self.pyload.api.getAccounts(False) if account.valid and account.premium] else: pluginMap = {} - accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.core.pluginManager.crypterPlugins.iterkeys()] + accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.pyload.pluginManager.crypterPlugins.iterkeys()] for plugin in self.plugins_cached(): name = remove_chars(plugin, "-.") @@ -211,7 +211,7 @@ class MultiHook(Hook): self.log_debug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) for plugin in self.supported: - hdict = self.core.pluginManager.plugins[self.plugintype][plugin] + hdict = self.pyload.pluginManager.plugins[self.plugintype][plugin] hdict['new_module'] = self.pluginmodule hdict['new_name'] = self.pluginname @@ -230,13 +230,13 @@ class MultiHook(Hook): self.log_debug("Regexp: %s" % regexp) - hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] + hdict = self.pyload.pluginManager.plugins[self.plugintype][self.pluginname] hdict['pattern'] = regexp hdict['re'] = re.compile(regexp) def unload_plugin(self, plugin): - hdict = self.core.pluginManager.plugins[self.plugintype][plugin] + hdict = self.pyload.pluginManager.plugins[self.plugintype][plugin] if "module" in hdict: hdict.pop('module', None) @@ -253,7 +253,7 @@ class MultiHook(Hook): self.unload_plugin(plugin) #: Reset pattern - hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] + hdict = self.pyload.pluginManager.plugins[self.plugintype][self.pluginname] hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') hdict['re'] = re.compile(hdict['pattern']) diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 7bba2f6d3..5ce880cde 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -87,8 +87,8 @@ class MultiHoster(SimpleHoster): self.retry_free() elif self.get_config("revertfailed", True) \ - and "new_module" in self.core.pluginManager.hosterPlugins[self.__name__]: - hdict = self.core.pluginManager.hosterPlugins[self.__name__] + and "new_module" in self.pyload.pluginManager.hosterPlugins[self.__name__]: + hdict = self.pyload.pluginManager.hosterPlugins[self.__name__] tmp_module = hdict['new_module'] tmp_name = hdict['new_name'] diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index a00f53947..c5cf5c2e9 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -28,6 +28,9 @@ class OCR(Plugin): def __init__(self): + self.pyload = pyfile.m.core + self.info = {} #: Provide information in dict here + self.logger = logging.getLogger("log") self.init() diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 7dec6344e..74f5e7c6d 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -121,8 +121,8 @@ class Plugin(object): def __init__(self, core): - self.core = core - self.info = {} #: Provide information in dict here + self.pyload = core + self.info = {} #: Provide information in dict here self.init() @@ -134,7 +134,7 @@ class Plugin(object): def _log(self, level, args): - log = getattr(self.core.log, level) + log = getattr(self.pyload.log, level) msg = fs_encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in args if a)) #@NOTE: `fs_encode` -> `encode` in 0.4.10 log("%(plugin)s%(id)s: %(msg)s" % {'plugin': self.__name__, 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", @@ -142,7 +142,7 @@ class Plugin(object): def log_debug(self, *args): - if self.core.debug: + if self.pyload.debug: return self._log("debug", args) @@ -170,15 +170,7 @@ class Plugin(object): :param value: :return: """ - self.core.config.setPlugin(self.__name__, option, value) - - - #: Deprecated method, use `set_config` instead - def setConf(self, *args, **kwargs): - """ - See `set_config` - """ - return self.set_config(*args, **kwargs) + self.pyload.config.setPlugin(self.__name__, option, value) def get_config(self, option, default="", plugin=None): @@ -189,64 +181,32 @@ class Plugin(object): :return: """ try: - return self.core.config.getPlugin(plugin or self.__name__, option) + return self.pyload.config.getPlugin(plugin or self.__name__, option) except KeyError: self.log_warning(_("Config option or plugin not found")) return default - #: Deprecated method, use `get_config` instead - def getConf(self, *args, **kwargs): - """ - See `get_config` - """ - return self.get_config(*args, **kwargs) - - def store(self, key, value): """ Saves a value persistently to the database """ - self.core.db.setStorage(self.__name__, key, value) - - - #: Deprecated method, use `store` instead - def setStorage(self, *args, **kwargs): - """ - Same as `store` - """ - return self.store(*args, **kwargs) + self.pyload.db.setStorage(self.__name__, key, value) def retrieve(self, key, default=None): """ Retrieves saved value or dict of all saved entries if key is None """ - return self.core.db.getStorage(self.__name__, key) or default - - - #: Deprecated method, use `retrieve` instead - def getStorage(self, *args, **kwargs): - """ - Same as `retrieve` - """ - return self.retrieve(*args, **kwargs) + return self.pyload.db.getStorage(self.__name__, key) or default def delete(self, key): """ Delete entry in db """ - self.core.db.delStorage(self.__name__, key) - - - #: Deprecated method, use `delete` instead - def delStorage(self, *args, **kwargs): - """ - Same as `delete` - """ - return self.delete(*args, **kwargs) + self.pyload.db.delStorage(self.__name__, key) def fail(self, reason): @@ -288,21 +248,21 @@ class Plugin(object): if not url or not isinstance(url, basestring): self.fail(_("No url given")) - if self.core.debug: - self.log_debug("Load url " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + if self.pyload.debug: + self.log_debug("LOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) if req is None: if hasattr(self, "req"): req = self.req else: - req = self.core.requestFactory.getRequest(self.__name__) + req = self.pyload.requestFactory.getRequest(self.__name__) res = req.load(url, get, post, ref, cookies, just_header, multipart, bool(decode)) if decode: res = html_unescape(res).decode(decode if isinstance(decode, basestring) else 'utf8') - if self.core.debug: + if self.pyload.debug: frame = inspect.currentframe() framefile = fs_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) try: diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index c76ef43e2..395025bc1 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -147,7 +147,7 @@ class SevenZip(UnRar): #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.manager.log_debug(" ".join(call)) + self.log_debug(" ".join(call)) p = subprocess.Popen(call, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return p diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index cd447db6b..b249e112d 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -53,7 +53,7 @@ class SimpleCrypter(Crypter, SimpleHoster): #@TODO: Remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") - account = self.core.accountManager.getAccountPlugin(account_name) + account = self.pyload.accountManager.getAccountPlugin(account_name) if account and account.can_use(): self.user, data = account.select_account() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 40489d404..3003723a6 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -231,7 +231,7 @@ class SimpleHoster(Hoster): self.LINK_PREMIUM_PATTERN = self.LINK_PATTERN if (self.MULTI_HOSTER - and (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] + and (self.__pattern__ != self.pyload.pluginManager.hosterPlugins[self.__name__]['pattern'] or re.match(self.__pattern__, self.pyfile.url) is None)): self.multihost = True return @@ -499,8 +499,8 @@ class SimpleHoster(Hoster): #: Deprecated method - def getFileInfo(self): - self.info = {} + def get_fileInfo(self): + self.info = {} self.check_info() return self.info @@ -548,5 +548,5 @@ class SimpleHoster(Hoster): return self.premium = False self.account = None - self.req = self.core.requestFactory.getRequest(self.__name__) + self.req = self.pyload.requestFactory.getRequest(self.__name__) raise Retry(_("Fallback to free download")) diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index abd94b4df..8f6038d64 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -191,7 +191,7 @@ class UnRar(Extractor): raise ArchiveError(_("Cannot open file")) if err.strip(): #: Only log error at this point - self.manager.log_error(err.strip()) + self.log_error(err.strip()) result = set() if not self.fullpath and self.VERSION.startswith('5'): @@ -237,7 +237,7 @@ class UnRar(Extractor): #@NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.manager.log_debug(" ".join(call)) + self.log_debug(" ".join(call)) p = subprocess.Popen(call, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return p diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index c371e97f5..a10256a55 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -32,7 +32,7 @@ class XFSCrypter(SimpleCrypter): account = self.account else: account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") - account = self.core.accountManager.getAccountPlugin(account_name) + account = self.pyload.accountManager.getAccountPlugin(account_name) if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: self.HOSTER_DOMAIN = account.HOSTER_DOMAIN diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index d4810f8b7..8adda0cd9 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -63,7 +63,7 @@ class XFSHoster(SimpleHoster): if self.account: account = self.account else: - account = self.core.accountManager.getAccountPlugin(self.__name__) + account = self.pyload.accountManager.getAccountPlugin(self.__name__) if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: self.HOSTER_DOMAIN = account.HOSTER_DOMAIN -- cgit v1.2.3 From a5b840079dfa281127c0fc0f5a13708b5ecb5031 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 12:13:02 +0200 Subject: [OCR] Fix __init__ --- module/plugins/internal/Hoster.py | 2 +- module/plugins/internal/OCR.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index f5566fb6e..e9fc81e7a 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -376,7 +376,7 @@ class Hoster(Plugin): if self.pyfile.abort: self.abort() - ocr = Ocr() + ocr = Ocr(self.pyfile) result = ocr.get_captcha(tmpCaptcha.name) else: captchaManager = self.pyload.captchaManager diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index c5cf5c2e9..566801e9a 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -27,11 +27,9 @@ class OCR(Plugin): __authors__ = [("pyLoad Team", "admin@pyload.org")] - def __init__(self): + def __init__(self, pyfile): self.pyload = pyfile.m.core self.info = {} #: Provide information in dict here - - self.logger = logging.getLogger("log") self.init() @@ -68,7 +66,7 @@ class OCR(Plugin): output = popen.stdout.read() + " | " + popen.stderr.read() popen.stdout.close() popen.stderr.close() - self.logger.debug("Tesseract ReturnCode %s Output: %s" % (popen.returncode, output)) + self.pyload.log_debug("Tesseract ReturnCode " + popen.returncode, "Output: " + output) def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True, pagesegmode=None): @@ -85,7 +83,7 @@ class OCR(Plugin): self.log_error(e) return - self.logger.debug("save tiff") + self.pyload.log_debug("Saving tiff...") self.image.save(tmpTif.name, 'TIFF') if os.name == "nt": @@ -114,9 +112,9 @@ class OCR(Plugin): tessparams.append("nobatch") tessparams.append(os.path.abspath(tmpSub.name)) - self.logger.debug("run tesseract") + self.pyload.log_debug("Running tesseract...") self.run(tessparams) - self.logger.debug("read txt") + self.pyload.log_debug("Reading txt...") try: with open(tmpTxt.name, 'r') as f: @@ -124,7 +122,7 @@ class OCR(Plugin): except Exception: self.result_captcha = "" - self.logger.debug(self.result_captcha) + self.pyload.log_info(_("OCR result: ") + self.result_captcha) try: os.remove(tmpTif.name) os.remove(tmpTxt.name) -- cgit v1.2.3 From 33e2b36605e41962a2e8eee304a7f3d29690ffa8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 13:42:29 +0200 Subject: [OCR] Fix __init__ (2) --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3003723a6..de3e78da8 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -500,7 +500,7 @@ class SimpleHoster(Hoster): #: Deprecated method def get_fileInfo(self): - self.info = {} + self.info = {} self.check_info() return self.info -- cgit v1.2.3 From d38e830b7c0b3c6561a0072c74bbccb5fcdf4a61 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 14:43:42 +0200 Subject: New __status__ magic key --- module/plugins/internal/Account.py | 1 + module/plugins/internal/AdYouLike.py | 1 + module/plugins/internal/Addon.py | 1 + module/plugins/internal/AdsCaptcha.py | 1 + module/plugins/internal/Captcha.py | 1 + module/plugins/internal/Container.py | 1 + module/plugins/internal/Crypter.py | 1 + module/plugins/internal/DeadCrypter.py | 1 + module/plugins/internal/DeadHoster.py | 1 + module/plugins/internal/Extractor.py | 1 + module/plugins/internal/Hook.py | 1 + module/plugins/internal/Hoster.py | 1 + module/plugins/internal/MultiHook.py | 1 + module/plugins/internal/MultiHoster.py | 1 + module/plugins/internal/OCR.py | 1 + module/plugins/internal/Plugin.py | 1 + module/plugins/internal/ReCaptcha.py | 1 + module/plugins/internal/SevenZip.py | 1 + module/plugins/internal/SimpleCrypter.py | 1 + module/plugins/internal/SimpleHoster.py | 1 + module/plugins/internal/SolveMedia.py | 1 + module/plugins/internal/UnRar.py | 1 + module/plugins/internal/UnZip.py | 1 + module/plugins/internal/XFSAccount.py | 1 + module/plugins/internal/XFSCrypter.py | 1 + module/plugins/internal/XFSHoster.py | 1 + 26 files changed, 26 insertions(+) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 72981a108..27525e1d2 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -17,6 +17,7 @@ class Account(Plugin): __name__ = "Account" __type__ = "account" __version__ = "0.05" + __status__ = "stable" __description__ = """Base account plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/AdYouLike.py b/module/plugins/internal/AdYouLike.py index 10f281494..fcb9b8372 100644 --- a/module/plugins/internal/AdYouLike.py +++ b/module/plugins/internal/AdYouLike.py @@ -10,6 +10,7 @@ class AdYouLike(Captcha): __name__ = "AdYouLike" __type__ = "captcha" __version__ = "0.07" + __status__ = "stable" __description__ = """AdYouLike captcha service plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index a6bf685b5..b29aa0936 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -26,6 +26,7 @@ class Addon(Plugin): __name__ = "Addon" __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 __version__ = "0.01" + __status__ = "stable" __config__ = [] #: [("name", "type", "desc", "default")] __threaded__ = [] #@TODO: Remove in 0.4.10 diff --git a/module/plugins/internal/AdsCaptcha.py b/module/plugins/internal/AdsCaptcha.py index 5f4dbaa86..b99697a84 100644 --- a/module/plugins/internal/AdsCaptcha.py +++ b/module/plugins/internal/AdsCaptcha.py @@ -10,6 +10,7 @@ class AdsCaptcha(Captcha): __name__ = "AdsCaptcha" __type__ = "captcha" __version__ = "0.10" + __status__ = "stable" __description__ = """AdsCaptcha captcha service plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 6ca754a06..814c36756 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -7,6 +7,7 @@ class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" __version__ = "0.31" + __status__ = "stable" __description__ = """Base captcha service plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 2c092f98a..8dfa1031d 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -13,6 +13,7 @@ class Container(Crypter): __name__ = "Container" __type__ = "container" __version__ = "0.05" + __status__ = "stable" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 653469079..034224b4e 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -10,6 +10,7 @@ class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" __version__ = "0.05" + __status__ = "stable" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config.get("general", "folder_per_package") diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 15ba68b28..5232403b8 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -7,6 +7,7 @@ class DeadCrypter(Crypter): __name__ = "DeadCrypter" __type__ = "crypter" __version__ = "0.08" + __status__ = "stable" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 5050ba059..b0c179ed1 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -7,6 +7,7 @@ class DeadHoster(Hoster): __name__ = "DeadHoster" __type__ = "hoster" __version__ = "0.18" + __status__ = "stable" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 97c2c63b1..e1fb2332d 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -23,6 +23,7 @@ class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" __version__ = "0.25" + __status__ = "stable" __description__ = """Base extractor plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index a17037a11..c3b14ef73 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -7,6 +7,7 @@ class Hook(Addon): __name__ = "Hook" __type__ = "hook" __version__ = "0.10" + __status__ = "stable" __config__ = [] #: [("name", "type", "desc", "default")] diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index e9fc81e7a..35e6ef23e 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -46,6 +46,7 @@ class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" __version__ = "0.05" + __status__ = "stable" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 4e6b0cf95..09f91bb2a 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -12,6 +12,7 @@ class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" __version__ = "0.51" + __status__ = "stable" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 5ce880cde..2fc319102 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -10,6 +10,7 @@ class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" __version__ = "0.43" + __status__ = "stable" __pattern__ = r'^unmatchable$' __config__ = [("use_premium" , "bool", "Use premium account if available" , True), diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 566801e9a..ee5571f77 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -21,6 +21,7 @@ class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" __version__ = "0.12" + __status__ = "stable" __description__ = """OCR base plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 74f5e7c6d..030c372aa 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -108,6 +108,7 @@ class Plugin(object): __name__ = "Plugin" __type__ = "hoster" __version__ = "0.13" + __status__ = "stable" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] diff --git a/module/plugins/internal/ReCaptcha.py b/module/plugins/internal/ReCaptcha.py index 1caf6b7b0..4d04c07db 100644 --- a/module/plugins/internal/ReCaptcha.py +++ b/module/plugins/internal/ReCaptcha.py @@ -14,6 +14,7 @@ class ReCaptcha(Captcha): __name__ = "ReCaptcha" __type__ = "captcha" __version__ = "0.18" + __status__ = "stable" __description__ = """ReCaptcha captcha service plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index 395025bc1..dc0d765cf 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -11,6 +11,7 @@ from module.utils import fs_encode, save_join as fs_join class SevenZip(UnRar): __name__ = "SevenZip" __version__ = "0.12" + __status__ = "stable" __description__ = """7-Zip extractor plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index b249e112d..0e54c30a4 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -11,6 +11,7 @@ class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" __version__ = "0.56" + __status__ = "stable" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index de3e78da8..9d1ebc0bf 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -25,6 +25,7 @@ class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" __version__ = "1.71" + __status__ = "stable" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), diff --git a/module/plugins/internal/SolveMedia.py b/module/plugins/internal/SolveMedia.py index 87537470f..5d701a5a2 100644 --- a/module/plugins/internal/SolveMedia.py +++ b/module/plugins/internal/SolveMedia.py @@ -10,6 +10,7 @@ class SolveMedia(Captcha): __name__ = "SolveMedia" __type__ = "captcha" __version__ = "0.15" + __status__ = "stable" __description__ = """SolveMedia captcha service plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 8f6038d64..a880ab320 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -23,6 +23,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" __version__ = "1.21" + __status__ = "stable" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index c2eb522a8..1088f571a 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -13,6 +13,7 @@ from module.utils import fs_encode class UnZip(Extractor): __name__ = "UnZip" __version__ = "1.13" + __status__ = "stable" __description__ = """Zip extractor plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 7ae6d816e..b5028eda6 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,6 +12,7 @@ class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" __version__ = "0.39" + __status__ = "stable" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index a10256a55..7470eb221 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -7,6 +7,7 @@ class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" __version__ = "0.11" + __status__ = "stable" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 8adda0cd9..09799aa28 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,6 +14,7 @@ class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" __version__ = "0.54" + __status__ = "stable" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 027cb529d79558de19c47da88a782b31745a65c9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 21 Jul 2015 22:53:37 +0200 Subject: New Captcha skeleton --- module/plugins/internal/AdYouLike.py | 12 ++-- module/plugins/internal/AdsCaptcha.py | 20 +++--- module/plugins/internal/Captcha.py | 102 ++++++++++++++++++++++++------ module/plugins/internal/CaptchaService.py | 46 ++++++++++++++ module/plugins/internal/Hoster.py | 92 ++------------------------- module/plugins/internal/ReCaptcha.py | 32 +++++----- module/plugins/internal/SimpleHoster.py | 6 +- module/plugins/internal/SolveMedia.py | 20 +++--- module/plugins/internal/XFSHoster.py | 2 +- 9 files changed, 179 insertions(+), 153 deletions(-) create mode 100644 module/plugins/internal/CaptchaService.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AdYouLike.py b/module/plugins/internal/AdYouLike.py index fcb9b8372..d14babb51 100644 --- a/module/plugins/internal/AdYouLike.py +++ b/module/plugins/internal/AdYouLike.py @@ -3,10 +3,10 @@ import re from module.common.json_layer import json_loads -from module.plugins.internal.Captcha import Captcha +from module.plugins.internal.CaptchaService import CaptchaService -class AdYouLike(Captcha): +class AdYouLike(CaptchaService): __name__ = "AdYouLike" __type__ = "captcha" __version__ = "0.07" @@ -21,8 +21,8 @@ class AdYouLike(Captcha): CALLBACK_PATTERN = r'(Adyoulike\.g\._jsonp_\d+)' - def detect_key(self, html=None): - html = html or self.retrieve_html() + def detect_key(self, data=None): + html = data or self.retrieve_data() m = re.search(self.AYL_PATTERN, html) n = re.search(self.CALLBACK_PATTERN, html) @@ -35,8 +35,8 @@ class AdYouLike(Captcha): return None - def challenge(self, key=None, html=None): - ayl, callback = key or self.retrieve_key(html) + def challenge(self, key=None, data=None): + ayl, callback = key or self.retrieve_key(data) #: {'adyoulike':{'key':"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, #: 'all':{'element_id':"ayl_private_cap_92300",'lang':"fr",'env':"prod"}} diff --git a/module/plugins/internal/AdsCaptcha.py b/module/plugins/internal/AdsCaptcha.py index b99697a84..f487042e2 100644 --- a/module/plugins/internal/AdsCaptcha.py +++ b/module/plugins/internal/AdsCaptcha.py @@ -3,10 +3,10 @@ import random import re -from module.plugins.internal.Captcha import Captcha +from module.plugins.internal.CaptchaService import CaptchaService -class AdsCaptcha(Captcha): +class AdsCaptcha(CaptchaService): __name__ = "AdsCaptcha" __type__ = "captcha" __version__ = "0.10" @@ -21,8 +21,8 @@ class AdsCaptcha(Captcha): PUBLICKEY_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?.*?PublicKey=([\w-]+)' - def detect_key(self, html=None): - html = html or self.retrieve_html() + def detect_key(self, data=None): + html = data or self.retrieve_data() m = re.search(self.PUBLICKEY_PATTERN, html) n = re.search(self.CAPTCHAID_PATTERN, html) @@ -35,8 +35,8 @@ class AdsCaptcha(Captcha): return None - def challenge(self, key=None, html=None): - PublicKey, CaptchaId = key or self.retrieve_key(html) + def challenge(self, key=None, data=None): + PublicKey, CaptchaId = key or self.retrieve_key(data) html = self.plugin.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, @@ -54,10 +54,10 @@ class AdsCaptcha(Captcha): def result(self, server, challenge): - result = self.plugin.decryptCaptcha("%sChallenge.aspx" % server, - get={'cid': challenge, 'dummy': random.random()}, - cookies=True, - imgtype="jpg") + result = self.decrypt_image("%sChallenge.aspx" % server, + get={'cid': challenge, 'dummy': random.random()}, + cookies=True, + input_type="jpg") self.log_debug("Result: %s" % result) diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 814c36756..af7f66ed5 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -6,24 +6,28 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.31" + __version__ = "0.01" __status__ = "stable" - __description__ = """Base captcha service plugin""" + __description__ = """Base anti-captcha plugin""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - def __init__(self, plugin): + def __init__(self, plugin): #@TODO: pass pyfile instead plugin, so store plugin's html in its associated pyfile as data self.pyload = plugin.core self.info = {} #: Provide information in dict here self.plugin = plugin - self.key = None #: Last key detected + self.task = None #: captchaManager task self.init() + def _log(self, type, args): + return super(Captcha, self)._log(type, (self.plugin.__name__,) + args) + + def init(self): """ Initialize additional data structures @@ -31,29 +35,87 @@ class Captcha(Plugin): pass - #@TODO: Recheck in 0.4.10 - def retrieve_key(self, html): - if self.detect_key(html): - return self.key + def decrypt_image(self, url, get={}, post={}, ref=False, cookies=False, decode=False, + input_type='png', output_type='textual', try_ocr=True): + image = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode) + return self.decrypt(image, input_type, output_type, try_ocr) + + + def decrypt(self, data, input_type='png', output_type='textual', try_ocr=True): + """ + Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param input_type: Type of the Image + :param output_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + :param try_ocr: if True, ocr is not used + + :return: result of decrypting + """ + id = ("%.2f" % time.time())[-6:].replace(".", "") + + with open(os.path.join("tmp", "tmpCaptcha_%s_%s.%s" % (self.plugin.__name__, id, input_type)), "wb") as tmpCaptcha: + tmpCaptcha.write(img) + + has_plugin = self.plugin.__name__ in self.pyload.pluginManager.ocrPlugins + + if self.pyload.captcha: + Ocr = self.pyload.pluginManager.loadClass("ocr", self.plugin.__name__) else: - self.fail(_("%s key not found") % self.__name__) + Ocr = None + if Ocr and try_ocr: + time.sleep(random.randint(3000, 5000) / 1000.0) + if self.pyfile.abort: + self.abort() - #@TODO: Recheck in 0.4.10 - def retrieve_html(self): - if hasattr(self.plugin, "html") and self.plugin.html: - return self.plugin.html + ocr = Ocr(self.pyfile) + result = ocr.get_captcha(tmpCaptcha.name) else: - self.fail(_("%s html not found") % self.__name__) + captchaManager = self.pyload.captchaManager + task = captchaManager.newTask(img, input_type, tmpCaptcha.name, output_type) + self.task = task + captchaManager.handleCaptcha(task) + + while task.isWaiting(): + if self.pyfile.abort: + captchaManager.removeTask(task) + self.abort() + time.sleep(1) + + captchaManager.removeTask(task) + + if task.error and has_plugin: #: Ignore default error message since the user could use try_ocr + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) + + result = task.result + self.log_debug("Received captcha result: %s" % result) + if not self.pyload.debug: + try: + os.remove(tmpCaptcha.name) + except Exception: + pass - def detect_key(self, html=None): - raise NotImplementedError + return result - def challenge(self, key=None, html=None): - raise NotImplementedError + def invalid(self): + self.log_error(_("Invalid captcha")) + if self.task: + self.task.invalid() - def result(self, server, challenge): - raise NotImplementedError + def correct(self): + self.log_info(_("Correct captcha")) + if self.task: + self.task.correct() diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py new file mode 100644 index 000000000..05af8ccec --- /dev/null +++ b/module/plugins/internal/CaptchaService.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.Plugin import Plugin + + +class Captcha(Plugin): + __name__ = "Captcha" + __type__ = "captcha" + __version__ = "0.31" + __status__ = "stable" + + __description__ = """Base anti-captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def init(self): + self.key = None #: Last key detected + + + #@TODO: Recheck in 0.4.10 + def retrieve_key(self, data): + if self.detect_key(data): + return self.key + else: + self.fail(_("%s key not found") % self.__name__) + + + #@TODO: Recheck in 0.4.10, html is now pyfile.data + def retrieve_data(self): + if hasattr(self.plugin, "html") and self.plugin.html: + return self.plugin.html + else: + self.fail(_("%s data not found") % self.__name__) + + + def detect_key(self, data=None): + raise NotImplementedError + + + def challenge(self, key=None, data=None): + raise NotImplementedError + + + def result(self, server, challenge): + raise NotImplementedError diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 35e6ef23e..af3e80acf 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -12,6 +12,7 @@ if os.name != "nt": import grp import pwd +from module.plugins.internal import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip chunks, fixurl as _fixurl, replace_patterns, seconds_to_midnight, set_cookies, parse_html_form, parse_html_tag_attr_value, @@ -122,8 +123,8 @@ class Hoster(Plugin): #: Js engine, see `JsEngine` self.js = self.pyload.js - #: Captcha task - self.c_task = None + #: Captcha stuff + self.captcha = Captcha(self) #: Some plugins store html code here self.html = None @@ -257,7 +258,7 @@ class Hoster(Plugin): time.sleep(1) else: while pyfile.waitUntil > time.time(): - self.thread.m.reconnecting.wait(2) + self.thread.m.reconnecting.wait(1) if pyfile.abort: self.abort() @@ -329,89 +330,6 @@ class Hoster(Plugin): raise Retry(reason) - def invalid_captcha(self): - self.log_error(_("Invalid captcha")) - if self.c_task: - self.c_task.invalid() - - - def correct_captcha(self): - self.log_info(_("Correct captcha")) - if self.c_task: - self.c_task.correct() - - - def decrypt_captcha(self, url, get={}, post={}, cookies=False, forceUser=False, - imgtype='jpg', result_type='textual'): - """ - Loads a captcha and decrypts it with ocr, plugin, user input - - :param url: url of captcha image - :param get: get part for request - :param post: post part for request - :param cookies: True if cookies should be enabled - :param forceUser: if True, ocr is not used - :param imgtype: Type of the Image - :param result_type: 'textual' if text is written on the captcha\ - or 'positional' for captcha where the user have to click\ - on a specific region on the captcha - - :return: result of decrypting - """ - img = self.load(url, get=get, post=post, cookies=cookies) - - id = ("%.2f" % time.time())[-6:].replace(".", "") - - with open(os.path.join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: - tmpCaptcha.write(img) - - has_plugin = self.__name__ in self.pyload.pluginManager.ocrPlugins - - if self.pyload.captcha: - Ocr = self.pyload.pluginManager.loadClass("ocr", self.__name__) - else: - Ocr = None - - if Ocr and not forceUser: - time.sleep(random.randint(3000, 5000) / 1000.0) - if self.pyfile.abort: - self.abort() - - ocr = Ocr(self.pyfile) - result = ocr.get_captcha(tmpCaptcha.name) - else: - captchaManager = self.pyload.captchaManager - task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) - self.c_task = task - captchaManager.handleCaptcha(task) - - while task.isWaiting(): - if self.pyfile.abort: - captchaManager.removeTask(task) - self.abort() - time.sleep(1) - - captchaManager.removeTask(task) - - if task.error and has_plugin: #: Ignore default error message since the user could use OCR - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) - - result = task.result - self.log_debug("Received captcha result: %s" % result) - - if not self.pyload.debug: - try: - os.remove(tmpCaptcha.name) - except Exception: - pass - - return result - - def fixurl(self, url): url = _fixurl(url) @@ -447,7 +365,7 @@ class Hoster(Plugin): if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) - self.correct_captcha() + self.captcha.correct() self.check_for_same_files() self.pyfile.setStatus("downloading") diff --git a/module/plugins/internal/ReCaptcha.py b/module/plugins/internal/ReCaptcha.py index 4d04c07db..b4f9ef1eb 100644 --- a/module/plugins/internal/ReCaptcha.py +++ b/module/plugins/internal/ReCaptcha.py @@ -7,10 +7,10 @@ import urlparse from base64 import b64encode -from module.plugins.internal.Captcha import Captcha +from module.plugins.internal.CaptchaService import CaptchaService -class ReCaptcha(Captcha): +class ReCaptcha(CaptchaService): __name__ = "ReCaptcha" __type__ = "captcha" __version__ = "0.18" @@ -27,8 +27,8 @@ class ReCaptcha(Captcha): KEY_V2_PATTERN = r'(?:data-sitekey=["\']|["\']sitekey["\']:\s*["\'])([\w-]+)' - def detect_key(self, html=None): - html = html or self.retrieve_html() + def detect_key(self, data=None): + html = data or self.retrieve_data() m = re.search(self.KEY_V2_PATTERN, html) or re.search(self.KEY_V1_PATTERN, html) if m: @@ -40,15 +40,15 @@ class ReCaptcha(Captcha): return None - def challenge(self, key=None, html=None, version=None): - key = key or self.retrieve_key(html) + def challenge(self, key=None, data=None, version=None): + key = key or self.retrieve_key(data) if version in (1, 2): return getattr(self, "_challenge_v%s" % version)(key) else: return self.challenge(key, - version=2 if re.search(self.KEY_V2_PATTERN, html or self.retrieve_html()) else 1) + version=2 if re.search(self.KEY_V2_PATTERN, html or self.retrieve_data()) else 1) def _challenge_v1(self, key): @@ -81,11 +81,11 @@ class ReCaptcha(Captcha): self.fail(_("ReCaptcha second challenge pattern not found")) self.log_debug("Second challenge: %s" % challenge) - result = self.plugin.decryptCaptcha("%simage" % server, - get={'c': challenge}, - cookies=True, - forceUser=True, - imgtype="jpg") + result = self.decrypt("%simage" % server, + get={'c': challenge}, + cookies=True, + input_type="jpg", + try_ocr=False) self.log_debug("Result: %s" % result) @@ -170,10 +170,10 @@ class ReCaptcha(Captcha): self.log_debug("Token #3: %s" % token3.group(1)) millis_captcha_loading = int(round(time.time() * 1000)) - captcha_response = self.plugin.decryptCaptcha("https://www.google.com/recaptcha/api2/payload", - get={'c':token3.group(1), 'k':key}, - cookies=True, - forceUser=True) + captcha_response = self.decrypt_image("https://www.google.com/recaptcha/api2/payload", + get={'c':token3.group(1), 'k':key}, + cookies=True, + try_ocr=False) response = b64encode('{"response":"%s"}' % captcha_response) self.log_debug("Result: %s" % response) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 9d1ebc0bf..5960794cc 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -301,8 +301,8 @@ class SimpleHoster(Hoster): def check_file(self): lastDownload = fs_encode(self.last_download) - if self.c_task and not self.last_download: - self.invalid_captcha() + if self.captcha.task and not self.last_download: + self.captcha.invalid() self.retry(10, reason=_("Wrong captcha")) elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, @@ -396,7 +396,7 @@ class SimpleHoster(Hoster): self.fail(_("Connection from your current IP address is not allowed")) elif re.search('captcha|code', errmsg, re.I): - self.invalid_captcha() + self.captcha.invalid() self.retry(10, reason=_("Wrong captcha")) elif re.search('countdown|expired', errmsg, re.I): diff --git a/module/plugins/internal/SolveMedia.py b/module/plugins/internal/SolveMedia.py index 5d701a5a2..ce4ebb007 100644 --- a/module/plugins/internal/SolveMedia.py +++ b/module/plugins/internal/SolveMedia.py @@ -3,10 +3,10 @@ import re from module.plugins.internal.Plugin import Fail -from module.plugins.internal.Captcha import Captcha +from module.plugins.internal.CaptchaService import CaptchaService -class SolveMedia(Captcha): +class SolveMedia(CaptchaService): __name__ = "SolveMedia" __type__ = "captcha" __version__ = "0.15" @@ -20,8 +20,8 @@ class SolveMedia(Captcha): KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']' - def detect_key(self, html=None): - html = html or self.retrieve_html() + def detect_key(self, data=None): + html = data or self.retrieve_data() m = re.search(self.KEY_PATTERN, html) if m: @@ -33,8 +33,8 @@ class SolveMedia(Captcha): return None - def challenge(self, key=None, html=None): - key = key or self.retrieve_key(html) + def challenge(self, key=None, data=None): + key = key or self.retrieve_key(data) html = self.plugin.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}) @@ -95,10 +95,10 @@ class SolveMedia(Captcha): def result(self, server, challenge): - result = self.plugin.decryptCaptcha(server, - get={'c': challenge}, - cookies=True, - imgtype="gif") + result = self.decrypt_image(server, + get={'c': challenge}, + cookies=True, + input_type="gif") self.log_debug("Result: %s" % result) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 09799aa28..ec9a18a48 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -221,7 +221,7 @@ class XFSHoster(SimpleHoster): m = re.search(self.CAPTCHA_PATTERN, self.html) if m: captcha_url = m.group(1) - inputs['code'] = self.decrypt_captcha(captcha_url) + inputs['code'] = self.captcha.decrypt_image(captcha_url) return m = re.search(self.CAPTCHA_BLOCK_PATTERN, self.html, re.S) -- cgit v1.2.3 From c2091ebec0ce394ec847f82db9bbaddc150d5fca Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 21 Jul 2015 22:55:54 +0200 Subject: [Extractor] is_usable -> find --- module/plugins/internal/Extractor.py | 6 ++--- module/plugins/internal/SevenZip.py | 24 +++++++++++-------- module/plugins/internal/UnRar.py | 45 +++++++++++++++++++----------------- module/plugins/internal/UnZip.py | 4 ++-- 4 files changed, 43 insertions(+), 36 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index e1fb2332d..3d2859f54 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.25" + __version__ = "0.26" __status__ = "stable" __description__ = """Base extractor plugin""" @@ -48,12 +48,12 @@ class Extractor(Plugin): @classmethod - def is_usable(cls): + def find(cls): """ Check if system statisfy dependencies :return: boolean """ - return None + pass @classmethod diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index dc0d765cf..f2194157c 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -10,7 +10,7 @@ from module.utils import fs_encode, save_join as fs_join class SevenZip(UnRar): __name__ = "SevenZip" - __version__ = "0.12" + __version__ = "0.13" __status__ = "stable" __description__ = """7-Zip extractor plugin""" @@ -38,19 +38,23 @@ class SevenZip(UnRar): @classmethod - def is_usable(cls): - if os.name == "nt": - cls.CMD = os.path.join(pypath, "7z.exe") - p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - else: + def find(cls): + try: + if os.name == "nt": + cls.CMD = os.path.join(pypath, "7z.exe") + p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() - m = cls.re_version.search(out) - cls.VERSION = m.group(1) if m else '(version unknown)' + except OSError: + return False + + else: + return True - return True + finally: + m = cls.re_version.search(out) + cls.VERSION = m.group(1) if m else '(version unknown)' def verify(self, password): diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index a880ab320..03b19bcf9 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.21" + __version__ = "1.22" __status__ = "stable" __description__ = """Rar extractor plugin""" @@ -49,34 +49,37 @@ class UnRar(Extractor): @classmethod - def is_usable(cls): - if os.name == "nt": - try: + def find(cls): + try: + if os.name == "nt": cls.CMD = os.path.join(pypath, "RAR.exe") - p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - cls.__name__ = "RAR" - cls.REPAIR = True + else: + cls.CMD = "rar" - except OSError: - cls.CMD = os.path.join(pypath, "UnRAR.exe") - p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - else: + p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + cls.__name__ = "RAR" + cls.REPAIR = True + + except OSError: try: - p = subprocess.Popen(["rar"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - cls.__name__ = "RAR" - cls.REPAIR = True + if os.name == "nt": + cls.CMD = os.path.join(pypath, "UnRAR.exe") + else: + cls.CMD = "unrar" - except OSError: #: Fallback to unrar p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() - m = cls.re_version.search(out) - cls.VERSION = m.group(1) if m else '(version unknown)' + except OSError: + return False - return True + else: + return True + + finally: + m = cls.re_version.search(out) + cls.VERSION = m.group(1) if m else '(version unknown)' @classmethod diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 1088f571a..ef2b2b56d 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -12,7 +12,7 @@ from module.utils import fs_encode class UnZip(Extractor): __name__ = "UnZip" - __version__ = "1.13" + __version__ = "1.14" __status__ = "stable" __description__ = """Zip extractor plugin""" @@ -25,7 +25,7 @@ class UnZip(Extractor): @classmethod - def is_usable(cls): + def find(cls): return sys.version_info[:2] >= (2, 6) -- cgit v1.2.3 From 4fc28dc09f9632eb4a15a1ef48778427f9dcae33 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 23 Jul 2015 18:53:06 +0200 Subject: Code cosmetics --- module/plugins/internal/Account.py | 2 +- module/plugins/internal/Crypter.py | 2 +- module/plugins/internal/Hoster.py | 4 ++-- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/UnRar.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 27525e1d2..96f1cb533 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -206,7 +206,7 @@ class Account(Plugin): def load_account_info(self, name, req=None): """ - This should be overwritten in account plugin,\ + This should be overwritten in account plugin and retrieving account information for user :param name: diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 034224b4e..627c198e1 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -13,7 +13,7 @@ class Crypter(Hoster): __status__ = "stable" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config.get("general", "folder_per_package") + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides pyload.config.get("general", "folder_per_package") ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] __description__ = """Base decrypter plugin""" diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index af3e80acf..ffd7c4ecc 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -83,7 +83,7 @@ class Hoster(Plugin): self.ocr = None #: Account handler instance, see :py:class:`Account` - self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) #: Premium status self.premium = False @@ -107,7 +107,7 @@ class Hoster(Plugin): #: Premium status self.premium = self.account.is_premium(self.user) else: - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) + self.req = self.pyload.requestFactory.getRequest(self.__name__) #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 0e54c30a4..efda7dc1a 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -14,7 +14,7 @@ class SimpleCrypter(Crypter, SimpleHoster): __status__ = "stable" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides pyload.config['general']['folder_per_package'] ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Simple decrypter plugin""" diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 03b19bcf9..1a2e64b94 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -37,7 +37,7 @@ class UnRar(Extractor): EXTENSIONS = [".rar"] - re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?(\.rev|\.bad)?',re.I) + re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?(\.rev|\.bad)?', re.I) re_filefixed = re.compile(r'Building (.+)') re_filelist = re.compile(r'^(.)(\s*[\w\.\-]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I) @@ -180,7 +180,7 @@ class UnRar(Extractor): #: eventually Multipart Files files.extend(fs_join(dir, os.path.basename(file)) for file in filter(self.is_multipart, os.listdir(dir)) - if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file)) + if re.sub(self.re_multipart, ".rar", name) == re.sub(self.re_multipart, ".rar", file)) return files -- cgit v1.2.3 From 6af9b38a8d5d49355b85aef6ddd003605d6bba05 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 23 Jul 2015 23:44:45 +0200 Subject: Improve Captcha --- module/plugins/internal/AdYouLike.py | 92 ---------------- module/plugins/internal/AdsCaptcha.py | 64 ----------- module/plugins/internal/Captcha.py | 89 ++++++++------- module/plugins/internal/OCR.py | 4 +- module/plugins/internal/ReCaptcha.py | 197 ---------------------------------- module/plugins/internal/SolveMedia.py | 105 ------------------ module/plugins/internal/XFSHoster.py | 6 +- 7 files changed, 53 insertions(+), 504 deletions(-) delete mode 100644 module/plugins/internal/AdYouLike.py delete mode 100644 module/plugins/internal/AdsCaptcha.py delete mode 100644 module/plugins/internal/ReCaptcha.py delete mode 100644 module/plugins/internal/SolveMedia.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AdYouLike.py b/module/plugins/internal/AdYouLike.py deleted file mode 100644 index d14babb51..000000000 --- a/module/plugins/internal/AdYouLike.py +++ /dev/null @@ -1,92 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.common.json_layer import json_loads -from module.plugins.internal.CaptchaService import CaptchaService - - -class AdYouLike(CaptchaService): - __name__ = "AdYouLike" - __type__ = "captcha" - __version__ = "0.07" - __status__ = "stable" - - __description__ = """AdYouLike captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - AYL_PATTERN = r'Adyoulike\.create\s*\((.+?)\)' - CALLBACK_PATTERN = r'(Adyoulike\.g\._jsonp_\d+)' - - - def detect_key(self, data=None): - html = data or self.retrieve_data() - - m = re.search(self.AYL_PATTERN, html) - n = re.search(self.CALLBACK_PATTERN, html) - if m and n: - self.key = (m.group(1).strip(), n.group(1).strip()) - self.log_debug("Ayl: %s | Callback: %s" % self.key) - return self.key #: Key is the tuple(ayl, callback) - else: - self.log_warning(_("Ayl or callback pattern not found")) - return None - - - def challenge(self, key=None, data=None): - ayl, callback = key or self.retrieve_key(data) - - #: {'adyoulike':{'key':"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, - #: 'all':{'element_id':"ayl_private_cap_92300",'lang':"fr",'env':"prod"}} - ayl = json_loads(ayl) - - html = self.plugin.load("http://api-ayl.appspot.com/challenge", - get={'key' : ayl['adyoulike']['key'], - 'env' : ayl['all']['env'], - 'callback': callback}) - try: - challenge = json_loads(re.search(callback + r'\s*\((.+?)\)', html).group(1)) - - except AttributeError: - self.fail(_("AdYouLike challenge pattern not found")) - - self.log_debug("Challenge: %s" % challenge) - - return self.result(ayl, challenge), challenge - - - def result(self, server, challenge): - #: Adyoulike.g._jsonp_5579316662423138 - #: ({'translations':{'fr':{'instructions_visual':"Recopiez « Soonnight » ci-dessous :"}}, - #: 'site_under':true,'clickable':true,'pixels':{'VIDEO_050':[],'DISPLAY':[],'VIDEO_000':[],'VIDEO_100':[], - #: 'VIDEO_025':[],'VIDEO_075':[]},'medium_type':"image/adyoulike", - #: 'iframes':{'big':""},'shares':{},'id':256, - #: 'token':"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1",'formats':{'small':{'y':300,'x':0,'w':300,'h':60}, - #: 'big':{'y':0,'x':0,'w':300,'h':250},'hover':{'y':440,'x':0,'w':300,'h':60}}, - #: 'tid':"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) - - if isinstance(server, basestring): - server = json_loads(server) - - if isinstance(challenge, basestring): - challenge = json_loads(challenge) - - try: - instructions_visual = challenge['translations'][server['all']['lang']]['instructions_visual'] - result = re.search(u'«(.+?)»', instructions_visual).group(1).strip() - - except AttributeError: - self.fail(_("AdYouLike result not found")) - - result = {'_ayl_captcha_engine' : "adyoulike", - '_ayl_env' : server['all']['env'], - '_ayl_tid' : challenge['tid'], - '_ayl_token_challenge': challenge['token'], - '_ayl_response' : response} - - self.log_debug("Result: %s" % result) - - return result diff --git a/module/plugins/internal/AdsCaptcha.py b/module/plugins/internal/AdsCaptcha.py deleted file mode 100644 index f487042e2..000000000 --- a/module/plugins/internal/AdsCaptcha.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- - -import random -import re - -from module.plugins.internal.CaptchaService import CaptchaService - - -class AdsCaptcha(CaptchaService): - __name__ = "AdsCaptcha" - __type__ = "captcha" - __version__ = "0.10" - __status__ = "stable" - - __description__ = """AdsCaptcha captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] - - - CAPTCHAID_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?.*?CaptchaId=(\d+)' - PUBLICKEY_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?.*?PublicKey=([\w-]+)' - - - def detect_key(self, data=None): - html = data or self.retrieve_data() - - m = re.search(self.PUBLICKEY_PATTERN, html) - n = re.search(self.CAPTCHAID_PATTERN, html) - if m and n: - self.key = (m.group(1).strip(), n.group(1).strip()) #: Key is the tuple(PublicKey, CaptchaId) - self.log_debug("Key: %s | ID: %s" % self.key) - return self.key - else: - self.log_warning(_("Key or id pattern not found")) - return None - - - def challenge(self, key=None, data=None): - PublicKey, CaptchaId = key or self.retrieve_key(data) - - html = self.plugin.load("http://api.adscaptcha.com/Get.aspx", - get={'CaptchaId': CaptchaId, - 'PublicKey': PublicKey}) - try: - challenge = re.search("challenge: '(.+?)',", html).group(1) - server = re.search("server: '(.+?)',", html).group(1) - - except AttributeError: - self.fail(_("AdsCaptcha challenge pattern not found")) - - self.log_debug("Challenge: %s" % challenge) - - return self.result(server, challenge), challenge - - - def result(self, server, challenge): - result = self.decrypt_image("%sChallenge.aspx" % server, - get={'cid': challenge, 'dummy': random.random()}, - cookies=True, - input_type="jpg") - - self.log_debug("Result: %s" % result) - - return result diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index af7f66ed5..942021f26 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- +import time + from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.01" + __version__ = "0.02" __status__ = "stable" __description__ = """Base anti-captcha plugin""" @@ -35,13 +37,21 @@ class Captcha(Plugin): pass - def decrypt_image(self, url, get={}, post={}, ref=False, cookies=False, decode=False, - input_type='png', output_type='textual', try_ocr=True): - image = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode) - return self.decrypt(image, input_type, output_type, try_ocr) + def recognize(self, image): + """ + Extend to build your custom anti-captcha ocr + """ + pass + + + def decrypt(self, url, get={}, post={}, ref=False, cookies=False, decode=False, + input_type='png', output_type='textual', ocr=True): + img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode) + return self._decrypt(img, input_type, output_type, ocr) - def decrypt(self, data, input_type='png', output_type='textual', try_ocr=True): + #@TODO: Definitely dhoose a better name for this method! + def _decrypt(self, raw, input_type='png', output_type='textual', ocr=None): """ Loads a captcha and decrypts it with ocr, plugin, user input @@ -53,56 +63,53 @@ class Captcha(Plugin): :param output_type: 'textual' if text is written on the captcha\ or 'positional' for captcha where the user have to click\ on a specific region on the captcha - :param try_ocr: if True, ocr is not used + :param ocr: if True, ocr is not used :return: result of decrypting """ - id = ("%.2f" % time.time())[-6:].replace(".", "") + time_ref = ("%.2f" % time.time())[-6:].replace(".", "") + + with open(os.path.join("tmp", "captcha_image_%s_%s.%s" % (self.plugin.__name__, time_ref, input_type)), "wb") as tmp_img: + tmp_img.write(raw) + + if ocr is not False: + if isinstance(ocr, basestring): + OCR = self.pyload.pluginManager.loadClass("captcha", ocr) #: Rename `captcha` to `ocr` in 0.4.10 - with open(os.path.join("tmp", "tmpCaptcha_%s_%s.%s" % (self.plugin.__name__, id, input_type)), "wb") as tmpCaptcha: - tmpCaptcha.write(img) + if self.plugin.pyfile.abort: + self.abort() - has_plugin = self.plugin.__name__ in self.pyload.pluginManager.ocrPlugins + result = OCR(self.plugin.pyfile).recognize(tmp_img.name) - if self.pyload.captcha: - Ocr = self.pyload.pluginManager.loadClass("ocr", self.plugin.__name__) - else: - Ocr = None + else: + result = self.recognize(tmp_img.name) - if Ocr and try_ocr: - time.sleep(random.randint(3000, 5000) / 1000.0) - if self.pyfile.abort: - self.abort() + else: + captchaManager = self.pyload.captchaManager - ocr = Ocr(self.pyfile) - result = ocr.get_captcha(tmpCaptcha.name) - else: - captchaManager = self.pyload.captchaManager - task = captchaManager.newTask(img, input_type, tmpCaptcha.name, output_type) - self.task = task - captchaManager.handleCaptcha(task) + try: + self.task = captchaManager.newTask(img, input_type, tmp_img.name, output_type) + captchaManager.handleCaptcha(self.task) - while task.isWaiting(): - if self.pyfile.abort: - captchaManager.removeTask(task) - self.abort() - time.sleep(1) + while self.task.isWaiting(): + if self.plugin.pyfile.abort: + self.abort() + time.sleep(1) + finally: + captchaManager.removeTask(self.task) - captchaManager.removeTask(task) + if self.task.error: + self.fail(task.error) - if task.error and has_plugin: #: Ignore default error message since the user could use try_ocr - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) + elif not self.task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) - result = task.result - self.log_debug("Received captcha result: %s" % result) + result = task.result + self.log_debug("Received captcha result: %s" % result) #@TODO: Remove from here? if not self.pyload.debug: try: - os.remove(tmpCaptcha.name) + os.remove(tmp_img.name) except Exception: pass diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index ee5571f77..5c22866c8 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -20,7 +20,7 @@ from module.utils import save_join as fs_join class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" - __version__ = "0.12" + __version__ = "0.13" __status__ = "stable" __description__ = """OCR base plugin""" @@ -133,7 +133,7 @@ class OCR(Plugin): pass - def get_captcha(self, name): + def recognize(self, name): raise NotImplementedError diff --git a/module/plugins/internal/ReCaptcha.py b/module/plugins/internal/ReCaptcha.py deleted file mode 100644 index b4f9ef1eb..000000000 --- a/module/plugins/internal/ReCaptcha.py +++ /dev/null @@ -1,197 +0,0 @@ -# -*- coding: utf-8 -*- - -import random -import re -import time -import urlparse - -from base64 import b64encode - -from module.plugins.internal.CaptchaService import CaptchaService - - -class ReCaptcha(CaptchaService): - __name__ = "ReCaptcha" - __type__ = "captcha" - __version__ = "0.18" - __status__ = "stable" - - __description__ = """ReCaptcha captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com"), - ("zapp-brannigan", "fuerst.reinje@web.de")] - - - KEY_V1_PATTERN = r'(?:recaptcha(?:/api|\.net)/(?:challenge|noscript)\?k=|Recaptcha\.create\s*\(\s*["\'])([\w-]+)' - KEY_V2_PATTERN = r'(?:data-sitekey=["\']|["\']sitekey["\']:\s*["\'])([\w-]+)' - - - def detect_key(self, data=None): - html = data or self.retrieve_data() - - m = re.search(self.KEY_V2_PATTERN, html) or re.search(self.KEY_V1_PATTERN, html) - if m: - self.key = m.group(1).strip() - self.log_debug("Key: %s" % self.key) - return self.key - else: - self.log_warning(_("Key pattern not found")) - return None - - - def challenge(self, key=None, data=None, version=None): - key = key or self.retrieve_key(data) - - if version in (1, 2): - return getattr(self, "_challenge_v%s" % version)(key) - - else: - return self.challenge(key, - version=2 if re.search(self.KEY_V2_PATTERN, html or self.retrieve_data()) else 1) - - - def _challenge_v1(self, key): - html = self.plugin.load("http://www.google.com/recaptcha/api/challenge", - get={'k': key}) - try: - challenge = re.search("challenge : '(.+?)',", html).group(1) - server = re.search("server : '(.+?)',", html).group(1) - - except AttributeError: - self.fail(_("ReCaptcha challenge pattern not found")) - - self.log_debug("Challenge: %s" % challenge) - - return self.result(server, challenge, key) - - - def result(self, server, challenge, key): - self.plugin.load("http://www.google.com/recaptcha/api/js/recaptcha.js") - html = self.plugin.load("http://www.google.com/recaptcha/api/reload", - get={'c' : challenge, - 'k' : key, - 'reason': "i", - 'type' : "image"}) - - try: - challenge = re.search('\(\'(.+?)\',',html).group(1) - - except AttributeError: - self.fail(_("ReCaptcha second challenge pattern not found")) - - self.log_debug("Second challenge: %s" % challenge) - result = self.decrypt("%simage" % server, - get={'c': challenge}, - cookies=True, - input_type="jpg", - try_ocr=False) - - self.log_debug("Result: %s" % result) - - return result, challenge - - - def _collect_api_info(self): - html = self.plugin.load("http://www.google.com/recaptcha/api.js") - a = re.search(r'po.src = \'(.*?)\';', html).group(1) - vers = a.split("/")[5] - - self.log_debug("API version: %s" % vers) - - language = a.split("__")[1].split(".")[0] - - self.log_debug("API language: %s" % language) - - html = self.plugin.load("https://apis.google.com/js/api.js") - b = re.search(r'"h":"(.*?)","', html).group(1) - jsh = b.decode('unicode-escape') - - self.log_debug("API jsh-string: %s" % jsh) - - return vers, language, jsh - - - def _prepare_time_and_rpc(self): - self.plugin.load("http://www.google.com/recaptcha/api2/demo") - - millis = int(round(time.time() * 1000)) - - self.log_debug("Time: %s" % millis) - - rand = random.randint(1, 99999999) - a = "0.%s" % str(rand * 2147483647) - rpc = int(100000000 * float(a)) - - self.log_debug("Rpc-token: %s" % rpc) - - return millis, rpc - - - def _challenge_v2(self, key, parent=None): - if parent is None: - try: - parent = urlparse.urljoin("http://", urlparse.urlparse(self.plugin.pyfile.url).netloc) - - except Exception: - parent = "" - - botguardstring = "!A" - vers, language, jsh = self._collect_api_info() - millis, rpc = self._prepare_time_and_rpc() - - html = self.plugin.load("https://www.google.com/recaptcha/api2/anchor", - get={'k' : key, - 'hl' : language, - 'v' : vers, - 'usegapi' : "1", - 'jsh' : "%s#id=IO_%s" % (jsh, millis), - 'parent' : parent, - 'pfname' : "", - 'rpctoken': rpc}) - - token1 = re.search(r'id="recaptcha-token" value="(.*?)">', html) - self.log_debug("Token #1: %s" % token1.group(1)) - - html = self.plugin.load("https://www.google.com/recaptcha/api2/frame", - get={'c' : token1.group(1), - 'hl' : language, - 'v' : vers, - 'bg' : botguardstring, - 'k' : key, - 'usegapi': "1", - 'jsh' : jsh}, - decode="unicode-escape") - - token2 = re.search(r'"finput","(.*?)",', html) - self.log_debug("Token #2: %s" % token2.group(1)) - - token3 = re.search(r'"rresp","(.*?)",', html) - self.log_debug("Token #3: %s" % token3.group(1)) - - millis_captcha_loading = int(round(time.time() * 1000)) - captcha_response = self.decrypt_image("https://www.google.com/recaptcha/api2/payload", - get={'c':token3.group(1), 'k':key}, - cookies=True, - try_ocr=False) - response = b64encode('{"response":"%s"}' % captcha_response) - - self.log_debug("Result: %s" % response) - - timeToSolve = int(round(time.time() * 1000)) - millis_captcha_loading - timeToSolveMore = timeToSolve + int(float("0." + str(random.randint(1, 99999999))) * 500) - - html = self.plugin.load("https://www.google.com/recaptcha/api2/userverify", - post={'k' : key, - 'c' : token3.group(1), - 'response': response, - 't' : timeToSolve, - 'ct' : timeToSolveMore, - 'bg' : botguardstring}) - - token4 = re.search(r'"uvresp","(.*?)",', html) - self.log_debug("Token #4: %s" % token4.group(1)) - - result = token4.group(1) - - return result, None diff --git a/module/plugins/internal/SolveMedia.py b/module/plugins/internal/SolveMedia.py deleted file mode 100644 index ce4ebb007..000000000 --- a/module/plugins/internal/SolveMedia.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from module.plugins.internal.Plugin import Fail -from module.plugins.internal.CaptchaService import CaptchaService - - -class SolveMedia(CaptchaService): - __name__ = "SolveMedia" - __type__ = "captcha" - __version__ = "0.15" - __status__ = "stable" - - __description__ = """SolveMedia captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] - - - KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']' - - - def detect_key(self, data=None): - html = data or self.retrieve_data() - - m = re.search(self.KEY_PATTERN, html) - if m: - self.key = m.group(1).strip() - self.log_debug("Key: %s" % self.key) - return self.key - else: - self.log_warning(_("Key pattern not found") - return None - - - def challenge(self, key=None, data=None): - key = key or self.retrieve_key(data) - - html = self.plugin.load("http://api.solvemedia.com/papi/challenge.noscript", - get={'k': key}) - - for i in xrange(1, 11): - try: - magic = re.search(r'name="magic" value="(.+?)"', html).group(1) - - except AttributeError: - self.log_warning(_("Magic pattern not found") - magic = None - - try: - challenge = re.search(r'', - html).group(1) - - except AttributeError: - self.fail(_("SolveMedia challenge pattern not found")) - - else: - self.log_debug("Challenge: %s" % challenge) - - try: - result = self.result("http://api.solvemedia.com/papi/media", challenge) - - except Fail, e: - self.log_warning(e) - self.plugin.invalidCaptcha() - result = None - - html = self.plugin.load("http://api.solvemedia.com/papi/verify.noscript", - post={'adcopy_response' : result, - 'k' : key, - 'l' : "en", - 't' : "img", - 's' : "standard", - 'magic' : magic, - 'adcopy_challenge': challenge, - 'ref' : self.plugin.pyfile.url}) - try: - redirect = re.search(r'URL=(.+?)">', html).group(1) - - except AttributeError: - self.fail(_("SolveMedia verify pattern not found")) - - else: - if "error" in html: - self.log_warning(_("Captcha code was invalid")) - self.log_debug("Retry #%d" % i) - html = self.plugin.load(redirect) - else: - break - - else: - self.fail(_("SolveMedia max retries exceeded")) - - return result, challenge - - - def result(self, server, challenge): - result = self.decrypt_image(server, - get={'c': challenge}, - cookies=True, - input_type="gif") - - self.log_debug("Result: %s" % result) - - return result diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index ec9a18a48..18a50a6b0 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -4,8 +4,8 @@ import pycurl import random import re -from module.plugins.internal.ReCaptcha import ReCaptcha -from module.plugins.internal.SolveMedia import SolveMedia +from module.plugins.captcha.ReCaptcha import ReCaptcha +from module.plugins.captcha.SolveMedia import SolveMedia from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, seconds_to_midnight from module.utils import html_unescape @@ -221,7 +221,7 @@ class XFSHoster(SimpleHoster): m = re.search(self.CAPTCHA_PATTERN, self.html) if m: captcha_url = m.group(1) - inputs['code'] = self.captcha.decrypt_image(captcha_url) + inputs['code'] = self.captcha.decrypt(captcha_url) return m = re.search(self.CAPTCHA_BLOCK_PATTERN, self.html, re.S) -- cgit v1.2.3 From 8f7e3150dcf97c2137514d72af69cc253d8c0aed Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 24 Jul 2015 00:16:08 +0200 Subject: [Plugin] Another fix about page decoding --- module/plugins/internal/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 030c372aa..35541f903 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -258,7 +258,7 @@ class Plugin(object): else: req = self.pyload.requestFactory.getRequest(self.__name__) - res = req.load(url, get, post, ref, cookies, just_header, multipart, bool(decode)) + res = req.load(url, get, post, ref, cookies, just_header, multipart, decode is True) if decode: res = html_unescape(res).decode(decode if isinstance(decode, basestring) else 'utf8') -- cgit v1.2.3 From 12cbb11279ebfa2f2945c5adb093b97924a8cfc3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 24 Jul 2015 00:33:19 +0200 Subject: Small __init__ fixes --- module/plugins/internal/Captcha.py | 6 +++--- module/plugins/internal/Extractor.py | 4 ++-- module/plugins/internal/OCR.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 942021f26..1d23eb16e 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -16,8 +16,8 @@ class Captcha(Plugin): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - def __init__(self, plugin): #@TODO: pass pyfile instead plugin, so store plugin's html in its associated pyfile as data - self.pyload = plugin.core + def __init__(self, plugin): #@TODO: Pass pyfile instead plugin, so store plugin's html in its associated pyfile as data + self.pyload = plugin.pyload self.info = {} #: Provide information in dict here self.plugin = plugin @@ -79,7 +79,7 @@ class Captcha(Plugin): if self.plugin.pyfile.abort: self.abort() - result = OCR(self.plugin.pyfile).recognize(tmp_img.name) + result = OCR(self.plugin).recognize(tmp_img.name) else: result = self.recognize(tmp_img.name) diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 3d2859f54..7189ddc25 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -75,7 +75,7 @@ class Extractor(Plugin): return targets - def __init__(self, manager, filename, out, + def __init__(self, plugin, filename, out, fullpath=True, overwrite=False, excludefiles=[], @@ -86,7 +86,7 @@ class Extractor(Plugin): """ Initialize extractor for specific file """ - self.pyload = manager.core + self.pyload = plugin.pyload self.info = {} #: Provide information in dict here self.filename = filename diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 5c22866c8..15d3fc305 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -28,8 +28,8 @@ class OCR(Plugin): __authors__ = [("pyLoad Team", "admin@pyload.org")] - def __init__(self, pyfile): - self.pyload = pyfile.m.core + def __init__(self, plugin): + self.pyload = plugin.pyload self.info = {} #: Provide information in dict here self.init() -- cgit v1.2.3 From f7df6ef48a7c0a8ab6351e046cd12160257c4ef5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 24 Jul 2015 02:15:31 +0200 Subject: Hotfixes --- module/plugins/internal/Captcha.py | 6 ++++-- module/plugins/internal/Container.py | 10 +++++++++- module/plugins/internal/Hoster.py | 18 ++++++++++++------ module/plugins/internal/OCR.py | 7 +++++-- module/plugins/internal/Plugin.py | 31 ++++++++++++++++++++++++++----- 5 files changed, 56 insertions(+), 16 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 1d23eb16e..1345f1784 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import time +import traceback from module.plugins.internal.Plugin import Plugin @@ -110,8 +111,9 @@ class Captcha(Plugin): if not self.pyload.debug: try: os.remove(tmp_img.name) - except Exception: - pass + except OSError, e: + self.log_warning(_("Error removing: %s") % tmp_img.name, e) + traceback.print_exc() return result diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 8dfa1031d..ce9426b0d 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -4,6 +4,7 @@ from __future__ import with_statement import os import re +import traceback from module.plugins.internal.Crypter import Crypter from module.utils import save_join as fs_join @@ -63,5 +64,12 @@ class Container(Crypter): def delete_tmp(self): - if self.pyfile.name.startswith("tmp_"): + if not self.pyfile.name.startswith("tmp_"): + return + + try: os.remove(self.pyfile.url) + except OSError, e: + self.log_warning(_("Error removing: %s") % self.pyfile.url, e) + if self.pyload.debug: + traceback.print_exc() diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index ffd7c4ecc..d26592718 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -6,6 +6,7 @@ import inspect import os import random import time +import traceback import urlparse if os.name != "nt": @@ -14,7 +15,7 @@ if os.name != "nt": from module.plugins.internal import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip - chunks, fixurl as _fixurl, replace_patterns, seconds_to_midnight, + chunks, encode, fixurl as _fixurl, replace_patterns, seconds_to_midnight, set_cookies, parse_html_form, parse_html_tag_attr_value, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join @@ -279,7 +280,7 @@ class Hoster(Plugin): """ Skip and give reason """ - raise Skip(fs_encode(reason)) + raise Skip(encode(reason)) def abort(self, reason=""): @@ -287,7 +288,7 @@ class Hoster(Plugin): Abort and give reason """ if reason: - self.pyfile.error = fs_encode(reason) + self.pyfile.error = encode(reason) raise Abort @@ -296,7 +297,7 @@ class Hoster(Plugin): Fail and indicate file is offline """ if reason: - self.pyfile.error = fs_encode(reason) + self.pyfile.error = encode(reason) raise Fail("offline") @@ -305,7 +306,7 @@ class Hoster(Plugin): Fail and indicates file ist temporary offline, the core may take consequences """ if reason: - self.pyfile.error = fs_encode(reason) + self.pyfile.error = encode(reason) raise Fail("temp. offline") @@ -489,7 +490,12 @@ class Hoster(Plugin): return name finally: if delete and do_delete: - os.remove(lastDownload) + try: + os.remove(lastDownload) + except OSError, e: + self.log_warning(_("Error removing: %s") % lastDownload, e) + if self.pyload.debug: + traceback.print_exc() def direct_link(self, url, follow_location=None): diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 15d3fc305..13089c9c0 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -12,6 +12,7 @@ import logging import os import subprocess # import tempfile +import traceback from module.plugins.internal.Plugin import Plugin from module.utils import save_join as fs_join @@ -129,8 +130,10 @@ class OCR(Plugin): os.remove(tmpTxt.name) if subset and (digits or lowercase or uppercase): os.remove(tmpSub.name) - except Exception: - pass + except OSError, e: + self.log_warning(e) + if self.pyload.debug: + traceback.print_exc() def recognize(self, name): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 35541f903..01cc1ce8a 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -11,6 +11,24 @@ from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as from module.utils import fs_encode, fs_decode, html_unescape, save_join as fs_join +#@TODO: Move to utils in 0.4.10 +def decode(string, encoding='utf8'): + """ Decode string to unicode with utf8 """ + if type(string) is str: + return string.decode(encoding, "replace") + else: + return string + + +#@TODO: Move to utils in 0.4.10 +def encode(string, encoding='utf8'): + """ Decode string to utf8 """ + if type(string) is unicode: + return string.encode(encoding, "replace") + else: + return string + + #@TODO: Move to utils in 0.4.10 def fixurl(url): return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip() @@ -136,7 +154,7 @@ class Plugin(object): def _log(self, level, args): log = getattr(self.pyload.log, level) - msg = fs_encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in args if a)) #@NOTE: `fs_encode` -> `encode` in 0.4.10 + msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in args if a)) #@NOTE: `fs_encode` -> `encode` in 0.4.10 log("%(plugin)s%(id)s: %(msg)s" % {'plugin': self.__name__, 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", 'msg' : msg or _(level.upper() + " MARK")}) @@ -214,7 +232,7 @@ class Plugin(object): """ Fail and give reason """ - raise Fail(fs_encode(reason)) + raise Fail(encode(reason)) #: Move to manager in 0.4.10 def error(self, reason="", type=_("Parse")): @@ -260,8 +278,11 @@ class Plugin(object): res = req.load(url, get, post, ref, cookies, just_header, multipart, decode is True) - if decode: - res = html_unescape(res).decode(decode if isinstance(decode, basestring) else 'utf8') + if decode: #@TODO: Move to network in 0.4.10 + res = html_unescape(res) + + if isinstance(decode, basestring): #@TODO: Move to network in 0.4.10 + res = decode(res, decode) if self.pyload.debug: frame = inspect.currentframe() @@ -272,7 +293,7 @@ class Plugin(object): with open(framefile, "wb") as f: del frame #: Delete the frame or it wont be cleaned - f.write(res.encode('utf8')) + f.write(encode(res)) except IOError, e: self.log_error(e) -- cgit v1.2.3 From 94d017cd2a5c1f194960827a8c7e46afc3682008 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 24 Jul 2015 06:55:49 +0200 Subject: Hotfixes (2) --- module/plugins/internal/Account.py | 6 ++--- module/plugins/internal/Addon.py | 2 +- module/plugins/internal/Captcha.py | 20 ++++++++++------- module/plugins/internal/CaptchaService.py | 2 +- module/plugins/internal/Container.py | 12 +++++++++- module/plugins/internal/Crypter.py | 2 +- module/plugins/internal/DeadCrypter.py | 2 +- module/plugins/internal/DeadHoster.py | 2 +- module/plugins/internal/Extractor.py | 2 +- module/plugins/internal/Hook.py | 2 +- module/plugins/internal/Hoster.py | 37 ++++++++++++++++++------------- module/plugins/internal/MultiHook.py | 6 ++--- module/plugins/internal/MultiHoster.py | 4 ++-- module/plugins/internal/OCR.py | 20 ++++++++--------- module/plugins/internal/Plugin.py | 14 ++++++------ module/plugins/internal/SevenZip.py | 4 ++-- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 16 ++++++------- module/plugins/internal/UnRar.py | 10 ++++----- module/plugins/internal/UnZip.py | 2 +- module/plugins/internal/XFSAccount.py | 2 +- module/plugins/internal/XFSCrypter.py | 2 +- module/plugins/internal/XFSHoster.py | 6 ++--- 23 files changed, 98 insertions(+), 79 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 96f1cb533..158a5c7bc 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -17,7 +17,7 @@ class Account(Plugin): __name__ = "Account" __type__ = "account" __version__ = "0.05" - __status__ = "stable" + __status__ = "testing" __description__ = """Base account plugin""" __license__ = "GPLv3" @@ -170,7 +170,7 @@ class Account(Plugin): try: infos = self.load_account_info(name, self.req) - if not type(infos) == dict: + if not type(infos) is dict: raise Exception("Wrong return format") except Exception, e: @@ -279,7 +279,7 @@ class Account(Plugin): if self.infos[user]['validuntil'] > 0 and time.time() > self.infos[user]['validuntil']: continue if "trafficleft" in self.infos[user]: - if self.infos[user]['trafficleft'] == 0: + if self.infos[user]['trafficleft'] is 0: continue usable.append((user, data)) diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index b29aa0936..23512d7c4 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -26,7 +26,7 @@ class Addon(Plugin): __name__ = "Addon" __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 __version__ = "0.01" - __status__ = "stable" + __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] __threaded__ = [] #@TODO: Remove in 0.4.10 diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 1345f1784..89cace612 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -9,8 +9,8 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.02" - __status__ = "stable" + __version__ = "0.31" + __status__ = "testing" __description__ = """Base anti-captcha plugin""" __license__ = "GPLv3" @@ -27,8 +27,8 @@ class Captcha(Plugin): self.init() - def _log(self, type, args): - return super(Captcha, self)._log(type, (self.plugin.__name__,) + args) + def _log(self, level, args): + return self.plugin._log(level, (self.__name__,) + args) def init(self): @@ -119,12 +119,16 @@ class Captcha(Plugin): def invalid(self): + if not self.task: + return + self.log_error(_("Invalid captcha")) - if self.task: - self.task.invalid() + self.task.invalid() def correct(self): + if not self.task: + return + self.log_info(_("Correct captcha")) - if self.task: - self.task.correct() + self.task.correct() diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 05af8ccec..971158a36 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -7,7 +7,7 @@ class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" __version__ = "0.31" - __status__ = "stable" + __status__ = "testing" __description__ = """Base anti-captcha service plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index ce9426b0d..cc732247e 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -14,7 +14,7 @@ class Container(Crypter): __name__ = "Container" __type__ = "container" __version__ = "0.05" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] @@ -39,6 +39,11 @@ class Container(Crypter): self._create_packages() + #: Deprecated method, use `_load2disk` instead + def loadToDisk(self, *args, **kwargs): + return self._load2disk(*args, **kwargs) + + def _load2disk(self): """ Loads container to disk if its stored remotely and overwrite url, @@ -63,6 +68,11 @@ class Container(Crypter): self.fail(_("File not exists")) + #: Deprecated method, use `delete_tmp` instead + def deleteTmp(self, *args, **kwargs): + return self.delete_tmp(*args, **kwargs) + + def delete_tmp(self): if not self.pyfile.name.startswith("tmp_"): return diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 627c198e1..715791949 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -10,7 +10,7 @@ class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" __version__ = "0.05" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides pyload.config.get("general", "folder_per_package") diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 5232403b8..7d14f5378 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -7,7 +7,7 @@ class DeadCrypter(Crypter): __name__ = "DeadCrypter" __type__ = "crypter" __version__ = "0.08" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index b0c179ed1..f8e68f2b3 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -7,7 +7,7 @@ class DeadHoster(Hoster): __name__ = "DeadHoster" __type__ = "hoster" __version__ = "0.18" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 7189ddc25..668207ea3 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -23,7 +23,7 @@ class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" __version__ = "0.26" - __status__ = "stable" + __status__ = "testing" __description__ = """Base extractor plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index c3b14ef73..fbd6934ce 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -7,7 +7,7 @@ class Hook(Addon): __name__ = "Hook" __type__ = "hook" __version__ = "0.10" - __status__ = "stable" + __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index d26592718..1cc127522 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -13,12 +13,12 @@ if os.name != "nt": import grp import pwd -from module.plugins.internal import Captcha -from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip +from module.plugins.internal.Captcha import Captcha +from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, chunks, encode, fixurl as _fixurl, replace_patterns, seconds_to_midnight, set_cookies, parse_html_form, parse_html_tag_attr_value, timestamp) -from module.utils import fs_decode, fs_encode, save_join as fs_join +from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename #@TODO: Remove in 0.4.10 @@ -48,7 +48,7 @@ class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" __version__ = "0.05" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] @@ -66,11 +66,11 @@ class Hoster(Plugin): self.info = {} #: Provide information in dict here #: Engage wan reconnection - self.want_reconnect = False + self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10 #: Enable simultaneous processing of multiple downloads - self.multi_dl = True - self.limit_dl = 0 + self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10 + self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 #: Chunk limit self.chunk_limit = 1 @@ -206,8 +206,8 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): reconnect = bool(reconnect) - self.log_debug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.want_reconnect)) - self.want_reconnect = reconnect + self.log_debug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + self.wantReconnect = reconnect def set_wait(self, seconds, reconnect=None): @@ -247,7 +247,7 @@ class Hoster(Plugin): pyfile.setStatus("waiting") self.log_info(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), - _("Reconnect: %s") % self.want_reconnect) + _("Reconnect: %s") % self.wantReconnect) if self.account: self.log_debug("Ignore reconnection due account logged") @@ -266,7 +266,7 @@ class Hoster(Plugin): if self.thread.m.reconnecting.isSet(): self.waiting = False - self.want_reconnect = False + self.wantReconnect = False raise Reconnect time.sleep(1) @@ -396,7 +396,7 @@ class Hoster(Plugin): filename = os.path.join(location, name) - self.pyload.addonManager.dispatchEvent("download-start", self.pyfile, url, filename) + self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename) try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, @@ -532,7 +532,7 @@ class Hoster(Plugin): value = value.strip() if key in header: - if type(header[key]) == list: + if type(header[key]) is list: header[key].append(value) else: header[key] = [header[key], value] @@ -550,7 +550,7 @@ class Hoster(Plugin): baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) location = urlparse.urljoin(baseurl, location) - if 'code' in header and header['code'] == 302: + if 'code' in header and header['code'] is 302: link = location if follow_location: @@ -597,7 +597,7 @@ class Hoster(Plugin): if traffic is None: return False - elif traffic == -1: + elif traffic is -1: return True else: size = self.pyfile.size / 1024 @@ -612,6 +612,11 @@ class Hoster(Plugin): return self.pyfile.package().password or "" + #: Deprecated method, use `check_for_same_files` instead + def checkForSameFiles(self, *args, **kwargs): + return self.check_for_same_files(*args, **kwargs) + + def check_for_same_files(self, starting=False): """ Checks if same file was/is downloaded within same package @@ -622,7 +627,7 @@ class Hoster(Plugin): pack = self.pyfile.package() for pyfile in self.pyload.files.cache.values(): - if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile != self.pyfile and pyfile.name is self.pyfile.name and pyfile.package().folder is pack.folder: if pyfile.status in (0, 12): #: Finished or downloading self.skip(pyfile.pluginname) elif pyfile.status in (5, 7) and starting: #: A download is waiting/starting and was appenrently started before diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 09f91bb2a..32d83e1ca 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -12,7 +12,7 @@ class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" __version__ = "0.51" - __status__ = "stable" + __status__ = "testing" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), @@ -119,7 +119,7 @@ class MultiHook(Hook): pluginlist = self.get_config('pluginlist', '').replace('|', ',').replace(';', ',').split(',') configset = self._plugin_set(pluginlist) - if configmode == "listed": + if configmode is "listed": pluginset &= configset else: pluginset -= configset @@ -186,7 +186,7 @@ class MultiHook(Hook): def override_plugins(self): excludedList = [] - if self.plugintype == "hoster": + if self.plugintype is "hoster": pluginMap = dict((name.lower(), name) for name in self.pyload.pluginManager.hosterPlugins.iterkeys()) accountList = [account.type.lower() for account in self.pyload.api.getAccounts(False) if account.valid and account.premium] else: diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 2fc319102..fe6fa29d6 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -10,7 +10,7 @@ class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" __version__ = "0.43" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' __config__ = [("use_premium" , "bool", "Use premium account if available" , True), @@ -26,7 +26,7 @@ class MultiHoster(SimpleHoster): def setup(self): self.chunk_limit = 1 - self.multi_dl = bool(self.account) + self.multiDL = bool(self.account) self.resume_download = self.premium diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 13089c9c0..064bf1d7d 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -22,7 +22,7 @@ class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" __version__ = "0.13" - __status__ = "stable" + __status__ = "testing" __description__ = """OCR base plugin""" __license__ = "GPLv3" @@ -88,7 +88,7 @@ class OCR(Plugin): self.pyload.log_debug("Saving tiff...") self.image.save(tmpTif.name, 'TIFF') - if os.name == "nt": + if os.name is "nt": tessparams = [os.path.join(pypath, "tesseract", "tesseract.exe")] else: tessparams = ["tesseract"] @@ -165,7 +165,7 @@ class OCR(Plugin): for x in xrange(w): for y in xrange(h): - if pixels[x, y] == 255: + if pixels[x, y] is 255: continue #: No point in processing white pixels since we only want to remove black pixel count = 0 @@ -198,7 +198,7 @@ class OCR(Plugin): #: Second pass: this time set all 1's to 255 (white) for x in xrange(w): for y in xrange(h): - if pixels[x, y] == 1: + if pixels[x, y] is 1: pixels[x, y] = 255 self.pixels = pixels @@ -213,7 +213,7 @@ class OCR(Plugin): for x in xrange(w): for y in xrange(h): - if pixels[x, y] == 0: + if pixels[x, y] is 0: pixels[x, y] = 155 highest = {} @@ -229,7 +229,7 @@ class OCR(Plugin): for x in xrange(w): for y in xrange(h): - if pixels[x, y] == 0: + if pixels[x, y] is 0: pixels[x, y] = 255 count = {} @@ -237,7 +237,7 @@ class OCR(Plugin): for x in xrange(w): count[x] = 0 for y in xrange(h): - if pixels[x, y] == 155: + if pixels[x, y] is 155: count[x] += 1 sum = 0 @@ -270,10 +270,10 @@ class OCR(Plugin): for x in xrange(w): for y in xrange(h): - if pixels[x, y] == 0: + if pixels[x, y] is 0: pixels[x, y] = 255 - if pixels[x, y] == 155: + if pixels[x, y] is 155: pixels[x, y] = 0 self.pixels = pixels @@ -327,7 +327,7 @@ class OCR(Plugin): for key, item in values.iteritems(): - if key.__class__ == str: + if key.__class__ is str: result = result.replace(key, item) else: for expr in key: diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 01cc1ce8a..e70c099b1 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -66,7 +66,7 @@ def replace_patterns(string, ruleslist): def set_cookies(cj, cookies): for cookie in cookies: - if isinstance(cookie, tuple) and len(cookie) == 3: + if isinstance(cookie, tuple) and len(cookie) is 3: domain, name, value = cookie cj.setCookie(domain, name, value) @@ -95,7 +95,7 @@ def parse_html_form(attr_str, html, input_names={}): #: Check input attributes for key, val in input_names.iteritems(): if key in inputs: - if isinstance(val, basestring) and inputs[key] == val: + if isinstance(val, basestring) and inputs[key] is val: continue elif isinstance(val, tuple) and inputs[key] in val: continue @@ -126,7 +126,7 @@ class Plugin(object): __name__ = "Plugin" __type__ = "hoster" __version__ = "0.13" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] @@ -232,7 +232,7 @@ class Plugin(object): """ Fail and give reason """ - raise Fail(encode(reason)) #: Move to manager in 0.4.10 + raise Fail(encode(reason)) #: Move `encode(reason)` to manager in 0.4.10 def error(self, reason="", type=_("Parse")): @@ -246,7 +246,7 @@ class Plugin(object): raise Fail(msg) - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=True, multipart=True, req=None): + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=True, req=None): """ Load content at url and returns it @@ -276,7 +276,7 @@ class Plugin(object): else: req = self.pyload.requestFactory.getRequest(self.__name__) - res = req.load(url, get, post, ref, cookies, just_header, multipart, decode is True) + res = req.load(url, get, post, ref, cookies, just_header, isinstance(post, dict), decode is True) #@TODO: Fix network multipart in 0.4.10 if decode: #@TODO: Move to network in 0.4.10 res = html_unescape(res) @@ -310,7 +310,7 @@ class Plugin(object): value = value.strip() if key in header: - if type(header[key]) == list: + if type(header[key]) is list: header[key].append(value) else: header[key] = [header[key], value] diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index f2194157c..dde12d17e 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -11,7 +11,7 @@ from module.utils import fs_encode, save_join as fs_join class SevenZip(UnRar): __name__ = "SevenZip" __version__ = "0.13" - __status__ = "stable" + __status__ = "testing" __description__ = """7-Zip extractor plugin""" __license__ = "GPLv3" @@ -40,7 +40,7 @@ class SevenZip(UnRar): @classmethod def find(cls): try: - if os.name == "nt": + if os.name is "nt": cls.CMD = os.path.join(pypath, "7z.exe") p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index efda7dc1a..dd3b97e91 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -11,7 +11,7 @@ class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" __version__ = "0.56" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides pyload.config['general']['folder_per_package'] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 5960794cc..2a5263449 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -25,7 +25,7 @@ class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" __version__ = "1.71" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -199,7 +199,7 @@ class SimpleHoster(Hoster): def setup(self): - self.resume_download = self.multi_dl = self.premium + self.resume_download = self.multiDL = self.premium def prepare(self): @@ -287,7 +287,7 @@ class SimpleHoster(Hoster): except Fail, e: #@TODO: Move to PluginThread in 0.4.10 err = str(e) #@TODO: Recheck in 0.4.10 - if err == _("No captcha result obtained in appropiate time by any of the plugins."): #@TODO: Fix in 0.4.10 + if err is _("No captcha result obtained in appropiate time by any of the plugins."): #@TODO: Fix in 0.4.10 self.check_file() elif self.get_config('fallback', True) and self.premium: @@ -322,7 +322,7 @@ class SimpleHoster(Hoster): pass self.log_warning(_("Check result: ") + errmsg, _("Waiting 1 minute and retry")) - self.want_reconnect = True + self.wantReconnect = True self.retry(wait_time=60, reason=errmsg) else: if self.CHECK_FILE: @@ -365,11 +365,11 @@ class SimpleHoster(Hoster): wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) - self.want_reconnect = wait_time > 300 + self.wantReconnect = wait_time > 300 self.retry(1, wait_time, _("Download limit exceeded")) if hasattr(self, 'HAPPY_HOUR_PATTERN') and re.search(self.HAPPY_HOUR_PATTERN, self.html): - self.multi_dl = True + self.multiDL = True if hasattr(self, 'ERROR_PATTERN'): m = re.search(self.ERROR_PATTERN, self.html) @@ -389,7 +389,7 @@ class SimpleHoster(Hoster): wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) - self.want_reconnect = wait_time > 300 + self.wantReconnect = wait_time > 300 self.retry(1, wait_time, _("Download limit exceeded")) elif re.search('country|ip|region|nation', errmsg, re.I): @@ -420,7 +420,7 @@ class SimpleHoster(Hoster): self.fail(_("File can be downloaded by premium users only")) else: - self.want_reconnect = True + self.wantReconnect = True self.retry(wait_time=60, reason=errmsg) elif hasattr(self, 'WAIT_PATTERN'): diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 1a2e64b94..90b6431ab 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -23,7 +23,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" __version__ = "1.22" - __status__ = "stable" + __status__ = "testing" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -51,7 +51,7 @@ class UnRar(Extractor): @classmethod def find(cls): try: - if os.name == "nt": + if os.name is "nt": cls.CMD = os.path.join(pypath, "RAR.exe") else: cls.CMD = "rar" @@ -63,7 +63,7 @@ class UnRar(Extractor): except OSError: try: - if os.name == "nt": + if os.name is "nt": cls.CMD = os.path.join(pypath, "UnRAR.exe") else: cls.CMD = "unrar" @@ -134,7 +134,7 @@ class UnRar(Extractor): if not c: break #: Reading a percentage sign -> set progress and restart - if c == '%': + if c is '%': self.notify_progress(int(s)) s = "" #: Not reading a digit -> therefore restart @@ -180,7 +180,7 @@ class UnRar(Extractor): #: eventually Multipart Files files.extend(fs_join(dir, os.path.basename(file)) for file in filter(self.is_multipart, os.listdir(dir)) - if re.sub(self.re_multipart, ".rar", name) == re.sub(self.re_multipart, ".rar", file)) + if re.sub(self.re_multipart, ".rar", name) is re.sub(self.re_multipart, ".rar", file)) return files diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index ef2b2b56d..aadc303aa 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -13,7 +13,7 @@ from module.utils import fs_encode class UnZip(Extractor): __name__ = "UnZip" __version__ = "1.14" - __status__ = "stable" + __status__ = "testing" __description__ = """Zip extractor plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index b5028eda6..8612bad63 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" __version__ = "0.39" - __status__ = "stable" + __status__ = "testing" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 7470eb221..fcb017466 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -7,7 +7,7 @@ class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" __version__ = "0.11" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 18a50a6b0..48772a817 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" __version__ = "0.54" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -53,7 +53,7 @@ class XFSHoster(SimpleHoster): def setup(self): self.chunk_limit = -1 if self.premium else 1 - self.resume_download = self.multi_dl = self.premium + self.resume_download = self.multiDL = self.premium def prepare(self): @@ -152,7 +152,7 @@ class XFSHoster(SimpleHoster): stmsg = inputs['st'] - if stmsg == 'OK': + if stmsg is 'OK': self.html = self.load(action, post=inputs) elif 'Can not leech file' in stmsg: -- cgit v1.2.3 From 761ca5c66e07559925ebbdbc6531f9ca658b12ce Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 24 Jul 2015 16:11:58 +0200 Subject: Code cosmetics --- module/plugins/internal/Account.py | 6 +++--- module/plugins/internal/Crypter.py | 2 +- module/plugins/internal/Hoster.py | 6 +++--- module/plugins/internal/MultiHook.py | 4 ++-- module/plugins/internal/OCR.py | 16 ++++++++-------- module/plugins/internal/Plugin.py | 4 ++-- module/plugins/internal/SevenZip.py | 2 +- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 18 +++++++++--------- module/plugins/internal/UnRar.py | 6 +++--- module/plugins/internal/XFSHoster.py | 2 +- 11 files changed, 34 insertions(+), 34 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 158a5c7bc..eaf988974 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -126,7 +126,7 @@ class Account(Plugin): if options: before = self.accounts[user]['options'] self.accounts[user]['options'].update(options) - return self.accounts[user]['options'] != before + return self.accounts[user]['options'] not is before else: self.accounts[user] = {'password': password, 'options': options, 'valid': True} self._login(user, self.accounts[user]) @@ -279,7 +279,7 @@ class Account(Plugin): if self.infos[user]['validuntil'] > 0 and time.time() > self.infos[user]['validuntil']: continue if "trafficleft" in self.infos[user]: - if self.infos[user]['trafficleft'] is 0: + if self.infos[user]['trafficleft'] == 0: continue usable.append((user, data)) @@ -291,7 +291,7 @@ class Account(Plugin): def can_use(self): - return self.select_account() != (None, None) + return self.select_account() not is (None, None) def parse_traffic(self, value, unit=None): #: Return kilobytes diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 715791949..9b09fc9c1 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -91,7 +91,7 @@ class Crypter(Hoster): setFolder(package_folder) self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) - elif not folder_per_package or name != folder: + elif not folder_per_package or name not is folder: if not folder: folder = urlparse.urlparse(name).path.split("/")[-1] diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 1cc127522..dbc5e88a8 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -408,7 +408,7 @@ class Hoster(Plugin): if newname: newname = urlparse.urlparse(newname).path.split('/')[-1] - if disposition and newname != name: + if disposition and newname not is name: self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) self.pyfile.name = newname filename = os.path.join(location, newname) @@ -550,7 +550,7 @@ class Hoster(Plugin): baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) location = urlparse.urljoin(baseurl, location) - if 'code' in header and header['code'] is 302: + if 'code' in header and header['code'] == 302: link = location if follow_location: @@ -597,7 +597,7 @@ class Hoster(Plugin): if traffic is None: return False - elif traffic is -1: + elif traffic == -1: return True else: size = self.pyfile.size / 1024 diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 32d83e1ca..0fa75df98 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -119,7 +119,7 @@ class MultiHook(Hook): pluginlist = self.get_config('pluginlist', '').replace('|', ',').replace(';', ',').split(',') configset = self._plugin_set(pluginlist) - if configmode is "listed": + if configmode == "listed": pluginset &= configset else: pluginset -= configset @@ -186,7 +186,7 @@ class MultiHook(Hook): def override_plugins(self): excludedList = [] - if self.plugintype is "hoster": + if self.plugintype == "hoster": pluginMap = dict((name.lower(), name) for name in self.pyload.pluginManager.hosterPlugins.iterkeys()) accountList = [account.type.lower() for account in self.pyload.api.getAccounts(False) if account.valid and account.premium] else: diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 064bf1d7d..36d259e0c 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -88,7 +88,7 @@ class OCR(Plugin): self.pyload.log_debug("Saving tiff...") self.image.save(tmpTif.name, 'TIFF') - if os.name is "nt": + if os.name == "nt": tessparams = [os.path.join(pypath, "tesseract", "tesseract.exe")] else: tessparams = ["tesseract"] @@ -165,7 +165,7 @@ class OCR(Plugin): for x in xrange(w): for y in xrange(h): - if pixels[x, y] is 255: + if pixels[x, y] == 255: continue #: No point in processing white pixels since we only want to remove black pixel count = 0 @@ -198,7 +198,7 @@ class OCR(Plugin): #: Second pass: this time set all 1's to 255 (white) for x in xrange(w): for y in xrange(h): - if pixels[x, y] is 1: + if pixels[x, y] == 1: pixels[x, y] = 255 self.pixels = pixels @@ -213,7 +213,7 @@ class OCR(Plugin): for x in xrange(w): for y in xrange(h): - if pixels[x, y] is 0: + if pixels[x, y] == 0: pixels[x, y] = 155 highest = {} @@ -229,7 +229,7 @@ class OCR(Plugin): for x in xrange(w): for y in xrange(h): - if pixels[x, y] is 0: + if pixels[x, y] == 0: pixels[x, y] = 255 count = {} @@ -237,7 +237,7 @@ class OCR(Plugin): for x in xrange(w): count[x] = 0 for y in xrange(h): - if pixels[x, y] is 155: + if pixels[x, y] == 155: count[x] += 1 sum = 0 @@ -270,10 +270,10 @@ class OCR(Plugin): for x in xrange(w): for y in xrange(h): - if pixels[x, y] is 0: + if pixels[x, y] == 0: pixels[x, y] = 255 - if pixels[x, y] is 155: + if pixels[x, y] == 155: pixels[x, y] = 0 self.pixels = pixels diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index e70c099b1..9ac89501c 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -42,7 +42,7 @@ def timestamp(): def seconds_to_midnight(gmt=0): now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) - if now.hour is 0 and now.minute < 10: + if now.hour == 0 and now.minute < 10: midnight = now else: midnight = now + datetime.timedelta(days=1) @@ -66,7 +66,7 @@ def replace_patterns(string, ruleslist): def set_cookies(cj, cookies): for cookie in cookies: - if isinstance(cookie, tuple) and len(cookie) is 3: + if isinstance(cookie, tuple) and len(cookie) == 3: domain, name, value = cookie cj.setCookie(domain, name, value) diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index dde12d17e..a5ccf2a26 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -40,7 +40,7 @@ class SevenZip(UnRar): @classmethod def find(cls): try: - if os.name is "nt": + if os.name == "nt": cls.CMD = os.path.join(pypath, "7z.exe") p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index dd3b97e91..7fa399276 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -132,7 +132,7 @@ class SimpleCrypter(Crypter, SimpleHoster): try: url = self.info['url'].strip() name = self.info['name'].strip() - if name and name != url: + if name and name not is url: self.pyfile.name = name except Exception: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2a5263449..f86e6fbe0 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -121,7 +121,7 @@ class SimpleHoster(Hoster): @classmethod def get_info(cls, url="", html=""): info = cls.api_info(url) - online = True if info['status'] is 2 else False + online = True if info['status'] == 2 else False try: info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: Pattern groups will be saved here @@ -134,17 +134,17 @@ class SimpleHoster(Hoster): info['error'] = "missing url" info['status'] = 1 - elif info['status'] is 3: + elif info['status'] == 3: try: html = get_url(url, cookies=cls.COOKIES, decode=cls.TEXT_ENCODING) except BadHeader, e: info['error'] = "%d: %s" % (e.code, e.content) - if e.code is 404: + if e.code == 404: info['status'] = 1 - elif e.code is 503: + elif e.code == 503: info['status'] = 6 except Exception: @@ -232,7 +232,7 @@ class SimpleHoster(Hoster): self.LINK_PREMIUM_PATTERN = self.LINK_PATTERN if (self.MULTI_HOSTER - and (self.__pattern__ != self.pyload.pluginManager.hosterPlugins[self.__name__]['pattern'] + and (self.__pattern__ not is self.pyload.pluginManager.hosterPlugins[self.__name__]['pattern'] or re.match(self.__pattern__, self.pyfile.url) is None)): self.multihost = True return @@ -448,13 +448,13 @@ class SimpleHoster(Hoster): try: status = self.info['status'] - if status is 1: + if status == 1: self.offline() - elif status is 6: + elif status == 6: self.temp_offline() - elif status is 8: + elif status == 8: self.fail(self.info['error'] if 'error' in self.info else _("Failed")) finally: @@ -471,7 +471,7 @@ class SimpleHoster(Hoster): try: url = self.info['url'].strip() name = self.info['name'].strip() - if name and name != url: + if name and name not is url: self.pyfile.name = name except Exception: diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 90b6431ab..a2c2070b8 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -51,7 +51,7 @@ class UnRar(Extractor): @classmethod def find(cls): try: - if os.name is "nt": + if os.name == "nt": cls.CMD = os.path.join(pypath, "RAR.exe") else: cls.CMD = "rar" @@ -63,7 +63,7 @@ class UnRar(Extractor): except OSError: try: - if os.name is "nt": + if os.name == "nt": cls.CMD = os.path.join(pypath, "UnRAR.exe") else: cls.CMD = "unrar" @@ -134,7 +134,7 @@ class UnRar(Extractor): if not c: break #: Reading a percentage sign -> set progress and restart - if c is '%': + if c == "%": self.notify_progress(int(s)) s = "" #: Not reading a digit -> therefore restart diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 48772a817..6508b9bd3 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -152,7 +152,7 @@ class XFSHoster(SimpleHoster): stmsg = inputs['st'] - if stmsg is 'OK': + if stmsg == 'OK': self.html = self.load(action, post=inputs) elif 'Can not leech file' in stmsg: -- cgit v1.2.3 From dd13825fbd3df9e441200638cd2a92e3924dfff6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 24 Jul 2015 23:57:04 +0200 Subject: Fix typo --- module/plugins/internal/Account.py | 4 ++-- module/plugins/internal/Crypter.py | 2 +- module/plugins/internal/Hoster.py | 2 +- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index eaf988974..03521a0ea 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -126,7 +126,7 @@ class Account(Plugin): if options: before = self.accounts[user]['options'] self.accounts[user]['options'].update(options) - return self.accounts[user]['options'] not is before + return self.accounts[user]['options'] is not before else: self.accounts[user] = {'password': password, 'options': options, 'valid': True} self._login(user, self.accounts[user]) @@ -291,7 +291,7 @@ class Account(Plugin): def can_use(self): - return self.select_account() not is (None, None) + return self.select_account() is not (None, None) def parse_traffic(self, value, unit=None): #: Return kilobytes diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 9b09fc9c1..7e97c8228 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -91,7 +91,7 @@ class Crypter(Hoster): setFolder(package_folder) self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) - elif not folder_per_package or name not is folder: + elif not folder_per_package or name is not folder: if not folder: folder = urlparse.urlparse(name).path.split("/")[-1] diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index dbc5e88a8..0d2e4dcc4 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -408,7 +408,7 @@ class Hoster(Plugin): if newname: newname = urlparse.urlparse(newname).path.split('/')[-1] - if disposition and newname not is name: + if disposition and newname is not name: self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) self.pyfile.name = newname filename = os.path.join(location, newname) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 7fa399276..f5cfa289d 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -132,7 +132,7 @@ class SimpleCrypter(Crypter, SimpleHoster): try: url = self.info['url'].strip() name = self.info['name'].strip() - if name and name not is url: + if name and name is not url: self.pyfile.name = name except Exception: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index f86e6fbe0..5ba207934 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -232,7 +232,7 @@ class SimpleHoster(Hoster): self.LINK_PREMIUM_PATTERN = self.LINK_PATTERN if (self.MULTI_HOSTER - and (self.__pattern__ not is self.pyload.pluginManager.hosterPlugins[self.__name__]['pattern'] + and (self.__pattern__ is not self.pyload.pluginManager.hosterPlugins[self.__name__]['pattern'] or re.match(self.__pattern__, self.pyfile.url) is None)): self.multihost = True return @@ -471,7 +471,7 @@ class SimpleHoster(Hoster): try: url = self.info['url'].strip() name = self.info['name'].strip() - if name and name not is url: + if name and name is not url: self.pyfile.name = name except Exception: -- cgit v1.2.3 From 8f17f875f6e28f73ddb10da59c6464bd04922222 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Jul 2015 04:59:27 +0200 Subject: Account rewritten --- module/plugins/internal/Account.py | 328 +++++++++++++++--------------- module/plugins/internal/Addon.py | 30 +-- module/plugins/internal/CaptchaService.py | 4 +- module/plugins/internal/Container.py | 4 +- module/plugins/internal/Hoster.py | 8 +- module/plugins/internal/SimpleCrypter.py | 4 +- module/plugins/internal/SimpleHoster.py | 4 +- module/plugins/internal/XFSAccount.py | 6 +- 8 files changed, 190 insertions(+), 198 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 03521a0ea..3a3fd4768 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- +from __future__ import with_statement + import random -import threading import time import traceback @@ -9,10 +10,6 @@ from module.plugins.internal.Plugin import Plugin from module.utils import compare_time, lock, parseFileSize as parse_size -class WrongPassword(Exception): - pass - - class Account(Plugin): __name__ = "Account" __type__ = "account" @@ -21,25 +18,20 @@ class Account(Plugin): __description__ = """Base account plugin""" __license__ = "GPLv3" - __authors__ = [("mkaay", "mkaay@mkaay.de")] + __authors__ = [("mkaay" , "mkaay@mkaay.de" ), + ("Walter Purcaro", "vuolter@gmail.com")] + + + LOGIN_TIMEOUT = 10 * 60 #: After that time (in minutes) pyload will relogin the account + INFO_THRESHOLD = 10 * 60 #: After that time (in minutes) account data will be reloaded def __init__(self, manager, accounts): self.pyload = manager.core self.info = {} #: Provide information in dict here - self.manager = manager - self.accounts = {} - self.infos = {} #: Cache for account information - self.lock = threading.RLock() - self.timestamps = {} - - self.login_timeout = 10 * 60 #: After that time (in minutes) pyload will relogin the account - self.info_threshold = 10 * 60 #: After that time (in minutes) account data will be reloaded - self.init() - - self.set_accounts(accounts) + self.init_accounts(accounts) def init(self): @@ -49,240 +41,239 @@ class Account(Plugin): pass - def login(self, user, data, req): + def login(self, user, password, data, req): """ Login into account, the cookies will be saved so user can be recognized - - :param user: loginname - :param data: data dictionary - :param req: `Request` instance """ pass @lock - def _login(self, user, data): - #: Set timestamp for login - self.timestamps[user] = time.time() - - self.req = self.get_account_request(user) + def _login(self, user): try: - self.login(user, data, self.req) + info = self.info[user] + info['login']['timestamp'] = time.time() #: Set timestamp for login - except WrongPassword: - self.log_warning( - _("Could not login with account %(user)s | %(msg)s") % {'user': user, - 'msg': _("Wrong Password")}) - success = data['valid'] = False + self.req = self.get_request(user) + self.login(user, info['login']['password'], info['data'], self.req) except Exception, e: self.log_warning( _("Could not login with account %(user)s | %(msg)s") % {'user': user, - 'msg': e}) - success = data['valid'] = False + 'msg' : e}) + res = info['login']['valid'] = False if self.pyload.debug: traceback.print_exc() else: - success = True + res = True finally: - if hasattr(self, "req"): - if self.req: - self.req.close() - del self.req - return success + if self.req: + self.req.close() + del self.req + + return res def relogin(self, user): - req = self.get_account_request(user) - if req: + with self.get_request(user) as req: req.clearCookies() - req.close() - if user in self.infos: - del self.infos[user] #: Delete old information + if user in self.info: + self.info[user]['login'].clear() - return self._login(user, self.accounts[user]) + return self._login(user) - def set_accounts(self, accounts): - self.accounts = accounts - for user, data in self.accounts.iteritems(): - self._login(user, data) - self.infos[user] = {} + def init_accounts(self, accounts): + for user, data in accounts.iteritems(): + self.add(user, data['password'], data['options']) + self._login(user) - def update_accounts(self, user, password=None, options={}): + def add(self, user, password=None, options): + if user not in self.info: + self.info[user] = {'login': {'valid': True, 'password': password or "", 'timestamp': 0}, #@NOTE: Do not remove `'valid': True` in 0.4.9 or accounts will not login + 'data' : {'options': options, 'timestamp': 0}} + self._login(user) + return True + else: + self.log_error(_("Error adding account"), _("User already exists")) + + + def update(self, user, password=None, options): """ Updates account and return true if anything changed """ - if user in self.accounts: - self.accounts[user]['valid'] = True #: Do not remove or accounts will not login + if user not in self.info: + return self.add(user, password, options) + + elif password or options: if password: - self.accounts[user]['password'] = password + self.info[user]['login']['password'] = password self.relogin(user) return True + if options: - before = self.accounts[user]['options'] - self.accounts[user]['options'].update(options) - return self.accounts[user]['options'] is not before - else: - self.accounts[user] = {'password': password, 'options': options, 'valid': True} - self._login(user, self.accounts[user]) - return True + before = self.info[user]['data'][user]['options'] + self.info[user]['data']['options'].update(options) + return self.info[user]['data']['options'] != before - #: Deprecated method, use `update_accounts` instead + #: Deprecated method, use `update` instead (Remove in 0.4.10) def updateAccounts(self, *args, **kwargs): - return self.update_accounts(*args, **kwargs) + return self.update(*args, **kwargs) + + def remove(self, user=None): # -> def remove + if not user: + self.info.clear() - def remove_account(self, user): - if user in self.accounts: - del self.accounts[user] - if user in self.infos: - del self.infos[user] - if user in self.timestamps: - del self.timestamps[user] + elif user in self.info: + del self.info[user] - #: Deprecated method, use `remove_account` instead + #: Deprecated method, use `remove` instead (Remove in 0.4.10) def removeAccount(self, *args, **kwargs): - return self.remove_account(*args, **kwargs) + return self.remove(*args, **kwargs) + + + def get_data(self, user, reload=False): + return self.get_info(user, reload)['data'] @lock - def get_account_info(self, name, force=False): + def get_info(self, user, reload=False): """ Retrieve account infos for an user, do **not** overwrite this method!\\ - just use it to retrieve infos in hoster plugins. see `load_account_info` + just use it to retrieve infos in hoster plugins. see `load_info` - :param name: username - :param force: reloads cached account information + :param user: username + :param reload: reloads cached account information :return: dictionary with information """ - data = Account.load_account_info(self, name) + if not reload and user in self.info: + info = self.info[user] - if force or name not in self.infos: - self.log_debug("Get Account Info for %s" % name) - self.req = self.get_account_request(name) + if info['data']['timestamp'] + self.INFO_THRESHOLD * 60 < time.time(): + self.log_debug("Reached timeout for account data") + self.schedule_refresh(user) - try: - infos = self.load_account_info(name, self.req) - if not type(infos) is dict: - raise Exception("Wrong return format") + else: + self.log_debug("Get Account Info for: %s" % user) + info = self.load_info(user) - except Exception, e: - infos = super(self.__class__, self).load_account_info(name, self.req) - infos['error'] = str(e) + self.log_debug("Account Info: %s" % info) + return info - if self.pyload.debug: - traceback.print_exc() - finally: - if hasattr(self, "req"): - if self.req: - self.req.close() - del self.req + def is_premium(self, user): + return self.get_info(user)['premium'] - self.log_debug("Account Info: %s" % infos) - infos['timestamp'] = time.time() - self.infos[name] = infos + def load_info(self, user): + self.log_critical(user in self.info) ############################# - elif "timestamp" in self.infos[name] and self.infos[name]['timestamp'] + self.info_threshold * 60 < time.time(): - self.log_debug("Reached timeout for account data") - self.schedule_refresh(name) + info = self.info[user] + data = info['data'] - data.update(self.infos[name]) - return data + #@TODO: Remove in 0.4.10 + data.update({'login': user, + 'type' : self.__name__}, + 'valid': self.info[user]['login']['valid']) + try: + self.req = self.get_request(user) + extra_info = self.parse_info(user, info['login']['password'], info, req) - def is_premium(self, user): - info = self.get_account_info(user) - return info['premium'] + if extra_info and isinstance(extra_info, dict): + data.update(extra_info) + data['timestamp'] = time.time() + except Exception, e: + self.log_warning(_("Error loading info for account %(user)s | %(err)s") % + {'user': user, 'err': e}) + data['error'] = str(e) - def load_account_info(self, name, req=None): + if self.pyload.debug: + traceback.print_exc(): + + else: + for key in ('premium', 'validuntil', 'trafficleft', 'maxtraffic'): + if key not in data: + data[key] = None + + finally: + if self.req: + self.req.close() + del self.req + + self.info[user].update(info) + return info + + + def parse_info(self, user, password, info, req): """ This should be overwritten in account plugin and retrieving account information for user - :param name: + :param user: :param req: `Request` instance :return: """ - return {'validuntil' : None, #: -1 for unlimited - 'login' : name, - # 'password' : self.accounts[name]['password'], #: Commented due security reason - 'options' : self.accounts[name]['options'], - 'valid' : self.accounts[name]['valid'], - 'trafficleft': None, #: In bytes, -1 for unlimited - 'maxtraffic' : None, - 'premium' : None, - 'timestamp' : 0, #: Time this info was retrieved - 'type' : self.__name__} - - - def get_all_accounts(self, force=False): - return [self.get_account_info(user, force) for user, data in self.accounts.iteritems()] + pass - #: Deprecated method, use `get_all_accounts` instead + #: Remove in 0.4.10 def getAllAccounts(self, *args, **kwargs): - return self.get_all_accounts(*args, **kwargs) + return [self.get_data(user, reload) for user, info in self.info.iteritems()] - def get_account_request(self, user=None): + def get_request(self, user=None): if not user: - user, data = self.select_account() + user, info = self.select() return self.pyload.requestFactory.getRequest(self.__name__, user) - def get_account_cookies(self, user=None): - if not user: - user, data = self.select_account() + def get_cookies(self, user=None): if not user: + user, info = self.select() return None - cj = self.pyload.requestFactory.getCookieJar(self.__name__, user) - return cj - + return self.pyload.requestFactory.getCookieJar(self.__name__, user) - def get_account_data(self, user): - return self.accounts[user] - - def select_account(self): + #@TODO: Random account only? Simply crazy... rewrite + def select(self): """ Returns an valid account name and data """ usable = [] - for user, data in self.accounts.iteritems(): - if not data['valid']: + for user, info in self.info.iteritems(): + if not info['login']['valid']: continue - if "time" in data['options'] and data['options']['time']: + options = info['data']['options'] + if "time" in options and options['time']: time_data = "" try: - time_data = data['options']['time'][0] + time_data = options['time'][0] start, end = time_data.split("-") if not compare_time(start.split(":"), end.split(":")): continue + except Exception: - self.log_warning(_("Your Time %s has wrong format, use 1:22-3:44") % time_data) + self.log_warning(_("Your time %s has wrong format, use 1:22-3:44") % time_data) - if user in self.infos: - if "validuntil" in self.infos[user]: - if self.infos[user]['validuntil'] > 0 and time.time() > self.infos[user]['validuntil']: - continue - if "trafficleft" in self.infos[user]: - if self.infos[user]['trafficleft'] == 0: - continue + if user in self.info: + if None is not self.info[user]['validuntil'] > 0 and time.time() > self.info[user]['validuntil']: + continue + if None is not self.info[user]['trafficleft'] == 0: + continue - usable.append((user, data)) + usable.append((user, info)) if not usable: return None, None @@ -291,44 +282,45 @@ class Account(Plugin): def can_use(self): - return self.select_account() is not (None, None) + return self.select() is not (None, None) def parse_traffic(self, value, unit=None): #: Return kilobytes if not unit and not isinstance(value, basestring): unit = "KB" - return parse_size(value, unit) - - def wrong_password(self): - raise WrongPassword + return parse_size(value, unit) def empty(self, user): - if user in self.infos: - self.log_warning(_("Account %s has not enough traffic, checking again in 30min") % user) + if user not in self.info: + return + + self.log_warning(_("Account %s has not enough traffic, checking again in 30min") % user) - self.infos[user].update({'trafficleft': 0}) - self.schedule_refresh(user, 30 * 60) + self.info[user]['data'].update({'trafficleft': 0}) + self.schedule_refresh(user, 30 * 60) def expired(self, user): - if user in self.infos: - self.log_warning(_("Account %s is expired, checking again in 1h") % user) + if user not in self.info: + return + + self.log_warning(_("Account %s is expired, checking again in 1h") % user) - self.infos[user].update({'validuntil': time.time() - 1}) - self.schedule_refresh(user, 60 * 60) + self.info[user]['data'].update({'validuntil': time.time() - 1}) + self.schedule_refresh(user, 60 * 60) - def schedule_refresh(self, user, time=0, force=True): + def schedule_refresh(self, user, time=0, reload=True): """ Add task to refresh account info to sheduler """ self.log_debug("Scheduled Account refresh for %s in %s seconds." % (user, time)) - self.pyload.scheduler.addJob(time, self.get_account_info, [user, force]) + self.pyload.scheduler.addJob(time, self.get_info, [user, reload]) - #: Deprecated method, use `schedule_refresh` instead + #: Deprecated method, use `schedule_refresh` instead (Remove in 0.4.10) def scheduleRefresh(self, *args, **kwargs): return self.schedule_refresh(*args, **kwargs) @@ -338,8 +330,8 @@ class Account(Plugin): """ Checks if user is still logged in """ - if user in self.timestamps: - if self.login_timeout > 0 and self.timestamps[user] + self.login_timeout * 60 < time.time(): + if user in self.info: + if self.LOGIN_TIMEOUT > 0 and self.info[user]['login']['timestamp'] + self.LOGIN_TIMEOUT * 60 < time.time(): self.log_debug("Reached login timeout for %s" % user) return self.relogin(user) else: diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 23512d7c4..16aab6188 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -71,7 +71,7 @@ class Addon(Plugin): self.event_map = None if self.event_list: - self.log_debug("Deprecated method `event_list`, use `event_map` instead") + self.log_debug("Deprecated method `event_list`, use `event_map` instead") (Remove in 0.4.10) for f in self.event_list: self.manager.addEvent(f, getattr(self, f)) @@ -83,7 +83,7 @@ class Addon(Plugin): self.cb = self.pyload.scheduler.addJob(max(0, delay), self._periodical, [threaded], threaded=threaded) - #: Deprecated method, use `init_periodical` instead + #: Deprecated method, use `init_periodical` instead (Remove in 0.4.10) def initPeriodical(self, *args, **kwargs): return self.init_periodical(*args, **kwargs) @@ -119,7 +119,7 @@ class Addon(Plugin): return self.get_config("activated") - #: Deprecated method, use `is_activated` instead + #: Deprecated method, use `is_activated` instead (Remove in 0.4.10) def isActivated(self, *args, **kwargs): return self.is_activated(*args, **kwargs) @@ -131,7 +131,7 @@ class Addon(Plugin): pass - #: Deprecated method, use `deactivate` instead + #: Deprecated method, use `deactivate` instead (Remove in 0.4.10) def unload(self, *args, **kwargs): return self.deactivate(*args, **kwargs) @@ -143,7 +143,7 @@ class Addon(Plugin): pass - #: Deprecated method, use `activate` instead + #: Deprecated method, use `activate` instead (Remove in 0.4.10) def coreReady(self, *args, **kwargs): return self.activate(*args, **kwargs) @@ -155,7 +155,7 @@ class Addon(Plugin): pass - #: Deprecated method, use `exit` instead + #: Deprecated method, use `exit` instead (Remove in 0.4.10) def coreExiting(self, *args, **kwargs): return self.exit(*args, **kwargs) @@ -164,7 +164,7 @@ class Addon(Plugin): pass - #: Deprecated method, use `download_preparing` instead + #: Deprecated method, use `download_preparing` instead (Remove in 0.4.10) def downloadPreparing(self, *args, **kwargs): return self.download_preparing(*args, **kwargs) @@ -173,7 +173,7 @@ class Addon(Plugin): pass - #: Deprecated method, use `download_finished` instead + #: Deprecated method, use `download_finished` instead (Remove in 0.4.10) def downloadFinished(self, *args, **kwargs): return self.download_finished(*args, **kwargs) @@ -182,7 +182,7 @@ class Addon(Plugin): pass - #: Deprecated method, use `download_failed` instead + #: Deprecated method, use `download_failed` instead (Remove in 0.4.10) def downloadFailed(self, *args, **kwargs): return self.download_failed(*args, **kwargs) @@ -191,7 +191,7 @@ class Addon(Plugin): pass - #: Deprecated method, use `package_finished` instead + #: Deprecated method, use `package_finished` instead (Remove in 0.4.10) def packageFinished(self, *args, **kwargs): return self.package_finished(*args, **kwargs) @@ -200,7 +200,7 @@ class Addon(Plugin): pass - #: Deprecated method, use `before_reconnect` instead + #: Deprecated method, use `before_reconnect` instead (Remove in 0.4.10) def beforeReconnecting(self, *args, **kwargs): return self.before_reconnect(*args, **kwargs) @@ -209,7 +209,7 @@ class Addon(Plugin): pass - #: Deprecated method, use `after_reconnect` instead + #: Deprecated method, use `after_reconnect` instead (Remove in 0.4.10) def afterReconnecting(self, ip): return self.after_reconnect(ip, None) @@ -221,7 +221,7 @@ class Addon(Plugin): pass - #: Deprecated method, use `captcha_task` instead + #: Deprecated method, use `captcha_task` instead (Remove in 0.4.10) def newCaptchaTask(self, *args, **kwargs): return self.captcha_task(*args, **kwargs) @@ -230,7 +230,7 @@ class Addon(Plugin): pass - #: Deprecated method, use `captcha_correct` instead + #: Deprecated method, use `captcha_correct` instead (Remove in 0.4.10) def captchaCorrect(self, *args, **kwargs): return self.captcha_correct(*args, **kwargs) @@ -239,6 +239,6 @@ class Addon(Plugin): pass - #: Deprecated method, use `captcha_invalid` instead + #: Deprecated method, use `captcha_invalid` instead (Remove in 0.4.10) def captchaInvalid(self, *args, **kwargs): return self.captcha_invalid(*args, **kwargs) diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 971158a36..d3eef43c8 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -3,8 +3,8 @@ from module.plugins.internal.Plugin import Plugin -class Captcha(Plugin): - __name__ = "Captcha" +class CaptchaService(Plugin): + __name__ = "CaptchaService" __type__ = "captcha" __version__ = "0.31" __status__ = "testing" diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index cc732247e..c0135fb1f 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -39,7 +39,7 @@ class Container(Crypter): self._create_packages() - #: Deprecated method, use `_load2disk` instead + #: Deprecated method, use `_load2disk` instead (Remove in 0.4.10) def loadToDisk(self, *args, **kwargs): return self._load2disk(*args, **kwargs) @@ -68,7 +68,7 @@ class Container(Crypter): self.fail(_("File not exists")) - #: Deprecated method, use `delete_tmp` instead + #: Deprecated method, use `delete_tmp` instead (Remove in 0.4.10) def deleteTmp(self, *args, **kwargs): return self.delete_tmp(*args, **kwargs) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 0d2e4dcc4..1d2728bd3 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -96,10 +96,10 @@ class Hoster(Plugin): self.account = None if self.account: - self.user, data = self.account.select_account() + self.user, data = self.account.select() #: Browser instance, see `network.Browser` - self.req = self.account.get_account_request(self.user) + self.req = self.account.get_request(self.user) self.chunk_limit = -1 #: Chunk limit, -1 for unlimited #: Enables resume (will be ignored if server dont accept chunks) @@ -593,7 +593,7 @@ class Hoster(Plugin): if not self.account: return True - traffic = self.account.get_account_info(self.user, True)['trafficleft'] + traffic = self.account.get_data(self.user, True)['trafficleft'] if traffic is None: return False @@ -612,7 +612,7 @@ class Hoster(Plugin): return self.pyfile.package().password or "" - #: Deprecated method, use `check_for_same_files` instead + #: Deprecated method, use `check_for_same_files` instead (Remove in 0.4.10) def checkForSameFiles(self, *args, **kwargs): return self.check_for_same_files(*args, **kwargs) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index f5cfa289d..308ba2966 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -57,8 +57,8 @@ class SimpleCrypter(Crypter, SimpleHoster): account = self.pyload.accountManager.getAccountPlugin(account_name) if account and account.can_use(): - self.user, data = account.select_account() - self.req = account.get_account_request(self.user) + self.user, data = account.select() + self.req = account.get_request(self.user) self.premium = account.is_premium(self.user) self.account = account diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 5ba207934..4a293b703 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -101,7 +101,7 @@ class SimpleHoster(Hoster): ('Html file' , r'\A\s* Date: Sat, 25 Jul 2015 09:34:18 +0200 Subject: Account rewritten (2) --- module/plugins/internal/Account.py | 95 +++++++++++++++++++++---------- module/plugins/internal/Addon.py | 2 +- module/plugins/internal/Captcha.py | 5 +- module/plugins/internal/CaptchaService.py | 8 ++- module/plugins/internal/Container.py | 3 +- module/plugins/internal/Hoster.py | 36 ++++++------ module/plugins/internal/Plugin.py | 15 +++-- module/plugins/internal/SimpleHoster.py | 4 +- 8 files changed, 102 insertions(+), 66 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 3a3fd4768..f33d44179 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -4,6 +4,7 @@ from __future__ import with_statement import random import time +import threading import traceback from module.plugins.internal.Plugin import Plugin @@ -29,6 +30,7 @@ class Account(Plugin): def __init__(self, manager, accounts): self.pyload = manager.core self.info = {} #: Provide information in dict here + self.lock = threading.RLock() self.init() self.init_accounts(accounts) @@ -58,9 +60,7 @@ class Account(Plugin): self.login(user, info['login']['password'], info['data'], self.req) except Exception, e: - self.log_warning( - _("Could not login with account %(user)s | %(msg)s") % {'user': user, - 'msg' : e}) + self.log_warning(_("Could not login with username ") + user, e) res = info['login']['valid'] = False if self.pyload.debug: traceback.print_exc() @@ -86,13 +86,15 @@ class Account(Plugin): return self._login(user) + #@TODO: Rewrite in 0.4.10 def init_accounts(self, accounts): for user, data in accounts.iteritems(): self.add(user, data['password'], data['options']) self._login(user) - def add(self, user, password=None, options): + @lock + def add(self, user, password=None, options={}): if user not in self.info: self.info[user] = {'login': {'valid': True, 'password': password or "", 'timestamp': 0}, #@NOTE: Do not remove `'valid': True` in 0.4.9 or accounts will not login 'data' : {'options': options, 'timestamp': 0}} @@ -102,7 +104,8 @@ class Account(Plugin): self.log_error(_("Error adding account"), _("User already exists")) - def update(self, user, password=None, options): + @lock + def update(self, user, password=None, options={}): """ Updates account and return true if anything changed """ @@ -131,7 +134,7 @@ class Account(Plugin): self.info.clear() elif user in self.info: - del self.info[user] + self.info.pop(user, None) #: Deprecated method, use `remove` instead (Remove in 0.4.10) @@ -140,64 +143,88 @@ class Account(Plugin): def get_data(self, user, reload=False): - return self.get_info(user, reload)['data'] + if not user: + return + + info = self.get_info(user, reload) + if info and 'data' in info: + return info['data'] + + #: Deprecated method, use `get_data` instead (Remove in 0.4.10) + def getAccountData(self, *args, **kwargs): + if 'force' in kwargs: + kwargs['reload'] = kwargs['force'] + kwargs.pop('force', None) + data = self.get_data(*args, **kwargs) or {} + if 'options' not in data: + data['options'] = {'limitdl': ['0']} + return data @lock def get_info(self, user, reload=False): """ Retrieve account infos for an user, do **not** overwrite this method!\\ - just use it to retrieve infos in hoster plugins. see `load_info` + just use it to retrieve infos in hoster plugins. see `parse_info` :param user: username :param reload: reloads cached account information :return: dictionary with information """ - if not reload and user in self.info: - info = self.info[user] + traceback.print_exc() ###################### - if info['data']['timestamp'] + self.INFO_THRESHOLD * 60 < time.time(): - self.log_debug("Reached timeout for account data") - self.schedule_refresh(user) + if user not in self.info: + self.log_error(_("User %s not found while retrieving account info") % user) + return - else: + elif reload: self.log_debug("Get Account Info for: %s" % user) - info = self.load_info(user) + info = self._parse_info(user) - self.log_debug("Account Info: %s" % info) + else: + info = self.info[user] + + if self.INFO_THRESHOLD > 0 and info['data']['timestamp'] + self.INFO_THRESHOLD * 60 < time.time(): + self.log_debug("Reached data timeout for %s" % user) + self.schedule_refresh(user) + + safe_info = info.copy() + safe_info['login']['password'] = "**********" + self.log_debug("Account info: %s" % safe_info) return info def is_premium(self, user): - return self.get_info(user)['premium'] + if not user: + return False + info = self.get_info(user, reload) + return info['premium'] if info and 'premium' in info else False - def load_info(self, user): - self.log_critical(user in self.info) ############################# + def _parse_info(self, user): info = self.info[user] data = info['data'] #@TODO: Remove in 0.4.10 data.update({'login': user, - 'type' : self.__name__}, - 'valid': self.info[user]['login']['valid']) + 'type' : self.__name__, + 'valid': self.info[user]['login']['valid']}) try: + data['timestamp'] = time.time() #: Set timestamp for login + self.req = self.get_request(user) - extra_info = self.parse_info(user, info['login']['password'], info, req) + extra_info = self.parse_info(user, info['login']['password'], info, self.req) if extra_info and isinstance(extra_info, dict): data.update(extra_info) - data['timestamp'] = time.time() except Exception, e: - self.log_warning(_("Error loading info for account %(user)s | %(err)s") % - {'user': user, 'err': e}) - data['error'] = str(e) + self.log_warning(_("Error loading info for ") + user, e) if self.pyload.debug: - traceback.print_exc(): + traceback.print_exc() else: for key in ('premium', 'validuntil', 'trafficleft', 'maxtraffic'): @@ -227,7 +254,11 @@ class Account(Plugin): #: Remove in 0.4.10 def getAllAccounts(self, *args, **kwargs): - return [self.get_data(user, reload) for user, info in self.info.iteritems()] + return [self.getAccountData(user, *args, **kwargs) for user, info in self.info.iteritems()] + + + def login_fail(self, reason=_("Login handshake has failed")): + return self.fail(reason) def get_request(self, user=None): @@ -240,7 +271,6 @@ class Account(Plugin): def get_cookies(self, user=None): if not user: user, info = self.select() - return None return self.pyload.requestFactory.getCookieJar(self.__name__, user) @@ -316,17 +346,20 @@ class Account(Plugin): """ Add task to refresh account info to sheduler """ - self.log_debug("Scheduled Account refresh for %s in %s seconds." % (user, time)) + self.log_debug("Scheduled refresh for %s in %s seconds" % (user, time)) self.pyload.scheduler.addJob(time, self.get_info, [user, reload]) #: Deprecated method, use `schedule_refresh` instead (Remove in 0.4.10) def scheduleRefresh(self, *args, **kwargs): + if 'force' in kwargs: + kwargs['reload'] = kwargs['force'] + kwargs.pop('force', None) return self.schedule_refresh(*args, **kwargs) @lock - def check_login(self, user): + def is_logged(self, user): """ Checks if user is still logged in """ diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 16aab6188..248518128 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -71,7 +71,7 @@ class Addon(Plugin): self.event_map = None if self.event_list: - self.log_debug("Deprecated method `event_list`, use `event_map` instead") (Remove in 0.4.10) + self.log_debug("Deprecated method `event_list`, use `event_map` instead") for f in self.event_list: self.manager.addEvent(f, getattr(self, f)) diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 89cace612..ec618e353 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import os import time import traceback @@ -27,10 +28,6 @@ class Captcha(Plugin): self.init() - def _log(self, level, args): - return self.plugin._log(level, (self.__name__,) + args) - - def init(self): """ Initialize additional data structures diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index d3eef43c8..1a73ce5d1 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.Plugin import Plugin +from module.plugins.internal.Captcha import Captcha -class CaptchaService(Plugin): +class CaptchaService(Captcha): __name__ = "CaptchaService" __type__ = "captcha" __version__ = "0.31" @@ -18,6 +18,10 @@ class CaptchaService(Plugin): self.key = None #: Last key detected + def _log(self, level, args): + return self.plugin._log(level, (self.__name__,) + args) + + #@TODO: Recheck in 0.4.10 def retrieve_key(self, data): if self.detect_key(data): diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index c0135fb1f..c7a6828c7 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -56,8 +56,9 @@ class Container(Crypter): try: with open(self.pyfile.url, "wb") as f: f.write(content) + except IOError, e: - self.fail(str(e)) + self.fail(str(e)) #@TODO: Remove `str` in 0.4.10 else: self.pyfile.name = os.path.basename(self.pyfile.url) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 1d2728bd3..840d73847 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -171,7 +171,7 @@ class Hoster(Plugin): self.req.renewHTTPRequest() if self.account: - self.account.check_login(self.user) + self.account.is_logged(self.user) else: self.req.clearCookies() @@ -206,7 +206,10 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): reconnect = bool(reconnect) - self.log_debug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + + self.log_info(_("Reconnect: %s") % reconnect) + self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) + self.wantReconnect = reconnect @@ -220,8 +223,8 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_debug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), - "Wait: %d (+1) seconds" % wait_time) + self.log_info(_("Wait: %d seconds") % wait_time) + self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until @@ -229,13 +232,13 @@ class Hoster(Plugin): self.set_reconnect(reconnect) - def wait(self, seconds=0, reconnect=None): + def wait(self, seconds=None, reconnect=None): """ Waits the time previously set """ pyfile = self.pyfile - if seconds > 0: + if seconds is not None: self.set_wait(seconds) if reconnect is not None: @@ -243,12 +246,9 @@ class Hoster(Plugin): self.waiting = True - status = pyfile.status + status = pyfile.status #@NOTE: Remove in 0.4.10 pyfile.setStatus("waiting") - self.log_info(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), - _("Reconnect: %s") % self.wantReconnect) - if self.account: self.log_debug("Ignore reconnection due account logged") @@ -256,11 +256,9 @@ class Hoster(Plugin): if pyfile.abort: self.abort() - time.sleep(1) + time.sleep(3) else: while pyfile.waitUntil > time.time(): - self.thread.m.reconnecting.wait(1) - if pyfile.abort: self.abort() @@ -268,12 +266,12 @@ class Hoster(Plugin): self.waiting = False self.wantReconnect = False raise Reconnect - - time.sleep(1) + + self.thread.m.reconnecting.wait(3) + time.sleep(3) self.waiting = False - - pyfile.status = status + pyfile.status = status #@NOTE: Remove in 0.4.10 def skip(self, reason=""): @@ -388,7 +386,7 @@ class Hoster(Plugin): os.chown(location, uid, gid) except Exception, e: - self.fail(e) + self.fail(str(e)) #@TODO: Remove `str` in 0.4.10 #: Convert back to unicode location = fs_decode(location) @@ -508,7 +506,7 @@ class Hoster(Plugin): redirect = max(follow_location, 1) else: - redirect = self.get_config("maxredirs", plugin="UserAgentSwitcher") + redirect = self.get_config("maxredirs", 10, "UserAgentSwitcher") for i in xrange(redirect): try: diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 9ac89501c..02ef326d6 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -5,6 +5,7 @@ from __future__ import with_statement import inspect import os import re +import traceback import urllib from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 @@ -80,12 +81,12 @@ def parse_html_form(attr_str, html, input_names={}): for form in re.finditer(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, html, re.S | re.I): inputs = {} - action = parseHtmlTagAttrValue("action", form.group('TAG')) + action = parse_html_tag_attr_value("action", form.group('TAG')) for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?= `encode` in 0.4.10 - log("%(plugin)s%(id)s: %(msg)s" % {'plugin': self.__name__, - 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", - 'msg' : msg or _(level.upper() + " MARK")}) + log("%(type)s %(plugin)s%(id)s: %(msg)s" % {'type': self.__type__.upper(), + 'plugin': self.__name__, + 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", + 'msg' : msg or _(level.upper() + " MARK")}) def log_debug(self, *args): @@ -204,6 +206,7 @@ class Plugin(object): except KeyError: self.log_warning(_("Config option or plugin not found")) + traceback.print_exc() return default diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 4a293b703..8eea9dd5a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -13,7 +13,7 @@ from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, Retry, fixurl, replace_patterns, set_cookies +from module.plugins.internal.Plugin import Fail, Retry, fixurl, replace_patterns, seconds_to_midnight, set_cookies from module.utils import fixup, fs_encode, parseFileSize as parse_size @@ -115,7 +115,7 @@ class SimpleHoster(Hoster): @classmethod def api_info(cls, url): - return super(SimpleHoster, self).get_info(url) + return super(SimpleHoster, cls).get_info(url) @classmethod -- cgit v1.2.3 From 952001324e1faf584b1adcb01c4a0406a3722932 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Jul 2015 09:42:49 +0200 Subject: =?UTF-8?q?Don't=20user=20dictionary=E2=80=99s=20iterator=20method?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/plugins/internal/Account.py | 6 +++--- module/plugins/internal/Addon.py | 2 +- module/plugins/internal/Crypter.py | 2 +- module/plugins/internal/Hoster.py | 4 ++-- module/plugins/internal/MultiHook.py | 4 ++-- module/plugins/internal/OCR.py | 4 ++-- module/plugins/internal/Plugin.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index f33d44179..b59a59932 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -88,7 +88,7 @@ class Account(Plugin): #@TODO: Rewrite in 0.4.10 def init_accounts(self, accounts): - for user, data in accounts.iteritems(): + for user, data in accounts.items(): self.add(user, data['password'], data['options']) self._login(user) @@ -254,7 +254,7 @@ class Account(Plugin): #: Remove in 0.4.10 def getAllAccounts(self, *args, **kwargs): - return [self.getAccountData(user, *args, **kwargs) for user, info in self.info.iteritems()] + return [self.getAccountData(user, *args, **kwargs) for user, info in self.info.items()] def login_fail(self, reason=_("Login handshake has failed")): @@ -281,7 +281,7 @@ class Account(Plugin): Returns an valid account name and data """ usable = [] - for user, info in self.info.iteritems(): + for user, info in self.info.items(): if not info['login']['valid']: continue diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 248518128..3c617f730 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -60,7 +60,7 @@ class Addon(Plugin): def init_events(self): if self.event_map: - for event, funcs in self.event_map.iteritems(): + for event, funcs in self.event_map.items(): if type(funcs) in (list, tuple): for f in funcs: self.manager.addEvent(event, getattr(self, f)) diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 7e97c8228..c677b2852 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -57,7 +57,7 @@ class Crypter(Hoster): """ Generate new packages from self.urls """ - packages = [(name, links, None) for name, links in self.pyload.api.generatePackages(self.urls).iteritems()] + packages = [(name, links, None) for name, links in self.pyload.api.generatePackages(self.urls).items()] self.packages.extend(packages) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 840d73847..10cab5616 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -362,7 +362,7 @@ class Hoster(Plugin): self.fail(_("No url given")) if self.pyload.debug: - self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) self.captcha.correct() self.check_for_same_files() @@ -474,7 +474,7 @@ class Hoster(Plugin): #: Produces encoding errors, better log to other file in the future? # self.log_debug("Content: %s" % content) - for name, rule in rules.iteritems(): + for name, rule in rules.items(): if isinstance(rule, basestring): if rule in content: do_delete = True diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 0fa75df98..c49a3c89a 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -187,11 +187,11 @@ class MultiHook(Hook): excludedList = [] if self.plugintype == "hoster": - pluginMap = dict((name.lower(), name) for name in self.pyload.pluginManager.hosterPlugins.iterkeys()) + pluginMap = dict((name.lower(), name) for name in self.pyload.pluginManager.hosterPlugins.keys()) accountList = [account.type.lower() for account in self.pyload.api.getAccounts(False) if account.valid and account.premium] else: pluginMap = {} - accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.pyload.pluginManager.crypterPlugins.iterkeys()] + accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.pyload.pluginManager.crypterPlugins.keys()] for plugin in self.plugins_cached(): name = remove_chars(plugin, "-.") diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 36d259e0c..9896837b5 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -260,7 +260,7 @@ class OCR(Plugin): hkey = 0 hvalue = 0 - for key, value in highest.iteritems(): + for key, value in highest.items(): if value > hvalue: hkey = key hvalue = value @@ -325,7 +325,7 @@ class OCR(Plugin): else: result = self.result_captcha - for key, item in values.iteritems(): + for key, item in values.items(): if key.__class__ is str: result = result.replace(key, item) diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 02ef326d6..7b3d03752 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -94,7 +94,7 @@ def parse_html_form(attr_str, html, input_names={}): if input_names: #: Check input attributes - for key, val in input_names.iteritems(): + for key, val in input_names.items(): if key in inputs: if isinstance(val, basestring) and inputs[key] is val: continue @@ -271,7 +271,7 @@ class Plugin(object): self.fail(_("No url given")) if self.pyload.debug: - self.log_debug("LOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + self.log_debug("LOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) if req is None: if hasattr(self, "req"): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 8eea9dd5a..b063bcac2 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -18,7 +18,7 @@ from module.utils import fixup, fs_encode, parseFileSize as parse_size #@TODO: Adapt and move to PyFile in 0.4.10 -statusMap = dict((v, k) for k, v in _statusMap.iteritems()) +statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): -- cgit v1.2.3 From f83389333ec10376452aa5f6d5ccd3963c6bafa1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Jul 2015 10:28:30 +0200 Subject: Update internal plugins --- module/plugins/internal/Account.py | 10 +- module/plugins/internal/Captcha.py | 26 +++-- module/plugins/internal/CaptchaService.py | 6 +- module/plugins/internal/Container.py | 18 ++-- module/plugins/internal/Extractor.py | 7 +- module/plugins/internal/Hoster.py | 67 +++++++----- module/plugins/internal/MultiHoster.py | 50 ++++----- module/plugins/internal/OCR.py | 8 +- module/plugins/internal/Plugin.py | 19 ++-- module/plugins/internal/SimpleCrypter.py | 74 ++++---------- module/plugins/internal/SimpleHoster.py | 163 ++++++++++++++++-------------- module/plugins/internal/XFSHoster.py | 8 +- 12 files changed, 231 insertions(+), 225 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index b59a59932..f1a30a9cb 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -14,7 +14,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.05" + __version__ = "0.06" __status__ = "testing" __description__ = """Base account plugin""" @@ -66,7 +66,7 @@ class Account(Plugin): traceback.print_exc() else: - res = True + res = info['login']['valid'] = True finally: if self.req: @@ -96,7 +96,7 @@ class Account(Plugin): @lock def add(self, user, password=None, options={}): if user not in self.info: - self.info[user] = {'login': {'valid': True, 'password': password or "", 'timestamp': 0}, #@NOTE: Do not remove `'valid': True` in 0.4.9 or accounts will not login + self.info[user] = {'login': {'valid': None, 'password': password or "", 'timestamp': 0}, 'data' : {'options': options, 'timestamp': 0}} self._login(user) return True @@ -171,8 +171,6 @@ class Account(Plugin): :param reload: reloads cached account information :return: dictionary with information """ - traceback.print_exc() ###################### - if user not in self.info: self.log_error(_("User %s not found while retrieving account info") % user) return @@ -312,7 +310,7 @@ class Account(Plugin): def can_use(self): - return self.select() is not (None, None) + return self.select() != (None, None) def parse_traffic(self, value, unit=None): #: Return kilobytes diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index ec618e353..32d882fac 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -10,7 +10,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.31" + __version__ = "0.32" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -20,9 +20,8 @@ class Captcha(Plugin): def __init__(self, plugin): #@TODO: Pass pyfile instead plugin, so store plugin's html in its associated pyfile as data self.pyload = plugin.pyload - self.info = {} #: Provide information in dict here - self.plugin = plugin + self.info = {} #: Provide information in dict here self.task = None #: captchaManager task self.init() @@ -35,6 +34,10 @@ class Captcha(Plugin): pass + def _log(self, level, args): + return self.plugin._log(level, (self.__name__,) + args) + + def recognize(self, image): """ Extend to build your custom anti-captcha ocr @@ -43,17 +46,17 @@ class Captcha(Plugin): def decrypt(self, url, get={}, post={}, ref=False, cookies=False, decode=False, - input_type='png', output_type='textual', ocr=True): + input_type='jpg', output_type='textual', ocr=True): img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode) return self._decrypt(img, input_type, output_type, ocr) #@TODO: Definitely dhoose a better name for this method! - def _decrypt(self, raw, input_type='png', output_type='textual', ocr=None): + def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=None): """ Loads a captcha and decrypts it with ocr, plugin, user input - :param url: url of captcha image + :param raw: image raw data :param get: get part for request :param post: post part for request :param cookies: True if cookies should be enabled @@ -75,7 +78,7 @@ class Captcha(Plugin): OCR = self.pyload.pluginManager.loadClass("captcha", ocr) #: Rename `captcha` to `ocr` in 0.4.10 if self.plugin.pyfile.abort: - self.abort() + self.plugin.abort() result = OCR(self.plugin).recognize(tmp_img.name) @@ -86,12 +89,12 @@ class Captcha(Plugin): captchaManager = self.pyload.captchaManager try: - self.task = captchaManager.newTask(img, input_type, tmp_img.name, output_type) + self.task = captchaManager.newTask(raw, input_type, tmp_img.name, output_type) captchaManager.handleCaptcha(self.task) while self.task.isWaiting(): if self.plugin.pyfile.abort: - self.abort() + self.plugin.abort() time.sleep(1) finally: captchaManager.removeTask(self.task) @@ -102,12 +105,13 @@ class Captcha(Plugin): elif not self.task.result: self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) - result = task.result - self.log_debug("Received captcha result: %s" % result) #@TODO: Remove from here? + result = self.task.result + self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? if not self.pyload.debug: try: os.remove(tmp_img.name) + except OSError, e: self.log_warning(_("Error removing: %s") % tmp_img.name, e) traceback.print_exc() diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 1a73ce5d1..20dc60427 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -6,7 +6,7 @@ from module.plugins.internal.Captcha import Captcha class CaptchaService(Captcha): __name__ = "CaptchaService" __type__ = "captcha" - __version__ = "0.31" + __version__ = "0.32" __status__ = "testing" __description__ = """Base anti-captcha service plugin""" @@ -18,10 +18,6 @@ class CaptchaService(Captcha): self.key = None #: Last key detected - def _log(self, level, args): - return self.plugin._log(level, (self.__name__,) + args) - - #@TODO: Recheck in 0.4.10 def retrieve_key(self, data): if self.detect_key(data): diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index c7a6828c7..ec2d429dd 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -13,7 +13,7 @@ from module.utils import save_join as fs_join class Container(Crypter): __name__ = "Container" __type__ = "container" - __version__ = "0.05" + __version__ = "0.06" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -24,18 +24,22 @@ class Container(Crypter): __authors__ = [("mkaay", "mkaay@mkaay.de")] - def preprocessing(self, thread): + def process(self, pyfile): """ - Prepare + Main method """ - self.setup() - self.thread = thread - self._load2disk() - self.decrypt(self.pyfile) + self.decrypt(pyfile) + self.delete_tmp() + if self.urls: + self._generate_packages() + + elif not self.packages: + self.error(_("No link grabbed"), "decrypt") + self._create_packages() diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 668207ea3..cf9a1f6e3 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.26" + __version__ = "0.27" __status__ = "testing" __description__ = """Base extractor plugin""" @@ -87,6 +87,7 @@ class Extractor(Plugin): Initialize extractor for specific file """ self.pyload = plugin.pyload + self.plugin = plugin self.info = {} #: Provide information in dict here self.filename = filename @@ -112,6 +113,10 @@ class Extractor(Plugin): pass + def _log(self, level, args): + return self.plugin._log(level, (self.__name__,) + args) + + def check(self): """ Quick Check by listing content of archive. diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 10cab5616..c35178547 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.05" + __version__ = "0.06" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -179,6 +179,7 @@ class Hoster(Plugin): self.pyfile.setStatus("starting") + self.log_debug("PROCESS URL " + self.pyfile.url) return self.process(self.pyfile) @@ -195,19 +196,10 @@ class Hoster(Plugin): return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) - def reset_account(self): - """ - Don't use account and retry download - """ - self.account = None - self.req = self.pyload.requestFactory.getRequest(self.__name__) - self.retry() - - def set_reconnect(self, reconnect): reconnect = bool(reconnect) - self.log_info(_("Reconnect: %s") % reconnect) + self.log_info(_("RECONNECT ") + ("enabled" if reconnect else "disabled")) self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) self.wantReconnect = reconnect @@ -223,7 +215,7 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_info(_("Wait: %d seconds") % wait_time) + self.log_info(_("WAIT %d seconds") % wait_time) self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until @@ -249,14 +241,16 @@ class Hoster(Plugin): status = pyfile.status #@NOTE: Remove in 0.4.10 pyfile.setStatus("waiting") - if self.account: - self.log_debug("Ignore reconnection due account logged") + if not self.wantReconnect or self.account: + if self.account: + self.log_warning("Ignore reconnection due logged account") while pyfile.waitUntil > time.time(): if pyfile.abort: self.abort() time.sleep(3) + else: while pyfile.waitUntil > time.time(): if pyfile.abort: @@ -266,7 +260,7 @@ class Hoster(Plugin): self.waiting = False self.wantReconnect = False raise Reconnect - + self.thread.m.reconnecting.wait(3) time.sleep(3) @@ -321,7 +315,7 @@ class Hoster(Plugin): self.retries[id] = 0 if 0 < max_tries <= self.retries[id]: - self.fail(reason or _("Max retries reached"), _("retry")) + self.fail(reason or _("Max retries reached")) self.wait(wait_time, False) @@ -329,6 +323,21 @@ class Hoster(Plugin): raise Retry(reason) + def restart(self, reason=None, reset=False): + if not reason: + reason = _("Fallback to free download") if reset else _("Restart") + + if reset: + if not self.premium: + return + + self.premium = False + self.account = None + self.req = self.pyload.requestFactory.getRequest(self.__name__) + + raise Retry(reason) + + def fixurl(self, url): url = _fixurl(url) @@ -362,7 +371,8 @@ class Hoster(Plugin): self.fail(_("No url given")) if self.pyload.debug: - self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + self.log_debug("DOWNLOAD URL " + url, + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) self.captcha.correct() self.check_for_same_files() @@ -432,7 +442,7 @@ class Hoster(Plugin): return self.last_download - def check_download(self, rules, delete=True, file_size=0, size_tolerance=1000, read_size=100000): + def check_download(self, rules, delete=False, file_size=0, size_tolerance=2048, read_size=100000): """ Checks the content of the last downloaded file, re match is saved to `lastCheck` @@ -444,14 +454,14 @@ class Hoster(Plugin): :return: dictionary key of the first rule that matched """ do_delete = False - lastDownload = fs_encode(self.last_download) + last_download = fs_encode(self.last_download) - if not self.last_download or not os.path.exists(lastDownload): + if not self.last_download or not os.path.exists(last_download): self.last_download = "" self.fail(self.pyfile.error or _("No file downloaded")) try: - download_size = os.stat(lastDownload).st_size + download_size = os.stat(last_download).st_size if download_size < 1: do_delete = True @@ -462,14 +472,13 @@ class Hoster(Plugin): if diff > size_tolerance: do_delete = True - self.fail(_("File size mismatch")) + self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") + % (file_size, download_size)) elif diff != 0: self.log_warning(_("File size is not equal to expected size")) - self.log_debug("Download Check triggered") - - with open(lastDownload, "rb") as f: + with open(last_download, "rb") as f: content = f.read(read_size) #: Produces encoding errors, better log to other file in the future? @@ -489,12 +498,16 @@ class Hoster(Plugin): finally: if delete and do_delete: try: - os.remove(lastDownload) + os.remove(last_download) + except OSError, e: - self.log_warning(_("Error removing: %s") % lastDownload, e) + self.log_warning(_("Error removing: %s") % last_download, e) if self.pyload.debug: traceback.print_exc() + else: + self.log_info(_("File deleted")) + def direct_link(self, url, follow_location=None): link = "" diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index fe6fa29d6..b569b04c3 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.43" + __version__ = "0.44" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -21,48 +21,49 @@ class MultiHoster(SimpleHoster): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + LEECH_HOSTER = False LOGIN_ACCOUNT = True def setup(self): self.chunk_limit = 1 - self.multiDL = bool(self.account) + self.multiDL = bool(self.account) self.resume_download = self.premium def prepare(self): - self.html = "" - self.link = "" #@TODO: Move to Hoster in 0.4.10 - self.direct_dl = False #@TODO: Move to Hoster in 0.4.10 + #@TODO: Recheck in 0.4.10 + plugin = self.pyload.pluginManager.hosterPlugins[self.__name__] + name = plugin['name'] + module = plugin['module'] + klass = getattr(module, name) - if not self.get_config('use_premium', True): - self.retry_free() - - if self.LOGIN_ACCOUNT and not self.account: - self.fail(_("Required account not found")) - - self.req.setOption("timeout", 120) - - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) + self.get_info = klass.get_info if self.DIRECT_LINK is None: - self.direct_dl = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) + direct_dl = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) else: - self.direct_dl = self.DIRECT_LINK + direct_dl = self.DIRECT_LINK - self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + super(MultiHoster, self).prepare() + + self.direct_dl = direct_dl def process(self, pyfile): try: self.prepare() + self.check_info() #@TODO: Remove in 0.4.10 if self.direct_dl: - self.check_info() - self.log_debug("Looking for direct download link...") + self.log_info(_("Looking for direct download link...")) self.handle_direct(pyfile) + if self.link or was_downloaded(): + self.log_info(_("Direct download link detected")) + else: + self.log_info(_("Direct download link not found")) + if not self.link and not self.last_download: self.preload() @@ -70,14 +71,17 @@ class MultiHoster(SimpleHoster): self.check_status(getinfo=False) if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): - self.log_debug("Handled as premium download") + self.log_info(_("Processing as premium download...")) self.handle_premium(pyfile) elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): - self.log_debug("Handled as free download") + self.log_info(_("Processing as free download...")) self.handle_free(pyfile) - self.download(self.link, ref=False, disposition=True) + if not self.last_download: + self.log_info(_("Downloading file...")) + self.download(self.link, ref=False, disposition=True) + self.check_file() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 9896837b5..37a33206f 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -21,7 +21,7 @@ from module.utils import save_join as fs_join class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" - __version__ = "0.13" + __version__ = "0.14" __status__ = "testing" __description__ = """OCR base plugin""" @@ -31,7 +31,9 @@ class OCR(Plugin): def __init__(self, plugin): self.pyload = plugin.pyload + self.plugin = plugin self.info = {} #: Provide information in dict here + self.init() @@ -42,6 +44,10 @@ class OCR(Plugin): pass + def _log(self, level, args): + return self.plugin._log(level, (self.__name__,) + args) + + def load_image(self, image): self.image = Image.open(image) self.pixels = self.image.load() diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 7b3d03752..f88660f24 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -5,7 +5,6 @@ from __future__ import with_statement import inspect import os import re -import traceback import urllib from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 @@ -126,7 +125,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.13" + __version__ = "0.14" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -155,11 +154,11 @@ class Plugin(object): def _log(self, level, args): log = getattr(self.pyload.log, level) - msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in args if a)) #@NOTE: `fs_encode` -> `encode` in 0.4.10 - log("%(type)s %(plugin)s%(id)s: %(msg)s" % {'type': self.__type__.upper(), - 'plugin': self.__name__, - 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", - 'msg' : msg or _(level.upper() + " MARK")}) + msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in args if a)) + log("%(type)s %(plugin)s%(id)s : %(msg)s" % {'type': self.__type__.upper(), + 'plugin': self.__name__, + 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", + 'msg' : msg or "---------- MARK ----------"}) def log_debug(self, *args): @@ -205,8 +204,7 @@ class Plugin(object): return self.pyload.config.getPlugin(plugin or self.__name__, option) except KeyError: - self.log_warning(_("Config option or plugin not found")) - traceback.print_exc() + self.log_warning(_("Config option `%s` not found, use default `%s`") % (option, default or None)) return default @@ -271,7 +269,8 @@ class Plugin(object): self.fail(_("No url given")) if self.pyload.debug: - self.log_debug("LOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + self.log_debug("LOAD URL " + url, + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) if req is None: if hasattr(self, "req"): diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 308ba2966..33ebd8cac 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.56" + __version__ = "0.57" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -51,6 +51,11 @@ class SimpleCrypter(Crypter, SimpleHoster): def load_page(self, page_n): return the html of the page number page_n """ + + DIRECT_LINK = True + LEECH_HOSTER = False + + #@TODO: Remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") @@ -64,27 +69,6 @@ class SimpleCrypter(Crypter, SimpleHoster): self.account = account - def prepare(self): - self.pyfile.error = "" #@TODO: Remove in 0.4.10 - - self.html = "" #@TODO: Recheck in 0.4.10 - self.link = "" #@TODO: Recheck in 0.4.10 - self.links = [] - - if self.LOGIN_PREMIUM and not self.premium: - self.fail(_("Required premium account not found")) - - if self.LOGIN_ACCOUNT and not self.account: - self.fail(_("Required account not found")) - - self.req.setOption("timeout", 120) - - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) - - self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - - def handle_direct(self, pyfile): for i in xrange(self.get_config("maxredirs", plugin="UserAgentSwitcher")): redirect = self.link or pyfile.url @@ -101,16 +85,19 @@ class SimpleCrypter(Crypter, SimpleHoster): def decrypt(self, pyfile): self.prepare() + self.check_info() #@TODO: Remove in 0.4.10 - self.log_debug("Looking for link redirect...") - self.handle_direct(pyfile) + if self.direct_dl: + self.log_debug(_("Looking for direct download link...")) + self.handle_direct(pyfile) - if self.link: - self.urls = [self.link] + if self.link or self.links or self.urls or self.packages: + self.log_info(_("Direct download link detected")) + else: + self.log_info(_("Direct download link not found")) - else: + if not (self.link or self.links or self.urls or self.packages): self.preload() - self.check_info() self.links = self.get_links() or list() @@ -119,33 +106,12 @@ class SimpleCrypter(Crypter, SimpleHoster): self.log_debug("Package has %d links" % len(self.links)) - if self.links: - self.packages = [(self.info['name'], self.links, self.info['folder'])] - - - def check_name_size(self, getinfo=True): - if not self.info or getinfo: - self.log_debug("File info (BEFORE): %s" % self.info) - self.info.update(self.get_info(self.pyfile.url, self.html)) - self.log_debug("File info (AFTER): %s" % self.info) - - try: - url = self.info['url'].strip() - name = self.info['name'].strip() - if name and name is not url: - self.pyfile.name = name - - except Exception: - pass - - try: - folder = self.info['folder'] = self.pyfile.name - - except Exception: - pass + if self.link: + self.urls.append(self.link) - self.log_debug("File name: %s" % self.pyfile.name, - "File folder: %s" % self.pyfile.name) + if self.links: + name = folder = pyfile.name + self.packages.append((name, self.links, folder)) def get_links(self): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index b063bcac2..e8281b943 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -24,7 +24,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.71" + __version__ = "1.72" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -107,7 +107,7 @@ class SimpleHoster(Hoster): DISPOSITION = True #: Set to True to use any content-disposition value in http header as file name LOGIN_ACCOUNT = False #: Set to True to require account login LOGIN_PREMIUM = False #: Set to True to require premium account login - MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handle_multi method) + LEECH_HOSTER = False #: Set to True to leech other hoster link (as defined in handle_multi method) TEXT_ENCODING = True #: Set to encoding name if encoding value in http header is not correct LINK_PATTERN = None @@ -203,18 +203,19 @@ class SimpleHoster(Hoster): def prepare(self): - self.pyfile.error = "" #@TODO: Remove in 0.4.10 - - self.html = "" #@TODO: Recheck in 0.4.10 - self.link = "" #@TODO: Recheck in 0.4.10 - self.direct_dl = False - self.multihost = False + self.pyfile.error = "" #@TODO: Remove in 0.4.10 + self.html = "" #@TODO: Recheck in 0.4.10 + self.link = "" #@TODO: Recheck in 0.4.10 + self.last_download = "" + self.direct_dl = False + self.leech_dl = False if not self.get_config('use_premium', True): self.retry_free() if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) + self.LOGIN_ACCOUNT = True if self.LOGIN_ACCOUNT and not self.account: self.fail(_("Required account not found")) @@ -231,18 +232,21 @@ class SimpleHoster(Hoster): if not hasattr(self, 'LINK_PREMIUM_PATTERN'): self.LINK_PREMIUM_PATTERN = self.LINK_PATTERN - if (self.MULTI_HOSTER + if (self.LEECH_HOSTER and (self.__pattern__ is not self.pyload.pluginManager.hosterPlugins[self.__name__]['pattern'] - or re.match(self.__pattern__, self.pyfile.url) is None)): - self.multihost = True - return + and re.match(self.__pattern__, self.pyfile.url) is None)): + self.leech_dl = True - if self.DIRECT_LINK is None: + if self.leech_dl: + self.direct_dl = False + + elif self.DIRECT_LINK is None: self.direct_dl = bool(self.account) else: self.direct_dl = self.DIRECT_LINK - self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + if not self.leech_dl: + self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) def preload(self): @@ -255,33 +259,43 @@ class SimpleHoster(Hoster): def process(self, pyfile): try: self.prepare() - self.check_info() - - if self.direct_dl: - self.log_debug("Looking for direct download link...") - self.handle_direct(pyfile) + self.check_info() #@TODO: Remove in 0.4.10 - if self.multihost and not self.link and not self.last_download: - self.log_debug("Looking for leeched download link...") + if self.leech_dl: + self.log_info(_("Processing as debrid download...")) self.handle_multi(pyfile) + if not self.link and not was_downloaded(): + self.log_info(_("Failed to leech url")) + + else: + if not self.link and self.direct_dl and not self.last_download: + self.log_info(_("Looking for direct download link...")) + self.handle_direct(pyfile) + + if self.link or self.last_download: + self.log_info(_("Direct download link detected")) + else: + self.log_info(_("Direct download link not found")) + if not self.link and not self.last_download: - self.MULTI_HOSTER = False - self.retry(1, reason=_("Multi hoster fails")) + self.preload() + + if 'status' not in self.info or self.info['status'] is 3: #@TODO: Recheck in 0.4.10 + self.check_info() - if not self.link and not self.last_download: - self.preload() - self.check_info() + if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): + self.log_info(_("Processing as premium download...")) + self.handle_premium(pyfile) - if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): - self.log_debug("Handled as premium download") - self.handle_premium(pyfile) + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): + self.log_info(_("Processing as free download...")) + self.handle_free(pyfile) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): - self.log_debug("Handled as free download") - self.handle_free(pyfile) + if not self.last_download: + self.log_info(_("Downloading file...")) + self.download(self.link, ref=False, disposition=self.DISPOSITION) - self.download(self.link, ref=False, disposition=self.DISPOSITION) self.check_file() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 @@ -292,25 +306,26 @@ class SimpleHoster(Hoster): elif self.get_config('fallback', True) and self.premium: self.log_warning(_("Premium download failed"), e) - self.retry_free() + self.restart(reset=True) else: raise Fail(err) def check_file(self): - lastDownload = fs_encode(self.last_download) + self.log_info(_("Checking file...")) if self.captcha.task and not self.last_download: self.captcha.invalid() self.retry(10, reason=_("Wrong captcha")) elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, - file_size=self.info['size']): + file_size=self.info['size'] if 'size' in self.info else 0, + delete=True): self.error(_("Empty file")) else: - self.log_debug("Checking last downloaded file with built-in rules") + self.log_debug("Using default check rules...") for r, p in self.FILE_ERRORS: errmsg = self.check_download({r: re.compile(p)}) if errmsg is not None: @@ -326,12 +341,13 @@ class SimpleHoster(Hoster): self.retry(wait_time=60, reason=errmsg) else: if self.CHECK_FILE: - self.log_debug("Checking last downloaded file with custom rules") - with open(lastDownload, "rb") as f: + self.log_debug("Using custom check rules...") + with open(fs_encode(self.last_download), "rb") as f: self.html = f.read(50000) #@TODO: Recheck in 0.4.10 self.check_errors() - self.log_debug("No file errors found") + self.log_debug("No errors found") + self.pyfile.error = "" def check_errors(self): @@ -440,13 +456,14 @@ class SimpleHoster(Hoster): def check_status(self, getinfo=True): if not self.info or getinfo: - self.log_debug("Update file info...") - self.log_debug("Previous file info: %s" % self.info) + self.log_info(_("Updating file info...")) + old_info = self.info.copy() self.info.update(self.get_info(self.pyfile.url, self.html)) - self.log_debug("Current file info: %s" % self.info) + self.log_debug("File info: %s" % self.info) + self.log_debug("Previous file info: %s" % old_info) try: - status = self.info['status'] + status = self.info['status'] or None if status == 1: self.offline() @@ -455,40 +472,49 @@ class SimpleHoster(Hoster): self.temp_offline() elif status == 8: - self.fail(self.info['error'] if 'error' in self.info else _("Failed")) + if 'error' in self.info: + self.fail(self.info['error']) + else: + self.fail(_("File status: " + statusMap[status])) finally: - self.log_debug("File status: %s" % statusMap[status]) + self.log_info(_("File status: ") + (statusMap[status] if status else _("Unknown"))) def check_name_size(self, getinfo=True): if not self.info or getinfo: - self.log_debug("Update file info...") - self.log_debug("Previous file info: %s" % self.info) + self.log_info(_("Updating file info...")) + old_info = self.info.copy() self.info.update(self.get_info(self.pyfile.url, self.html)) - self.log_debug("Current file info: %s" % self.info) + self.log_debug("File info: %s" % self.info) + self.log_debug("Previous file info: %s" % old_info) try: url = self.info['url'].strip() name = self.info['name'].strip() + + except KeyError: + pass + + else: if name and name is not url: self.pyfile.name = name - except Exception: - pass + if 'size' in self.info and self.info['size'] > 0: + self.pyfile.size = int(self.info['size']) #@TODO: Fix int conversion in 0.4.10 - try: - size = self.info['size'] - if size > 0: - self.pyfile.size = size + # self.pyfile.sync() - except Exception: - pass + name = self.pyfile.name + size = self.pyfile.size + folder = self.info['folder'] = name - self.log_debug("File name: %s" % self.pyfile.name, - "File size: %s byte" % self.pyfile.size if self.pyfile.size > 0 else "File size: Unknown") + self.log_info(_("File name: ") + name) + self.log_info(_("File size: %s bytes") % size if size > 0 else _("File size: Unknown")) + # self.log_info("File folder: " + folder) + #@TODO: Rewrite in 0.4.10 def check_info(self): self.check_name_size() @@ -507,13 +533,7 @@ class SimpleHoster(Hoster): def handle_direct(self, pyfile): - link = self.direct_link(pyfile.url, self.resume_download) - - if link: - self.log_info(_("Direct download link detected")) - self.link = link - else: - self.log_debug("Direct download link not found") + self.link = self.direct_link(pyfile.url, self.resume_download) def handle_multi(self, pyfile): #: Multi-hoster handler @@ -534,7 +554,7 @@ class SimpleHoster(Hoster): def handle_premium(self, pyfile): if not hasattr(self, 'LINK_PREMIUM_PATTERN'): self.log_error(_("Premium download not implemented")) - self.log_debug("Handled as free download") + self.log_info(_("Processing as free download...")) self.handle_free(pyfile) m = re.search(self.LINK_PREMIUM_PATTERN, self.html) @@ -542,12 +562,3 @@ class SimpleHoster(Hoster): self.error(_("Premium download link not found")) else: self.link = m.group(1) - - - def retry_free(self): - if not self.premium: - return - self.premium = False - self.account = None - self.req = self.pyload.requestFactory.getRequest(self.__name__) - raise Retry(_("Fallback to free download")) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 6508b9bd3..962cffb06 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -13,7 +13,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.54" + __version__ = "0.55" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -27,7 +27,7 @@ class XFSHoster(SimpleHoster): HOSTER_DOMAIN = None - MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... + LEECH_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... NAME_PATTERN = r'(Filename[ ]*:[ ]*()?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P.+?)(\s*<|")' SIZE_PATTERN = r'(Size[ ]*:[ ]*()?|File:.*>|\s*\(|<[\w^_]+ class="size">)\s*(?P[\d.,]+)\s*(?P[\w^_]+)' @@ -110,8 +110,8 @@ class XFSHoster(SimpleHoster): if m: break else: - self.log_error(data['op'] if 'op' in data else _("UNKNOWN")) - return "" + if 'op' in data: + self.error(_("Missing OP data after: ") + data['op']) self.link = m.group(1) -- cgit v1.2.3 From 34f48259060656077b5cb45edd8f9d92bb0282de Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Jul 2015 20:19:03 +0200 Subject: Bunch of fixups --- module/plugins/internal/Account.py | 40 +++++++++++++++++++-------------- module/plugins/internal/Container.py | 5 +++-- module/plugins/internal/Hoster.py | 21 ++++++++--------- module/plugins/internal/MultiHoster.py | 2 +- module/plugins/internal/Plugin.py | 16 +++++++++++-- module/plugins/internal/SimpleHoster.py | 4 ++-- module/plugins/internal/XFSAccount.py | 8 +++---- 7 files changed, 58 insertions(+), 38 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index f1a30a9cb..b3d9c97f9 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -60,7 +60,7 @@ class Account(Plugin): self.login(user, info['login']['password'], info['data'], self.req) except Exception, e: - self.log_warning(_("Could not login with username ") + user, e) + self.log_warning(_("Could not login user `%s`") % user, e) res = info['login']['valid'] = False if self.pyload.debug: traceback.print_exc() @@ -77,6 +77,8 @@ class Account(Plugin): def relogin(self, user): + self.log_info(_("Relogin user `%s`...") % user) + with self.get_request(user) as req: req.clearCookies() @@ -90,18 +92,27 @@ class Account(Plugin): def init_accounts(self, accounts): for user, data in accounts.items(): self.add(user, data['password'], data['options']) - self._login(user) @lock def add(self, user, password=None, options={}): if user not in self.info: - self.info[user] = {'login': {'valid': None, 'password': password or "", 'timestamp': 0}, - 'data' : {'options': options, 'timestamp': 0}} + self.info[user] = {'login': {'valid' : None, + 'password' : password or "", + 'timestamp': 0}, + 'data' : {'options' : options, + 'premium' : None, + 'validuntil' : None, + 'trafficleft': None, + 'maxtraffic' : None, + 'timestamp' : 0}} + + self.log_info(_("Login user `%s`...") % user) self._login(user) return True + else: - self.log_error(_("Error adding account"), _("User already exists")) + self.log_error(_("Error adding user `%s`") % user, _("User already exists")) @lock @@ -172,11 +183,11 @@ class Account(Plugin): :return: dictionary with information """ if user not in self.info: - self.log_error(_("User %s not found while retrieving account info") % user) + self.log_error(_("User `%s` not found while retrieving account info") % user) return elif reload: - self.log_debug("Get Account Info for: %s" % user) + self.log_info(_("Parsing account info for user `%s`...") % user) info = self._parse_info(user) else: @@ -188,7 +199,7 @@ class Account(Plugin): safe_info = info.copy() safe_info['login']['password'] = "**********" - self.log_debug("Account info: %s" % safe_info) + self.log_debug("Account info for user `%s`: %s" % (user, safe_info)) return info @@ -219,16 +230,11 @@ class Account(Plugin): data.update(extra_info) except Exception, e: - self.log_warning(_("Error loading info for ") + user, e) + self.log_warning(_("Error loading info for user `%s`") % user, e) if self.pyload.debug: traceback.print_exc() - else: - for key in ('premium', 'validuntil', 'trafficleft', 'maxtraffic'): - if key not in data: - data[key] = None - finally: if self.req: self.req.close() @@ -324,7 +330,7 @@ class Account(Plugin): if user not in self.info: return - self.log_warning(_("Account %s has not enough traffic, checking again in 30min") % user) + self.log_warning(_("Account `%s` has not enough traffic") % user, _("Checking again in 30 minutes")) self.info[user]['data'].update({'trafficleft': 0}) self.schedule_refresh(user, 30 * 60) @@ -334,7 +340,7 @@ class Account(Plugin): if user not in self.info: return - self.log_warning(_("Account %s is expired, checking again in 1h") % user) + self.log_warning(_("Account `%s` is expired") % user, _("Checking again in 60 minutes")) self.info[user]['data'].update({'validuntil': time.time() - 1}) self.schedule_refresh(user, 60 * 60) @@ -344,7 +350,7 @@ class Account(Plugin): """ Add task to refresh account info to sheduler """ - self.log_debug("Scheduled refresh for %s in %s seconds" % (user, time)) + self.log_debug("Scheduled refresh for user `%s` in %s seconds" % (user, time)) self.pyload.scheduler.addJob(time, self.get_info, [user, reload]) diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index ec2d429dd..729592a0d 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -7,6 +7,7 @@ import re import traceback from module.plugins.internal.Crypter import Crypter +from module.plugins.internal.Plugin import exists from module.utils import save_join as fs_join @@ -66,8 +67,8 @@ class Container(Crypter): else: self.pyfile.name = os.path.basename(self.pyfile.url) - if not os.path.exists(self.pyfile.url): - if os.path.exists(fs_join(pypath, self.pyfile.url)): + if not exists(self.pyfile.url): + if exists(fs_join(pypath, self.pyfile.url)): self.pyfile.url = fs_join(pypath, self.pyfile.url) else: self.fail(_("File not exists")) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index c35178547..8a11499fa 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -15,9 +15,9 @@ if os.name != "nt": from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, encode, fixurl as _fixurl, replace_patterns, seconds_to_midnight, - set_cookies, parse_html_form, parse_html_tag_attr_value, - timestamp) + chunks, encode, exists, fixurl as _fixurl, replace_patterns, + seconds_to_midnight, set_cookies, parse_html_form, + parse_html_tag_attr_value, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename @@ -249,7 +249,7 @@ class Hoster(Plugin): if pyfile.abort: self.abort() - time.sleep(3) + time.sleep(2) else: while pyfile.waitUntil > time.time(): @@ -261,8 +261,8 @@ class Hoster(Plugin): self.wantReconnect = False raise Reconnect - self.thread.m.reconnecting.wait(3) - time.sleep(3) + self.thread.m.reconnecting.wait(2) + time.sleep(2) self.waiting = False pyfile.status = status #@NOTE: Remove in 0.4.10 @@ -386,7 +386,7 @@ class Hoster(Plugin): location = fs_join(download_folder, self.pyfile.package().folder) - if not os.path.exists(location): + if not exists(location): try: os.makedirs(location, int(self.pyload.config.get("permission", "folder"), 8)) @@ -456,7 +456,7 @@ class Hoster(Plugin): do_delete = False last_download = fs_encode(self.last_download) - if not self.last_download or not os.path.exists(last_download): + if not self.last_download or not exists(last_download): self.last_download = "" self.fail(self.pyfile.error or _("No file downloaded")) @@ -506,6 +506,7 @@ class Hoster(Plugin): traceback.print_exc() else: + self.last_download = "" self.log_info(_("File deleted")) @@ -647,14 +648,14 @@ class Hoster(Plugin): download_folder = self.pyload.config.get("general", "download_folder") location = fs_join(download_folder, pack.folder, self.pyfile.name) - if starting and self.pyload.config.get("download", "skip_existing") and os.path.exists(location): + if starting and self.pyload.config.get("download", "skip_existing") and exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: self.skip("File exists") pyfile = self.pyload.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: - if os.path.exists(location): + if exists(location): self.skip(pyfile[0]) self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name) diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index b569b04c3..5058cf515 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -80,7 +80,7 @@ class MultiHoster(SimpleHoster): if not self.last_download: self.log_info(_("Downloading file...")) - self.download(self.link, ref=False, disposition=True) + self.download(self.link, disposition=True) self.check_file() diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index f88660f24..e781ca297 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -29,6 +29,18 @@ def encode(string, encoding='utf8'): return string +#@TODO: Move to utils in 0.4.10 +def exists(path): + if os.path.exists(path): + if os.name == "nt": + dir, name = os.path.split(path) + return name in os.listdir(dir) + else: + return True + else: + return False + + #@TODO: Move to utils in 0.4.10 def fixurl(url): return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip() @@ -158,7 +170,7 @@ class Plugin(object): log("%(type)s %(plugin)s%(id)s : %(msg)s" % {'type': self.__type__.upper(), 'plugin': self.__name__, 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", - 'msg' : msg or "---------- MARK ----------"}) + 'msg' : msg}) def log_debug(self, *args): @@ -290,7 +302,7 @@ class Plugin(object): frame = inspect.currentframe() framefile = fs_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) try: - if not os.path.exists(os.path.join("tmp", self.__name__)): + if not exists(os.path.join("tmp", self.__name__)): os.makedirs(os.path.join("tmp", self.__name__)) with open(framefile, "wb") as f: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e8281b943..488f87114 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -294,7 +294,7 @@ class SimpleHoster(Hoster): if not self.last_download: self.log_info(_("Downloading file...")) - self.download(self.link, ref=False, disposition=self.DISPOSITION) + self.download(self.link, disposition=self.DISPOSITION) self.check_file() @@ -346,7 +346,7 @@ class SimpleHoster(Hoster): self.html = f.read(50000) #@TODO: Recheck in 0.4.10 self.check_errors() - self.log_debug("No errors found") + self.log_info(_("No errors found")) self.pyfile.error = "" diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 7df169ee9..6d6cc97e7 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -53,7 +53,7 @@ class XFSAccount(Account): set_cookies(req.cj, self.COOKIES) - def parse_info(self, user, req): + def parse_info(self, user, password, data, req): validuntil = None trafficleft = None leechtraffic = None @@ -150,7 +150,7 @@ class XFSAccount(Account): 'premium' : premium} - def login(self, user, password, info, req): + def login(self, user, password, data, req): if not self.HOSTER_URL: #@TODO: Remove in 0.4.10 raise Exception(_("Missing HOSTER_DOMAIN")) @@ -164,7 +164,7 @@ class XFSAccount(Account): 'redirect': self.HOSTER_URL} inputs.update({'login' : user, - 'password': data['password']}) + 'password': password}) if action: url = urlparse.urljoin("http://", action) @@ -174,4 +174,4 @@ class XFSAccount(Account): html = self.load(url, post=inputs) if re.search(self.LOGIN_FAIL_PATTERN, html): - self.fail() + self.login_fail() -- cgit v1.2.3 From 0fd83be485e6cca2f6b18a08dfa4328d2e793230 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Jul 2015 20:16:07 +0200 Subject: Fix captcha timeout --- module/plugins/internal/Captcha.py | 15 +++++++++------ module/plugins/internal/SimpleHoster.py | 7 ++----- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 32d882fac..8ba1a1829 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -10,7 +10,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.32" + __version__ = "0.33" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -46,13 +46,13 @@ class Captcha(Plugin): def decrypt(self, url, get={}, post={}, ref=False, cookies=False, decode=False, - input_type='jpg', output_type='textual', ocr=True): + input_type='jpg', output_type='textual', ocr=True, timeout=300): img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode) - return self._decrypt(img, input_type, output_type, ocr) + return self._decrypt(img, input_type, output_type, ocr, timeout) - #@TODO: Definitely dhoose a better name for this method! - def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=None): + #@TODO: Definitely choose a better name for this method! + def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=None, timeout=300): """ Loads a captcha and decrypts it with ocr, plugin, user input @@ -91,11 +91,13 @@ class Captcha(Plugin): try: self.task = captchaManager.newTask(raw, input_type, tmp_img.name, output_type) captchaManager.handleCaptcha(self.task) + self.task.setWaiting(max(timeout, 50)) #@TODO: Move to `CaptchaManager` in 0.4.10 while self.task.isWaiting(): if self.plugin.pyfile.abort: self.plugin.abort() time.sleep(1) + finally: captchaManager.removeTask(self.task) @@ -103,7 +105,8 @@ class Captcha(Plugin): self.fail(task.error) elif not self.task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) + self.captcha.invalid() + self.plugin.retry(reason=_("No captcha result obtained in appropiate time")) result = self.task.result self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 488f87114..a80bc94cb 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -24,7 +24,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.72" + __version__ = "1.73" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -301,10 +301,7 @@ class SimpleHoster(Hoster): except Fail, e: #@TODO: Move to PluginThread in 0.4.10 err = str(e) #@TODO: Recheck in 0.4.10 - if err is _("No captcha result obtained in appropiate time by any of the plugins."): #@TODO: Fix in 0.4.10 - self.check_file() - - elif self.get_config('fallback', True) and self.premium: + if self.get_config('fallback', True) and self.premium: self.log_warning(_("Premium download failed"), e) self.restart(reset=True) -- cgit v1.2.3 From 95ed76d34290e08876dccce6840c3e09138a2047 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Jul 2015 23:47:47 +0200 Subject: Spare code fixes --- module/plugins/internal/DeadCrypter.py | 5 ++--- module/plugins/internal/DeadHoster.py | 5 ++--- module/plugins/internal/Hoster.py | 12 +++++++++--- module/plugins/internal/MultiHoster.py | 10 ++++------ module/plugins/internal/Plugin.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 8 +++----- module/plugins/internal/UnRar.py | 4 ++-- module/plugins/internal/XFSAccount.py | 4 ++-- 8 files changed, 26 insertions(+), 26 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 7d14f5378..e7cc6ff39 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -6,7 +6,7 @@ from module.plugins.internal.Crypter import Crypter, create_getInfo class DeadCrypter(Crypter): __name__ = "DeadCrypter" __type__ = "crypter" - __version__ = "0.08" + __version__ = "0.09" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -24,8 +24,7 @@ class DeadCrypter(Crypter): def setup(self): - self.pyfile.error = "Crypter is no longer available" - self.offline() #@TODO: self.offline("Crypter is no longer available") + self.offline(_("Crypter is no longer available")) getInfo = create_getInfo(DeadCrypter) diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index f8e68f2b3..250287d25 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -6,7 +6,7 @@ from module.plugins.internal.Hoster import Hoster, create_getInfo class DeadHoster(Hoster): __name__ = "DeadHoster" __type__ = "hoster" - __version__ = "0.18" + __version__ = "0.19" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -24,8 +24,7 @@ class DeadHoster(Hoster): def setup(self): - self.pyfile.error = "Hoster is no longer available" - self.offline() #@TODO: self.offline("Hoster is no longer available") + self.offline(_("Hoster is no longer available")) getInfo = create_getInfo(DeadHoster) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 8a11499fa..3d4c11af9 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -272,15 +272,17 @@ class Hoster(Plugin): """ Skip and give reason """ - raise Skip(encode(reason)) + raise Skip(encode(reason)) #@TODO: Remove `encode` in 0.4.10 def abort(self, reason=""): """ Abort and give reason """ + #@TODO: Remove in 0.4.10 if reason: self.pyfile.error = encode(reason) + raise Abort @@ -288,8 +290,10 @@ class Hoster(Plugin): """ Fail and indicate file is offline """ + #@TODO: Remove in 0.4.10 if reason: self.pyfile.error = encode(reason) + raise Fail("offline") @@ -297,8 +301,10 @@ class Hoster(Plugin): """ Fail and indicates file ist temporary offline, the core may take consequences """ + #@TODO: Remove in 0.4.10 if reason: self.pyfile.error = encode(reason) + raise Fail("temp. offline") @@ -320,7 +326,7 @@ class Hoster(Plugin): self.wait(wait_time, False) self.retries[id] += 1 - raise Retry(reason) + raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 def restart(self, reason=None, reset=False): @@ -335,7 +341,7 @@ class Hoster(Plugin): self.account = None self.req = self.pyload.requestFactory.getRequest(self.__name__) - raise Retry(reason) + raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 def fixurl(self, url): diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 5058cf515..e32c3cdc5 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -2,7 +2,7 @@ import re -from module.plugins.internal.Plugin import Fail, Retry +from module.plugins.internal.Plugin import Fail from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies @@ -85,11 +85,9 @@ class MultiHoster(SimpleHoster): self.check_file() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 - err = str(e) #@TODO: Recheck in 0.4.10 - if self.premium: self.log_warning(_("Premium download failed")) - self.retry_free() + self.restart(reset=True) elif self.get_config("revertfailed", True) \ and "new_module" in self.pyload.pluginManager.hosterPlugins[self.__name__]: @@ -105,10 +103,10 @@ class MultiHoster(SimpleHoster): hdict['new_module'] = tmp_module hdict['new_name'] = tmp_name - raise Retry(_("Revert to original hoster plugin")) + self.restart(_("Revert to original hoster plugin")) else: - raise Fail(err) + raise Fail(encode(e)) #@TODO: Remove `encode` in 0.4.10 def handle_premium(self, pyfile): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index e781ca297..52c22b527 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -245,7 +245,7 @@ class Plugin(object): """ Fail and give reason """ - raise Fail(encode(reason)) #: Move `encode(reason)` to manager in 0.4.10 + raise Fail(encode(reason)) #@TODO: Remove `encode` in 0.4.10 def error(self, reason="", type=_("Parse")): @@ -256,7 +256,7 @@ class Plugin(object): msg += (": %s" % reason.strip()) if reason else "" msg += _(" | Plugin may be out of date") - raise Fail(msg) + raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=True, req=None): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index a80bc94cb..936887342 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -13,7 +13,7 @@ from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, Retry, fixurl, replace_patterns, seconds_to_midnight, set_cookies +from module.plugins.internal.Plugin import Fail, encode, fixurl, replace_patterns, seconds_to_midnight, set_cookies from module.utils import fixup, fs_encode, parseFileSize as parse_size @@ -211,7 +211,7 @@ class SimpleHoster(Hoster): self.leech_dl = False if not self.get_config('use_premium', True): - self.retry_free() + self.restart(reset=True) if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) @@ -299,14 +299,12 @@ class SimpleHoster(Hoster): self.check_file() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 - err = str(e) #@TODO: Recheck in 0.4.10 - if self.get_config('fallback', True) and self.premium: self.log_warning(_("Premium download failed"), e) self.restart(reset=True) else: - raise Fail(err) + raise Fail(encode(e)) #@TODO: Remove `encode` in 0.4.10 def check_file(self): diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index a2c2070b8..30b763922 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.22" + __version__ = "1.23" __status__ = "testing" __description__ = """Rar extractor plugin""" @@ -65,7 +65,7 @@ class UnRar(Extractor): try: if os.name == "nt": cls.CMD = os.path.join(pypath, "UnRAR.exe") - else: + else: cls.CMD = "unrar" p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 6d6cc97e7..93a04dc86 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -151,8 +151,8 @@ class XFSAccount(Account): def login(self, user, password, data, req): - if not self.HOSTER_URL: #@TODO: Remove in 0.4.10 - raise Exception(_("Missing HOSTER_DOMAIN")) + if not self.HOSTER_URL: + self.login_fail(_("Missing HOSTER_DOMAIN")) if not self.LOGIN_URL: self.LOGIN_URL = urlparse.urljoin(self.HOSTER_URL, "login.html") -- cgit v1.2.3 From 2e4f84afb8ae79241c805e441281ff14a2c50783 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 00:14:43 +0200 Subject: Fix https://github.com/pyload/pyload/pull/1564 (thx Nippey) --- module/plugins/internal/Captcha.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 8ba1a1829..57105caea 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -46,13 +46,13 @@ class Captcha(Plugin): def decrypt(self, url, get={}, post={}, ref=False, cookies=False, decode=False, - input_type='jpg', output_type='textual', ocr=True, timeout=300): + input_type='jpg', output_type='textual', ocr=True, timeout=120): img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode) return self._decrypt(img, input_type, output_type, ocr, timeout) #@TODO: Definitely choose a better name for this method! - def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=None, timeout=300): + def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=None, timeout=120): """ Loads a captcha and decrypts it with ocr, plugin, user input -- cgit v1.2.3 From 6f387ff66b93e24cc52db7a5ea277265d35f051d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 00:29:19 +0200 Subject: Missed to bump up some plugin __version__ --- module/plugins/internal/Captcha.py | 2 +- module/plugins/internal/Hoster.py | 2 +- module/plugins/internal/MultiHoster.py | 2 +- module/plugins/internal/Plugin.py | 2 +- module/plugins/internal/SimpleHoster.py | 2 +- module/plugins/internal/XFSAccount.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 57105caea..db2d25a8a 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -10,7 +10,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.33" + __version__ = "0.34" __status__ = "testing" __description__ = """Base anti-captcha plugin""" diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 3d4c11af9..f3b78a348 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.07" __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index e32c3cdc5..aec83e246 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.44" + __version__ = "0.45" __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 52c22b527..fb62712fa 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -137,7 +137,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.14" + __version__ = "0.15" __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 936887342..3f88dfbcc 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -24,7 +24,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.73" + __version__ = "1.74" __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 93a04dc86..5bf959400 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -11,7 +11,7 @@ from module.plugins.internal.Plugin import parse_html_form, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.39" + __version__ = "0.40" __status__ = "testing" __description__ = """XFileSharing account plugin""" -- cgit v1.2.3 From ecf37227de05c73b7ffe2da89a5eda1259a72543 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 01:09:59 +0200 Subject: Improve _log method --- module/plugins/internal/Captcha.py | 6 +++--- module/plugins/internal/Extractor.py | 6 +++--- module/plugins/internal/Hook.py | 7 ++++++- module/plugins/internal/OCR.py | 6 +++--- module/plugins/internal/Plugin.py | 25 +++++++++++++------------ 5 files changed, 28 insertions(+), 22 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index db2d25a8a..486d92c19 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -10,7 +10,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.34" + __version__ = "0.35" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -34,8 +34,8 @@ class Captcha(Plugin): pass - def _log(self, level, args): - return self.plugin._log(level, (self.__name__,) + args) + def _log(self, level, plugintype, pluginname, messages): + return self.plugin._log(level, plugintype, pluginname, (self.__name__,) + messages) def recognize(self, image): diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index cf9a1f6e3..cbfae5a4d 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.27" + __version__ = "0.28" __status__ = "testing" __description__ = """Base extractor plugin""" @@ -113,8 +113,8 @@ class Extractor(Plugin): pass - def _log(self, level, args): - return self.plugin._log(level, (self.__name__,) + args) + def _log(self, level, plugintype, pluginname, messages): + return self.plugin._log(level, plugintype, pluginname, (self.__name__,) + messages) def check(self): diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index fbd6934ce..3d8fb0dde 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -6,7 +6,7 @@ from module.plugins.internal.Addon import Addon class Hook(Addon): __name__ = "Hook" __type__ = "hook" - __version__ = "0.10" + __version__ = "0.11" __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] @@ -21,3 +21,8 @@ class Hook(Addon): def __init__(self, core, manager): super(Hook, self).__init__(core, manager) self.init_periodical(10) + + + #@TODO: Remove in 0.4.10 + def _log(self, level, plugintype, pluginname, messages): + return super(self.__name__, self)._log(level, plugintype, pluginname.replace("Hook", ""), messages) diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 37a33206f..998b3f040 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -21,7 +21,7 @@ from module.utils import save_join as fs_join class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" - __version__ = "0.14" + __version__ = "0.15" __status__ = "testing" __description__ = """OCR base plugin""" @@ -44,8 +44,8 @@ class OCR(Plugin): pass - def _log(self, level, args): - return self.plugin._log(level, (self.__name__,) + args) + def _log(self, level, plugintype, pluginname, messages): + return self.plugin._log(level, plugintype, pluginname, (self.__name__,) + messages) def load_image(self, image): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index fb62712fa..638a1ee1f 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -137,7 +137,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.15" + __version__ = "0.16" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -164,34 +164,35 @@ class Plugin(object): pass - def _log(self, level, args): + def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) - msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in args if a)) - log("%(type)s %(plugin)s%(id)s : %(msg)s" % {'type': self.__type__.upper(), - 'plugin': self.__name__, - 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", - 'msg' : msg}) + msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in messages if a)) + log("%(plugintype)s %(pluginname)s%(id)s : %(msg)s" + % {'plugintype': plugintype.upper(), + 'pluginname': pluginname, + 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", + 'msg' : msg}) def log_debug(self, *args): if self.pyload.debug: - return self._log("debug", args) + return self._log("debug", self.__type__, self.__name__, args) def log_info(self, *args): - return self._log("info", args) + return self._log("info", self.__type__, self.__name__, args) def log_warning(self, *args): - return self._log("warning", args) + return self._log("warning", self.__type__, self.__name__, args) def log_error(self, *args): - return self._log("error", args) + return self._log("error", self.__type__, self.__name__, args) def log_critical(self, *args): - return self._log("critical", args) + return self._log("critical", self.__type__, self.__name__, args) def set_config(self, option, value): -- cgit v1.2.3 From 179b7a30c15e298b612509c474503155df9dddb9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 01:47:40 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1575 --- module/plugins/internal/Captcha.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 486d92c19..dd2c7c762 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import with_statement + import os import time import traceback @@ -10,7 +12,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.35" + __version__ = "0.36" __status__ = "testing" __description__ = """Base anti-captcha plugin""" -- cgit v1.2.3 From 4135729a0d32abdbe14e422ff91aef74a8e2092c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 02:53:25 +0200 Subject: [Hook] Hotfix _log method --- module/plugins/internal/Hook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 3d8fb0dde..36c85fc19 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -6,7 +6,7 @@ from module.plugins.internal.Addon import Addon class Hook(Addon): __name__ = "Hook" __type__ = "hook" - __version__ = "0.11" + __version__ = "0.12" __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] @@ -25,4 +25,4 @@ class Hook(Addon): #@TODO: Remove in 0.4.10 def _log(self, level, plugintype, pluginname, messages): - return super(self.__name__, self)._log(level, plugintype, pluginname.replace("Hook", ""), messages) + return super(Addon, self)._log(level, plugintype, pluginname.replace("Hook", ""), messages) -- cgit v1.2.3 From 127e42ec74b276fd43618af2a8f3d450a8437af7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 04:43:10 +0200 Subject: [Account] Hotfix --- module/plugins/internal/Account.py | 94 +++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 32 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index b3d9c97f9..766fe3f71 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement - import random import time import threading import traceback +from operator import itemgetter + from module.plugins.internal.Plugin import Plugin from module.utils import compare_time, lock, parseFileSize as parse_size @@ -14,7 +14,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.06" + __version__ = "0.07" __status__ = "testing" __description__ = """Base account plugin""" @@ -32,6 +32,8 @@ class Account(Plugin): self.info = {} #: Provide information in dict here self.lock = threading.RLock() + self.accounts = accounts #@TODO: Remove in 0.4.10 + self.init() self.init_accounts(accounts) @@ -62,11 +64,14 @@ class Account(Plugin): except Exception, e: self.log_warning(_("Could not login user `%s`") % user, e) res = info['login']['valid'] = False + self.accounts[user]['valid'] = False #@TODO: Remove in 0.4.10 + if self.pyload.debug: traceback.print_exc() else: res = info['login']['valid'] = True + self.accounts[user]['valid'] = True #@TODO: Remove in 0.4.10 finally: if self.req: @@ -79,8 +84,10 @@ class Account(Plugin): def relogin(self, user): self.log_info(_("Relogin user `%s`...") % user) - with self.get_request(user) as req: + req = self.get_request(user) + if req: req.clearCookies() + req.close() if user in self.info: self.info[user]['login'].clear() @@ -107,6 +114,13 @@ class Account(Plugin): 'maxtraffic' : None, 'timestamp' : 0}} + #@TODO: Remove in 0.4.10 + self.accounts[user] = self.info[user]['data'] + self.accounts[user].update({'login' : user, + 'type' : self.__name__, + 'valid' : self.info[user]['login']['valid'], + 'password': self.info[user]['login']['password']}) + self.log_info(_("Login user `%s`...") % user) self._login(user) return True @@ -120,20 +134,25 @@ class Account(Plugin): """ Updates account and return true if anything changed """ + if not password or not options: + return + if user not in self.info: return self.add(user, password, options) - elif password or options: + else: if password: self.info[user]['login']['password'] = password + self.accounts[user]['password'] = password #@TODO: Remove in 0.4.10 self.relogin(user) - return True if options: before = self.info[user]['data'][user]['options'] self.info[user]['data']['options'].update(options) return self.info[user]['data']['options'] != before + return True + #: Deprecated method, use `update` instead (Remove in 0.4.10) def updateAccounts(self, *args, **kwargs): @@ -143,9 +162,11 @@ class Account(Plugin): def remove(self, user=None): # -> def remove if not user: self.info.clear() + self.accounts.clear() #@TODO: Remove in 0.4.10 elif user in self.info: self.info.pop(user, None) + self.accounts.pop(user, None) #@TODO: Remove in 0.4.10 #: Deprecated method, use `remove` instead (Remove in 0.4.10) @@ -167,11 +188,14 @@ class Account(Plugin): if 'force' in kwargs: kwargs['reload'] = kwargs['force'] kwargs.pop('force', None) + data = self.get_data(*args, **kwargs) or {} if 'options' not in data: data['options'] = {'limitdl': ['0']} + return data + @lock def get_info(self, user, reload=False): """ @@ -199,7 +223,9 @@ class Account(Plugin): safe_info = info.copy() safe_info['login']['password'] = "**********" + safe_info['data']['password'] = "**********" #@TODO: Remove in 0.4.10 self.log_debug("Account info for user `%s`: %s" % (user, safe_info)) + return info @@ -213,21 +239,15 @@ class Account(Plugin): def _parse_info(self, user): info = self.info[user] - data = info['data'] - - #@TODO: Remove in 0.4.10 - data.update({'login': user, - 'type' : self.__name__, - 'valid': self.info[user]['login']['valid']}) try: - data['timestamp'] = time.time() #: Set timestamp for login + info['data']['timestamp'] = time.time() #: Set timestamp for login self.req = self.get_request(user) extra_info = self.parse_info(user, info['login']['password'], info, self.req) if extra_info and isinstance(extra_info, dict): - data.update(extra_info) + info['data'].update(extra_info) except Exception, e: self.log_warning(_("Error loading info for user `%s`") % user, e) @@ -279,40 +299,50 @@ class Account(Plugin): return self.pyload.requestFactory.getCookieJar(self.__name__, user) - #@TODO: Random account only? Simply crazy... rewrite def select(self): """ - Returns an valid account name and data + Returns a valid account name and data """ - usable = [] + free_accounts = [] + premium_accounts = [] + for user, info in self.info.items(): if not info['login']['valid']: continue - options = info['data']['options'] - if "time" in options and options['time']: + data = info['data'] + + if "time" in data['options'] and data['options']['time']: time_data = "" try: - time_data = options['time'][0] + time_data = data['options']['time'][0] start, end = time_data.split("-") + if not compare_time(start.split(":"), end.split(":")): continue except Exception: - self.log_warning(_("Your time %s has wrong format, use 1:22-3:44") % time_data) + self.log_warning(_("Wrong time format `%s` for account `%s`, use 1:22-3:44") % (user, time_data)) - if user in self.info: - if None is not self.info[user]['validuntil'] > 0 and time.time() > self.info[user]['validuntil']: - continue - if None is not self.info[user]['trafficleft'] == 0: - continue + if data['trafficleft'] == 0: + continue + + if data['validuntil'] > 0 and time.time() > data['validuntil']: + continue + + if data['premium']: + premium_accounts.append((user, info)) + + else: + free_accounts.append((user, info)) - usable.append((user, info)) + all_accounts = premium_accounts or free_accounts + fav_accounts = [(user, info) for user, info in all_accounts if info['validuntil'] is not None] - if not usable: - return None, None + accounts = sorted(fav_accounts or all_accounts, + key=itemgetter("validuntil"), reverse=True) or [(None, None)] - return random.choice(usable) + return accounts[0] def can_use(self): @@ -332,7 +362,7 @@ class Account(Plugin): self.log_warning(_("Account `%s` has not enough traffic") % user, _("Checking again in 30 minutes")) - self.info[user]['data'].update({'trafficleft': 0}) + self.info[user]['data']['trafficleft'] = 0 self.schedule_refresh(user, 30 * 60) @@ -342,7 +372,7 @@ class Account(Plugin): self.log_warning(_("Account `%s` is expired") % user, _("Checking again in 60 minutes")) - self.info[user]['data'].update({'validuntil': time.time() - 1}) + self.info[user]['data']['validuntil'] = time.time() - 1 self.schedule_refresh(user, 60 * 60) -- cgit v1.2.3 From 44e7152a5fa2e6a1b85f5e68100d4d19c03f61e4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 16:50:39 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1577 and https://github.com/pyload/pyload/issues/1580 --- module/plugins/internal/Hook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 36c85fc19..1f566f824 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.Addon import Addon +from module.plugins.internal.Addon import Addon, threaded class Hook(Addon): __name__ = "Hook" __type__ = "hook" - __version__ = "0.12" + __version__ = "0.13" __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] -- cgit v1.2.3 From babf7f311d257d621a101b2d44e74c8e562629bd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 17:08:59 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1578 --- module/plugins/internal/Account.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 766fe3f71..15078b100 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -14,7 +14,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.07" + __version__ = "0.08" __status__ = "testing" __description__ = """Base account plugin""" @@ -327,7 +327,7 @@ class Account(Plugin): if data['trafficleft'] == 0: continue - if data['validuntil'] > 0 and time.time() > data['validuntil']: + if time.time() > data['validuntil'] > 0: continue if data['premium']: @@ -337,7 +337,7 @@ class Account(Plugin): free_accounts.append((user, info)) all_accounts = premium_accounts or free_accounts - fav_accounts = [(user, info) for user, info in all_accounts if info['validuntil'] is not None] + fav_accounts = [(user, info) for user, info in all_accounts if info['data']['validuntil'] is not None] accounts = sorted(fav_accounts or all_accounts, key=itemgetter("validuntil"), reverse=True) or [(None, None)] -- cgit v1.2.3 From 554a4ddde1115ea4fa307bddcd66aec0cbd3f8ee Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 18:43:26 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1578 (2) --- module/plugins/internal/Account.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 15078b100..b8230a09d 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -14,7 +14,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.08" + __version__ = "0.09" __status__ = "testing" __description__ = """Base account plugin""" @@ -301,10 +301,10 @@ class Account(Plugin): def select(self): """ - Returns a valid account name and data + Returns a valid account name and info """ - free_accounts = [] - premium_accounts = [] + free_accounts = {} + premium_accounts = {} for user, info in self.info.items(): if not info['login']['valid']: @@ -331,18 +331,24 @@ class Account(Plugin): continue if data['premium']: - premium_accounts.append((user, info)) + premium_accounts[user] = info else: - free_accounts.append((user, info)) + free_accounts[user] = info - all_accounts = premium_accounts or free_accounts - fav_accounts = [(user, info) for user, info in all_accounts if info['data']['validuntil'] is not None] + account_list = (premium_accounts or free_accounts).items() - accounts = sorted(fav_accounts or all_accounts, - key=itemgetter("validuntil"), reverse=True) or [(None, None)] + if not account_list: + return None, None - return accounts[0] + validuntil_list = [(user, info) for user, info in account_list if info['data']['validuntil']] + + if not validuntil_list: + return random.choice(account_list) #@TODO: Random account?! Recheck in 0.4.10 + + return sorted(validuntil_list, + key=lambda user, info: info['data']['validuntil'], + reverse=True)[0] def can_use(self): -- cgit v1.2.3 From ca5eb403dd6797a3517f6eb5aae7922f586c5c45 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 18:55:09 +0200 Subject: Fix http://forum.pyload.org/viewtopic.php?f=12&t=4410 --- module/plugins/internal/Hoster.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index f3b78a348..15344b86b 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.07" + __version__ = "0.08" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -448,7 +448,7 @@ class Hoster(Plugin): return self.last_download - def check_download(self, rules, delete=False, file_size=0, size_tolerance=2048, read_size=100000): + def check_download(self, rules, delete=False, file_size=0, size_tolerance=1024, read_size=100000): """ Checks the content of the last downloaded file, re match is saved to `lastCheck` diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3f88dfbcc..7a220415c 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -24,7 +24,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.74" + __version__ = "1.75" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -316,6 +316,7 @@ class SimpleHoster(Hoster): elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, file_size=self.info['size'] if 'size' in self.info else 0, + size_tolerance=10240, delete=True): self.error(_("Empty file")) -- cgit v1.2.3 From 9e44724d6dda3e1f47cf4d6de4b3e521a244ba4b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 19:15:28 +0200 Subject: Fix content-disposition --- module/plugins/internal/Account.py | 2 -- module/plugins/internal/Hoster.py | 6 +++--- module/plugins/internal/MultiHoster.py | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index b8230a09d..788b62e7e 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -5,8 +5,6 @@ import time import threading import traceback -from operator import itemgetter - from module.plugins.internal.Plugin import Plugin from module.utils import compare_time, lock, parseFileSize as parse_size diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 15344b86b..fdfe92593 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.08" + __version__ = "0.09" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -355,7 +355,7 @@ class Hoster(Plugin): return url - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ Downloads the content at url to download folder @@ -420,7 +420,7 @@ class Hoster(Plugin): self.pyfile.size = self.req.size if newname: - newname = urlparse.urlparse(newname).path.split('/')[-1] + newname = urlparse.urlparse(newname).path.split('/')[-1].split('*=')[-1] #@TODO: Remove in 0.4.10 if disposition and newname is not name: self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index aec83e246..8e336fbc6 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.45" + __version__ = "0.46" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -80,7 +80,7 @@ class MultiHoster(SimpleHoster): if not self.last_download: self.log_info(_("Downloading file...")) - self.download(self.link, disposition=True) + self.download(self.link, disposition=self.DISPOSITION) self.check_file() -- cgit v1.2.3 From 5745baca2dd9c8831631489781ef950af5184081 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 22:06:31 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1520 --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index fdfe92593..6fe68bb6f 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.09" + __version__ = "0.10" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -420,7 +420,7 @@ class Hoster(Plugin): self.pyfile.size = self.req.size if newname: - newname = urlparse.urlparse(newname).path.split('/')[-1].split('*=')[-1] #@TODO: Remove in 0.4.10 + newname = urlparse.urlparse(newname).path.split('*=')[-1].split('/')[-1] #@TODO: Remove in 0.4.10 if disposition and newname is not name: self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) -- cgit v1.2.3 From 9e3ef8bddc2c599403c4ac5090cd12245e802dee Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 22:19:52 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1578 (3) --- module/plugins/internal/Account.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 788b62e7e..3084f02c7 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -12,7 +12,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.09" + __version__ = "0.10" __status__ = "testing" __description__ = """Base account plugin""" @@ -345,7 +345,7 @@ class Account(Plugin): return random.choice(account_list) #@TODO: Random account?! Recheck in 0.4.10 return sorted(validuntil_list, - key=lambda user, info: info['data']['validuntil'], + key=lambda a: a[1]['data']['validuntil'], reverse=True)[0] -- cgit v1.2.3 From e01689c78d83074a44bb86cc4bede38984082a10 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 23:04:21 +0200 Subject: Fix missing `encode` --- module/plugins/internal/MultiHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 8e336fbc6..06466e286 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -2,14 +2,14 @@ import re -from module.plugins.internal.Plugin import Fail +from module.plugins.internal.Plugin import Fail, encode from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.46" + __version__ = "0.47" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 7ddcad0fef8d0fe3aade83d786c0f1e0fbe1d818 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 29 Jul 2015 07:53:30 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1587 --- module/plugins/internal/Crypter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index c677b2852..13d82c2ea 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -9,7 +9,7 @@ from module.utils import save_path as safe_filename class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.05" + __version__ = "0.06" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -78,7 +78,7 @@ class Crypter(Hoster): "%d links" % len(links), "Saved to folder: %s" % folder if folder else "Saved to download folder") - pid = self.pyload.api.addPackage(name, self.fixurl(links), package_queue) + pid = self.pyload.api.addPackage(name, map(self.fixurl, links), package_queue) if package_password: self.pyload.api.setPackageData(pid, {'password': package_password}) -- cgit v1.2.3 From 5a2781c923ecd13f3e671366fa6fa311d92d8547 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 29 Jul 2015 08:21:04 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1586 --- module/plugins/internal/Captcha.py | 9 ++++++--- module/plugins/internal/Extractor.py | 9 ++++++--- module/plugins/internal/SevenZip.py | 14 ++++++-------- module/plugins/internal/UnRar.py | 20 ++++++++------------ module/plugins/internal/UnZip.py | 4 ++-- 5 files changed, 28 insertions(+), 28 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index dd2c7c762..92bcf9f84 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -12,7 +12,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.36" + __version__ = "0.37" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -37,7 +37,10 @@ class Captcha(Plugin): def _log(self, level, plugintype, pluginname, messages): - return self.plugin._log(level, plugintype, pluginname, (self.__name__,) + messages) + return self.plugin._log(level, + plugintype, + "%s: %s" % (self.plugin.__name__, self.__name__), + messages) def recognize(self, image): @@ -107,7 +110,7 @@ class Captcha(Plugin): self.fail(task.error) elif not self.task.result: - self.captcha.invalid() + self.invalid() self.plugin.retry(reason=_("No captcha result obtained in appropiate time")) result = self.task.result diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index cbfae5a4d..907113a7e 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.28" + __version__ = "0.29" __status__ = "testing" __description__ = """Base extractor plugin""" @@ -33,7 +33,7 @@ class Extractor(Plugin): EXTENSIONS = [] REPAIR = False - VERSION = "" + VERSION = None @classmethod @@ -114,7 +114,10 @@ class Extractor(Plugin): def _log(self, level, plugintype, pluginname, messages): - return self.plugin._log(level, plugintype, pluginname, (self.__name__,) + messages) + return self.plugin._log(level, + plugintype, + "%s: %s" % (self.plugin.__name__, self.__name__), + messages) def check(self): diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index a5ccf2a26..5811c28de 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -10,7 +10,7 @@ from module.utils import fs_encode, save_join as fs_join class SevenZip(UnRar): __name__ = "SevenZip" - __version__ = "0.13" + __version__ = "0.14" __status__ = "testing" __description__ = """7-Zip extractor plugin""" @@ -19,9 +19,7 @@ class SevenZip(UnRar): ("Walter Purcaro", "vuolter@gmail.com")] - CMD = "7z" - VERSION = "" - + CMD = "7z" EXTENSIONS = [".7z", ".xz", ".zip", ".gz", ".gzip", ".tgz", ".bz2", ".bzip2", ".tbz2", ".tbz", ".tar", ".wim", ".swm", ".lzma", ".rar", ".cab", ".arj", ".z", ".taz", ".cpio", ".rpm", ".deb", ".lzh", ".lha", @@ -50,11 +48,11 @@ class SevenZip(UnRar): return False else: - return True - - finally: m = cls.re_version.search(out) - cls.VERSION = m.group(1) if m else '(version unknown)' + if m is not None: + cls.VERSION = m.group(1) + + return True def verify(self, password): diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 30b763922..92128c77f 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.23" + __version__ = "1.24" __status__ = "testing" __description__ = """Rar extractor plugin""" @@ -32,11 +32,9 @@ class UnRar(Extractor): ("Immenz" , "immenz@gmx.net" )] - CMD = "unrar" - VERSION = "" + CMD = "unrar" EXTENSIONS = [".rar"] - re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?(\.rev|\.bad)?', re.I) re_filefixed = re.compile(r'Building (.+)') @@ -58,7 +56,7 @@ class UnRar(Extractor): p = subprocess.Popen([cls.CMD], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() - cls.__name__ = "RAR" + # cls.__name__ = "RAR" cls.REPAIR = True except OSError: @@ -74,12 +72,11 @@ class UnRar(Extractor): except OSError: return False - else: - return True + m = cls.re_version.search(out) + if m is not None: + cls.VERSION = m.group(1) - finally: - m = cls.re_version.search(out) - cls.VERSION = m.group(1) if m else '(version unknown)' + return True @classmethod @@ -206,8 +203,7 @@ class UnRar(Extractor): result.add(fs_join(self.out, os.path.basename(f))) else: for f in fs_decode(out).splitlines(): - f = f.strip() - result.add(fs_join(self.out, f)) + result.add(fs_join(self.out, f.strip())) return list(result) diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index aadc303aa..9a01611bf 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -12,7 +12,7 @@ from module.utils import fs_encode class UnZip(Extractor): __name__ = "UnZip" - __version__ = "1.14" + __version__ = "1.15" __status__ = "testing" __description__ = """Zip extractor plugin""" @@ -20,7 +20,7 @@ class UnZip(Extractor): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - VERSION ="(python %s.%s.%s)" % (sys.version_info[0], sys.version_info[1], sys.version_info[2]) + VERSION = "%s.%s.%s" % (sys.version_info[0], sys.version_info[1], sys.version_info[2]) EXTENSIONS = [".zip", ".zip64"] -- cgit v1.2.3 From e42f5783442fcbaa53e6c0faf943dd33c397e0b3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 29 Jul 2015 08:40:17 +0200 Subject: Fix _log method --- module/plugins/internal/MultiCrypter.py | 29 +++++++++++++++++++++++++++++ module/plugins/internal/MultiHoster.py | 15 ++++++++++++++- module/plugins/internal/OCR.py | 7 +++++-- module/plugins/internal/Plugin.py | 4 ++-- 4 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 module/plugins/internal/MultiCrypter.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiCrypter.py b/module/plugins/internal/MultiCrypter.py new file mode 100644 index 000000000..5771e7e96 --- /dev/null +++ b/module/plugins/internal/MultiCrypter.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleCrypter import SimpleCrypter + + +class MultiCrypter(SimpleCrypter): + __name__ = "MultiCrypter" + __type__ = "hoster" + __version__ = "0.01" + __status__ = "testing" + + __pattern__ = r'^unmatchable$' + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] + + __description__ = """Multi decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + def init(self): + self.CRYPTER_NAME = self.pyload.pluginManager.crypterPlugins[self.__name__]['name'] + + + def _log(self, level, plugintype, pluginname, messages): + return super(MultiCrypter, self)._log(level, + plugintype, + "%s: %s" % (pluginname, self.CRYPTER_NAME), + messages) diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 06466e286..5dcdfa46b 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.47" + __version__ = "0.48" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -21,10 +21,23 @@ class MultiHoster(SimpleHoster): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + HOSTER_NAME = None + LEECH_HOSTER = False LOGIN_ACCOUNT = True + def init(self): + self.HOSTER_NAME = self.pyload.pluginManager.hosterPlugins[self.__name__]['name'] + + + def _log(self, level, plugintype, pluginname, messages): + return super(MultiHoster, self)._log(level, + plugintype, + "%s: %s" % (pluginname, self.HOSTER_NAME), + messages) + + def setup(self): self.chunk_limit = 1 self.multiDL = bool(self.account) diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 998b3f040..8eb240051 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -21,7 +21,7 @@ from module.utils import save_join as fs_join class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" - __version__ = "0.15" + __version__ = "0.16" __status__ = "testing" __description__ = """OCR base plugin""" @@ -45,7 +45,10 @@ class OCR(Plugin): def _log(self, level, plugintype, pluginname, messages): - return self.plugin._log(level, plugintype, pluginname, (self.__name__,) + messages) + return self.plugin._log(level, + plugintype, + "%s: %s" % (self.plugin.__name__, self.__name__), + messages) def load_image(self, image): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 638a1ee1f..84dd5bdd4 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -137,7 +137,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.16" + __version__ = "0.17" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -167,7 +167,7 @@ class Plugin(object): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in messages if a)) - log("%(plugintype)s %(pluginname)s%(id)s : %(msg)s" + log("%(plugintype)s %(pluginname)s%(id)s: %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", -- cgit v1.2.3 From d06daeb274da6d7886ab96ba639e18cebb67cc6f Mon Sep 17 00:00:00 2001 From: TodsDeath Date: Wed, 29 Jul 2015 10:44:18 +0200 Subject: Update Account.py the needed premium key is not directly in info, but in the sub-directory 'data': {'data': {'login': '***', 'maxtraffic': None, 'options': {'limitdl': ['0']}, 'password': '**********', 'premium': True, 'timestamp': 1438159062.570853, 'trafficleft': -1, 'type': 'RealdebridCom', 'valid': True, 'validuntil': 1450976583.0}, 'login': {'password': '**********', 'timestamp': 1438159044.071365, 'valid': True}} --- module/plugins/internal/Account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 3084f02c7..55ef6c5f9 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -232,7 +232,7 @@ class Account(Plugin): return False info = self.get_info(user, reload) - return info['premium'] if info and 'premium' in info else False + return info['data']['premium'] if info and 'data' in info and 'premium' in info['data'] else False def _parse_info(self, user): -- cgit v1.2.3 From fd7b164840d2bcf1af86e7eece46ada9e91e2f41 Mon Sep 17 00:00:00 2001 From: TodsDeath Date: Wed, 29 Jul 2015 11:10:04 +0200 Subject: Update Account.py better readability than conditional expression --- module/plugins/internal/Account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 55ef6c5f9..d6e4ce8ab 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -232,7 +232,7 @@ class Account(Plugin): return False info = self.get_info(user, reload) - return info['data']['premium'] if info and 'data' in info and 'premium' in info['data'] else False + return info.get('data', {}).get('premium', False) def _parse_info(self, user): -- cgit v1.2.3 From e91d1b7b50c97fdbec0f99c071d3fd19b25e57bc Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 29 Jul 2015 12:23:29 +0200 Subject: [Plugin] Fix set_cookies --- module/plugins/internal/Plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 84dd5bdd4..0049826da 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -80,7 +80,7 @@ def set_cookies(cj, cookies): for cookie in cookies: if isinstance(cookie, tuple) and len(cookie) == 3: domain, name, value = cookie - cj.setCookie(domain, name, value) + cj.setCookie(domain, name, encode(value)) #@TODO: Remove `encode` in 0.4.10 def parse_html_tag_attr_value(attr_name, tag): @@ -137,7 +137,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.17" + __version__ = "0.18" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 66d5610dff9b7a5e56075e195f88dc0f60441e72 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 29 Jul 2015 12:37:00 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1594 --- module/plugins/internal/Extractor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 907113a7e..d511b7602 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.29" + __version__ = "0.30" __status__ = "testing" __description__ = """Base extractor plugin""" @@ -67,8 +67,8 @@ class Extractor(Plugin): processed = [] for fname, id, fout in files_ids: - if cls.isArchive(fname): - pname = re.sub(cls.re_multipart, "", fname) if cls.isMultipart(fname) else os.path.splitext(fname)[0] + if cls.is_archive(fname): + pname = re.sub(cls.re_multipart, "", fname) if cls.is_multipart(fname) else os.path.splitext(fname)[0] if pname not in processed: processed.append(pname) targets.append((fname, id, fout)) -- cgit v1.2.3 From 064c4bca39b545cdab7e0b45816ed1a260e192a2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 29 Jul 2015 14:05:49 +0200 Subject: Fix http://forum.pyload.org/viewtopic.php?f=12&t=4421 --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 6fe68bb6f..d30c56035 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.10" + __version__ = "0.11" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -420,7 +420,7 @@ class Hoster(Plugin): self.pyfile.size = self.req.size if newname: - newname = urlparse.urlparse(newname).path.split('*=')[-1].split('/')[-1] #@TODO: Remove in 0.4.10 + newname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] #@TODO: Remove in 0.4.10 if disposition and newname is not name: self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) -- cgit v1.2.3 From a7d2fd070b8b407f944e8114afa4b0feaa73ce4d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 29 Jul 2015 14:06:46 +0200 Subject: [MultiHook] Better error handling for plugins_cached --- module/plugins/internal/MultiHook.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index c49a3c89a..b64ff08c0 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -11,7 +11,7 @@ from module.utils import decode, remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.51" + __version__ = "0.52" __status__ = "testing" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), @@ -100,16 +100,16 @@ class MultiHook(Hook): if self.plugins: return self.plugins - for _i in xrange(2): + for _i in xrange(5): try: pluginset = self._plugin_set(self.get_hosters()) break except Exception, e: - self.log_debug(e, "Waiting 1 minute and retry") + self.log_warning(e, _("Waiting 1 minute and retry")) time.sleep(60) else: - self.log_warning(_("Fallback to default reload interval due plugin parse error")) + self.log_error(_("No hoster list retrieved")) self.interval = self.MIN_RELOAD_INTERVAL return list() -- cgit v1.2.3 From dc27f688159b11188f5db4914ff8324e0d633dec Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 29 Jul 2015 17:38:15 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1602 --- module/plugins/internal/Plugin.py | 3 ++- module/plugins/internal/SimpleHoster.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 0049826da..76056c66d 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -2,6 +2,7 @@ from __future__ import with_statement +import datetime import inspect import os import re @@ -137,7 +138,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.18" + __version__ = "0.19" __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 7a220415c..84dbfd011 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -2,7 +2,6 @@ from __future__ import with_statement -import datetime import mimetypes import os import re -- cgit v1.2.3 From 91e0803c1f47444072ca7381d789a8e98160ae78 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 29 Jul 2015 21:11:29 +0200 Subject: Still improving _log methods --- module/plugins/internal/Captcha.py | 6 +++--- module/plugins/internal/Extractor.py | 6 +++--- module/plugins/internal/MultiCrypter.py | 6 +++--- module/plugins/internal/MultiHoster.py | 6 +++--- module/plugins/internal/OCR.py | 6 +++--- module/plugins/internal/Plugin.py | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 92bcf9f84..f126ca85e 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -12,7 +12,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.37" + __version__ = "0.38" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -39,8 +39,8 @@ class Captcha(Plugin): def _log(self, level, plugintype, pluginname, messages): return self.plugin._log(level, plugintype, - "%s: %s" % (self.plugin.__name__, self.__name__), - messages) + self.plugin.__name__, + (self.__name__,) + messages) def recognize(self, image): diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index d511b7602..c16b1a846 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.30" + __version__ = "0.31" __status__ = "testing" __description__ = """Base extractor plugin""" @@ -116,8 +116,8 @@ class Extractor(Plugin): def _log(self, level, plugintype, pluginname, messages): return self.plugin._log(level, plugintype, - "%s: %s" % (self.plugin.__name__, self.__name__), - messages) + self.plugin.__name__, + (self.__name__,) + messages) def check(self): diff --git a/module/plugins/internal/MultiCrypter.py b/module/plugins/internal/MultiCrypter.py index 5771e7e96..ca7b03941 100644 --- a/module/plugins/internal/MultiCrypter.py +++ b/module/plugins/internal/MultiCrypter.py @@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter class MultiCrypter(SimpleCrypter): __name__ = "MultiCrypter" __type__ = "hoster" - __version__ = "0.01" + __version__ = "0.02" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -25,5 +25,5 @@ class MultiCrypter(SimpleCrypter): def _log(self, level, plugintype, pluginname, messages): return super(MultiCrypter, self)._log(level, plugintype, - "%s: %s" % (pluginname, self.CRYPTER_NAME), - messages) + pluginname, + (self.CRYPTER_NAME,) + messages) diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 5dcdfa46b..cdb668b82 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.48" + __version__ = "0.49" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -34,8 +34,8 @@ class MultiHoster(SimpleHoster): def _log(self, level, plugintype, pluginname, messages): return super(MultiHoster, self)._log(level, plugintype, - "%s: %s" % (pluginname, self.HOSTER_NAME), - messages) + pluginname, + (self.HOSTER_NAME,) + messages) def setup(self): diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 8eb240051..805755f39 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -21,7 +21,7 @@ from module.utils import save_join as fs_join class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" - __version__ = "0.16" + __version__ = "0.17" __status__ = "testing" __description__ = """OCR base plugin""" @@ -47,8 +47,8 @@ class OCR(Plugin): def _log(self, level, plugintype, pluginname, messages): return self.plugin._log(level, plugintype, - "%s: %s" % (self.plugin.__name__, self.__name__), - messages) + self.plugin.__name__, + (self.__name__,) + messages) def load_image(self, image): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 76056c66d..59528a441 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -168,7 +168,7 @@ class Plugin(object): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in messages if a)) - log("%(plugintype)s %(pluginname)s%(id)s: %(msg)s" + log("%(plugintype)s %(pluginname)s%(id)s : %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", -- cgit v1.2.3 From 88f98ab3bd6e091c1b6ca6ac9fddc37b2e29694c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 30 Jul 2015 14:37:44 +0200 Subject: Fix get_code call in captcha hooks --- module/plugins/internal/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 59528a441..76056c66d 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -168,7 +168,7 @@ class Plugin(object): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in messages if a)) - log("%(plugintype)s %(pluginname)s%(id)s : %(msg)s" + log("%(plugintype)s %(pluginname)s%(id)s: %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", -- cgit v1.2.3 From 503906b69e7ffcad398a2529c64b806cf7c5b558 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 30 Jul 2015 21:04:38 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1624 --- module/plugins/internal/Hoster.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index d30c56035..1b96f1201 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.11" + __version__ = "0.12" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -385,11 +385,7 @@ class Hoster(Plugin): self.pyfile.setStatus("downloading") - if disposition: - self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name - download_folder = self.pyload.config.get("general", "download_folder") - location = fs_join(download_folder, self.pyfile.package().folder) if not exists(location): @@ -402,7 +398,7 @@ class Hoster(Plugin): os.chown(location, uid, gid) except Exception, e: - self.fail(str(e)) #@TODO: Remove `str` in 0.4.10 + self.fail(e) #: Convert back to unicode location = fs_decode(location) -- cgit v1.2.3 From 092112b44af84c7a59b1fa2cfff5c5875e778a8f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 02:21:35 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1625 --- module/plugins/internal/Crypter.py | 8 ++--- module/plugins/internal/Hoster.py | 66 +++++++++++++++----------------------- module/plugins/internal/Plugin.py | 33 ++++++++++++++++++- 3 files changed, 62 insertions(+), 45 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 13d82c2ea..0fdec559d 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -84,19 +84,19 @@ class Crypter(Hoster): self.pyload.api.setPackageData(pid, {'password': package_password}) #: Workaround to do not break API addPackage method - setFolder = lambda x: self.pyload.api.setPackageData(pid, {'folder': x or ""}) + set_folder = lambda x: self.pyload.api.setPackageData(pid, {'folder': x or ""}) if use_subfolder: if not subfolder_per_package: - setFolder(package_folder) + set_folder(package_folder) self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) elif not folder_per_package or name is not folder: if not folder: folder = urlparse.urlparse(name).path.split("/")[-1] - setFolder(safe_filename(folder)) + set_folder(safe_filename(folder)) self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) elif folder_per_package: - setFolder(None) + set_folder(None) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 1b96f1201..c54adf4b3 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.12" + __version__ = "0.13" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -179,7 +179,7 @@ class Hoster(Plugin): self.pyfile.setStatus("starting") - self.log_debug("PROCESS URL " + self.pyfile.url) + self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) return self.process(self.pyfile) @@ -190,12 +190,6 @@ class Hoster(Plugin): raise NotImplementedError - def get_chunk_count(self): - if self.chunk_limit <= 0: - return self.pyload.config.get("download", "chunks") - return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) - - def set_reconnect(self, reconnect): reconnect = bool(reconnect) @@ -385,29 +379,27 @@ class Hoster(Plugin): self.pyfile.setStatus("downloading") - download_folder = self.pyload.config.get("general", "download_folder") - location = fs_join(download_folder, self.pyfile.package().folder) + download_folder = self.pyload.config.get("general", "download_folder") + download_location = fs_join(download_folder, self.pyfile.package().folder) - if not exists(location): + if not exists(download_location): try: - os.makedirs(location, int(self.pyload.config.get("permission", "folder"), 8)) - - if self.pyload.config.get("permission", "change_dl") and os.name != "nt": - uid = pwd.getpwnam(self.pyload.config.get("permission", "user"))[2] - gid = grp.getgrnam(self.pyload.config.get("permission", "group"))[2] - os.chown(location, uid, gid) + os.makedirs(download_location) except Exception, e: self.fail(e) - #: Convert back to unicode - location = fs_decode(location) - name = safe_filename(self.pyfile.name) + self.set_permissions(download_location) + location = fs_decode(download_location) + name = safe_filename(urlparse.urlparse(self.pyfile.name).path.split('/')[-1] or self.pyfile.name) filename = os.path.join(location, name) self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename) + if self.pyfile.abort: + self.abort() + try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.get_chunk_count(), resume=self.resume_download, @@ -415,32 +407,26 @@ class Hoster(Plugin): finally: self.pyfile.size = self.req.size - if newname: - newname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] #@TODO: Remove in 0.4.10 - - if disposition and newname is not name: - self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) - self.pyfile.name = newname - filename = os.path.join(location, newname) + #@TODO: Recheck in 0.4.10 + if disposition: + finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] - fs_filename = fs_encode(filename) + if finalname != newname != name: + try: + os.rename(fs_join(location, newname), fs_join(location, finalname)) - if self.pyload.config.get("permission", "change_file"): - try: - os.chmod(fs_filename, int(self.pyload.config.get("permission", "file"), 8)) - except Exception, e: - self.log_warning(_("Setting file mode failed"), e) + except OSError, e: + self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e) + finalname = newname - if self.pyload.config.get("permission", "change_dl") and os.name != "nt": - try: - uid = pwd.getpwnam(self.pyload.config.get("permission", "user"))[2] - gid = grp.getgrnam(self.pyload.config.get("permission", "group"))[2] - os.chown(fs_filename, uid, gid) + self.log_info(_("`%s` saved as `%s`") % (name, finalname)) + self.pyfile.name = finalname + filename = os.path.join(location, finalname) - except Exception, e: - self.log_warning(_("Setting User and Group failed"), e) + self.set_permissions(fs_encode(filename)) self.last_download = filename + return self.last_download diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 76056c66d..70494561c 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -138,7 +138,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.19" + __version__ = "0.20" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -196,6 +196,37 @@ class Plugin(object): return self._log("critical", self.__type__, self.__name__, args) + def set_permissions(self, path): + if not os.path.exists(path): + return + + try: + if self.pyload.config.get("permission", "change_file"): + if os.path.isfile(path): + os.chmod(path, int(self.pyload.config.get("permission", "file"), 8)) + + elif os.path.isdir(path): + os.chmod(path, int(self.pyload.config.get("permission", "folder"), 8)) + + except OSError, e: + self.log_warning(_("Setting path mode failed"), e) + + try: + if os.name != "nt" and self.pyload.config.get("permission", "change_dl"): + uid = pwd.getpwnam(self.pyload.config.get("permission", "user"))[2] + gid = grp.getgrnam(self.pyload.config.get("permission", "group"))[2] + os.chown(path, uid, gid) + + except OSError, e: + self.log_warning(_("Setting owner and group failed"), e) + + + def get_chunk_count(self): + if self.chunk_limit <= 0: + return self.pyload.config.get("download", "chunks") + return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) + + def set_config(self, option, value): """ Set config value for current plugin -- cgit v1.2.3 From faf4627b29619730a0c4e32d9c36932d2ba944c9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 02:23:39 +0200 Subject: [Addon] Fix unload method not called on exit --- module/plugins/internal/Addon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 3c617f730..ef2932b11 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -25,7 +25,7 @@ def threaded(fn): class Addon(Plugin): __name__ = "Addon" __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 - __version__ = "0.01" + __version__ = "0.02" __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] @@ -157,6 +157,7 @@ class Addon(Plugin): #: Deprecated method, use `exit` instead (Remove in 0.4.10) def coreExiting(self, *args, **kwargs): + self.unload(*args, **kwargs) #@TODO: Fix in 0.4.10 return self.exit(*args, **kwargs) -- cgit v1.2.3 From 3f2def49161ad864ae900b2902767c29cab6e3b3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 03:46:01 +0200 Subject: [Account] Fix ********** bug --- module/plugins/internal/Account.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index d6e4ce8ab..a33cdffac 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import copy import random import time import threading @@ -12,7 +13,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.10" + __version__ = "0.11" __status__ = "testing" __description__ = """Base account plugin""" @@ -219,7 +220,7 @@ class Account(Plugin): self.log_debug("Reached data timeout for %s" % user) self.schedule_refresh(user) - safe_info = info.copy() + safe_info = copy.deepcopy(info) safe_info['login']['password'] = "**********" safe_info['data']['password'] = "**********" #@TODO: Remove in 0.4.10 self.log_debug("Account info for user `%s`: %s" % (user, safe_info)) -- cgit v1.2.3 From 104dff1d1757dc6784ca06090c3d8b06b8f1d03e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 04:03:44 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1602 (2) --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 84dbfd011..f015ce9ff 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -23,7 +23,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.75" + __version__ = "1.76" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -315,7 +315,7 @@ class SimpleHoster(Hoster): elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, file_size=self.info['size'] if 'size' in self.info else 0, - size_tolerance=10240, + size_tolerance=1048576, delete=True): self.error(_("Empty file")) -- cgit v1.2.3 From dc47212c8e801d380e4db4bd1bec0bf183de9de1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 05:48:44 +0200 Subject: [Captcha] Fix _decrypt method --- module/plugins/internal/Captcha.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index f126ca85e..14308fbf5 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -12,7 +12,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.38" + __version__ = "0.39" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -57,7 +57,7 @@ class Captcha(Plugin): #@TODO: Definitely choose a better name for this method! - def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=None, timeout=120): + def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=False, timeout=120): """ Loads a captcha and decrypts it with ocr, plugin, user input @@ -73,31 +73,28 @@ class Captcha(Plugin): :return: result of decrypting """ + result = "" time_ref = ("%.2f" % time.time())[-6:].replace(".", "") with open(os.path.join("tmp", "captcha_image_%s_%s.%s" % (self.plugin.__name__, time_ref, input_type)), "wb") as tmp_img: tmp_img.write(raw) - if ocr is not False: + if ocr: if isinstance(ocr, basestring): OCR = self.pyload.pluginManager.loadClass("captcha", ocr) #: Rename `captcha` to `ocr` in 0.4.10 - - if self.plugin.pyfile.abort: - self.plugin.abort() - result = OCR(self.plugin).recognize(tmp_img.name) - else: result = self.recognize(tmp_img.name) - else: + if not result: captchaManager = self.pyload.captchaManager try: self.task = captchaManager.newTask(raw, input_type, tmp_img.name, output_type) + captchaManager.handleCaptcha(self.task) - self.task.setWaiting(max(timeout, 50)) #@TODO: Move to `CaptchaManager` in 0.4.10 + self.task.setWaiting(max(timeout, 50)) #@TODO: Move to `CaptchaManager` in 0.4.10 while self.task.isWaiting(): if self.plugin.pyfile.abort: self.plugin.abort() @@ -114,7 +111,6 @@ class Captcha(Plugin): self.plugin.retry(reason=_("No captcha result obtained in appropiate time")) result = self.task.result - self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? if not self.pyload.debug: try: @@ -124,6 +120,8 @@ class Captcha(Plugin): self.log_warning(_("Error removing: %s") % tmp_img.name, e) traceback.print_exc() + self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? + return result -- cgit v1.2.3 From e3799bcb0731e8cdf4d3f86ae6ee67e9cc5e4d80 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 06:35:21 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1601 --- module/plugins/internal/Plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 70494561c..159668c28 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -138,7 +138,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.20" + __version__ = "0.21" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -292,7 +292,7 @@ class Plugin(object): raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=True, req=None): + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=True, multipart=False, req=None): """ Load content at url and returns it @@ -323,7 +323,7 @@ class Plugin(object): else: req = self.pyload.requestFactory.getRequest(self.__name__) - res = req.load(url, get, post, ref, cookies, just_header, isinstance(post, dict), decode is True) #@TODO: Fix network multipart in 0.4.10 + res = req.load(url, get, post, ref, cookies, just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 if decode: #@TODO: Move to network in 0.4.10 res = html_unescape(res) -- cgit v1.2.3 From 8a8323f3fd86985a5ca876e8d54f1699e75b24a8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 07:37:10 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1628 --- module/plugins/internal/Hoster.py | 4 ---- module/plugins/internal/Plugin.py | 6 +++++- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index c54adf4b3..4d69720ac 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -9,10 +9,6 @@ import time import traceback import urlparse -if os.name != "nt": - import grp - import pwd - from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, chunks, encode, exists, fixurl as _fixurl, replace_patterns, diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 159668c28..fa0ccaadd 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -8,6 +8,10 @@ import os import re import urllib +if os.name != "nt": + import grp + import pwd + from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 from module.utils import fs_encode, fs_decode, html_unescape, save_join as fs_join @@ -138,7 +142,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.21" + __version__ = "0.22" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From cc3c575a68fd4b585f0e4af586e027fddb0fe605 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 10:38:51 +0200 Subject: Fix and improve account logic --- module/plugins/internal/Account.py | 71 ++++++++++++------------ module/plugins/internal/Hoster.py | 92 +++++++++++++++++--------------- module/plugins/internal/MultiHook.py | 5 +- module/plugins/internal/SimpleCrypter.py | 12 ++--- module/plugins/internal/SimpleHoster.py | 2 +- 5 files changed, 89 insertions(+), 93 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index a33cdffac..a402c4ad7 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -13,7 +13,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.11" + __version__ = "0.12" __status__ = "testing" __description__ = """Base account plugin""" @@ -23,14 +23,14 @@ class Account(Plugin): LOGIN_TIMEOUT = 10 * 60 #: After that time (in minutes) pyload will relogin the account - INFO_THRESHOLD = 10 * 60 #: After that time (in minutes) account data will be reloaded + INFO_THRESHOLD = 30 * 60 #: After that time (in minutes) account data will be reloaded def __init__(self, manager, accounts): - self.pyload = manager.core - self.info = {} #: Provide information in dict here - self.lock = threading.RLock() - + self.pyload = manager.core + self.info = {} #: Provide information in dict here + self.lock = threading.RLock() + self.req = None self.accounts = accounts #@TODO: Remove in 0.4.10 self.init() @@ -73,9 +73,7 @@ class Account(Plugin): self.accounts[user]['valid'] = True #@TODO: Remove in 0.4.10 finally: - if self.req: - self.req.close() - del self.req + self.del_request() return res @@ -86,7 +84,7 @@ class Account(Plugin): req = self.get_request(user) if req: req.clearCookies() - req.close() + self.del_request() if user in self.info: self.info[user]['login'].clear() @@ -110,8 +108,7 @@ class Account(Plugin): 'premium' : None, 'validuntil' : None, 'trafficleft': None, - 'maxtraffic' : None, - 'timestamp' : 0}} + 'maxtraffic' : None}} #@TODO: Remove in 0.4.10 self.accounts[user] = self.info[user]['data'] @@ -133,7 +130,7 @@ class Account(Plugin): """ Updates account and return true if anything changed """ - if not password or not options: + if not (password or options): return if user not in self.info: @@ -173,6 +170,7 @@ class Account(Plugin): return self.remove(*args, **kwargs) + #@NOTE: Remove in 0.4.10? def get_data(self, user, reload=False): if not user: return @@ -213,18 +211,18 @@ class Account(Plugin): self.log_info(_("Parsing account info for user `%s`...") % user) info = self._parse_info(user) + safe_info = copy.deepcopy(info) + safe_info['login']['password'] = "**********" + safe_info['data']['password'] = "**********" #@TODO: Remove in 0.4.10 + self.log_debug("Account info for user `%s`: %s" % (user, safe_info)) + else: info = self.info[user] - if self.INFO_THRESHOLD > 0 and info['data']['timestamp'] + self.INFO_THRESHOLD * 60 < time.time(): + if self.INFO_THRESHOLD > 0 and info['login']['timestamp'] + self.INFO_THRESHOLD < time.time(): self.log_debug("Reached data timeout for %s" % user) self.schedule_refresh(user) - safe_info = copy.deepcopy(info) - safe_info['login']['password'] = "**********" - safe_info['data']['password'] = "**********" #@TODO: Remove in 0.4.10 - self.log_debug("Account info for user `%s`: %s" % (user, safe_info)) - return info @@ -232,32 +230,31 @@ class Account(Plugin): if not user: return False - info = self.get_info(user, reload) - return info.get('data', {}).get('premium', False) + info = self.get_info(user) + return info['data']['premium'] def _parse_info(self, user): info = self.info[user] - try: - info['data']['timestamp'] = time.time() #: Set timestamp for login + if not info['login']['valid']: + return info + try: self.req = self.get_request(user) extra_info = self.parse_info(user, info['login']['password'], info, self.req) if extra_info and isinstance(extra_info, dict): info['data'].update(extra_info) - except Exception, e: + except (Fail, Exception), e: self.log_warning(_("Error loading info for user `%s`") % user, e) if self.pyload.debug: traceback.print_exc() finally: - if self.req: - self.req.close() - del self.req + self.del_request() self.info[user].update(info) return info @@ -291,6 +288,13 @@ class Account(Plugin): return self.pyload.requestFactory.getRequest(self.__name__, user) + def del_request(self): + try: + self.req.close() + finally: + self.req = None + + def get_cookies(self, user=None): if not user: user, info = self.select() @@ -350,10 +354,6 @@ class Account(Plugin): reverse=True)[0] - def can_use(self): - return self.select() != (None, None) - - def parse_traffic(self, value, unit=None): #: Return kilobytes if not unit and not isinstance(value, basestring): unit = "KB" @@ -392,20 +392,19 @@ class Account(Plugin): #: Deprecated method, use `schedule_refresh` instead (Remove in 0.4.10) def scheduleRefresh(self, *args, **kwargs): if 'force' in kwargs: - kwargs['reload'] = kwargs['force'] - kwargs.pop('force', None) + kwargs.pop('force', None) #@TODO: Recheck in 0.4.10 return self.schedule_refresh(*args, **kwargs) @lock - def is_logged(self, user): + def is_logged(self, user, relogin=False): """ Checks if user is still logged in """ if user in self.info: - if self.LOGIN_TIMEOUT > 0 and self.info[user]['login']['timestamp'] + self.LOGIN_TIMEOUT * 60 < time.time(): + if self.LOGIN_TIMEOUT > 0 and self.info[user]['login']['timestamp'] + self.LOGIN_TIMEOUT < time.time(): self.log_debug("Reached login timeout for %s" % user) - return self.relogin(user) + return self.relogin(user) if relogin else False else: return True else: diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 4d69720ac..1aea3804d 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.13" + __version__ = "0.14" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -68,43 +68,14 @@ class Hoster(Plugin): self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10 self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 - #: Chunk limit - self.chunk_limit = 1 - self.resume_download = False - #: time.time() + wait in seconds self.wait_until = 0 - self.waiting = False - - #: Captcha reader instance - self.ocr = None + self.waiting = False #: Account handler instance, see :py:class:`Account` - self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - - #: Premium status - self.premium = False - - #: username/login - self.user = None - - if self.account and not self.account.can_use(): - self.account = None - - if self.account: - self.user, data = self.account.select() - - #: Browser instance, see `network.Browser` - self.req = self.account.get_request(self.user) - self.chunk_limit = -1 #: Chunk limit, -1 for unlimited - - #: Enables resume (will be ignored if server dont accept chunks) - self.resume_download = True - - #: Premium status - self.premium = self.account.is_premium(self.user) - else: - self.req = self.pyload.requestFactory.getRequest(self.__name__) + self.account = None + self.user = None + self.req = None #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile @@ -129,6 +100,7 @@ class Hoster(Plugin): #: Dict of the amount of retries already made self.retries = {} + self._setup() self.init() @@ -136,9 +108,9 @@ class Hoster(Plugin): def get_info(cls, url="", html=""): url = _fixurl(url) url_p = urlparse.urlparse(url) - return {'name' : (url_p.path.split('/')[-1] - or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] - or url_p.netloc.split('.', 1)[0]), + return {'name' : (url_p.path.split('/')[-1] or + url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or + url_p.netloc.split('.', 1)[0]), 'size' : 0, 'status': 3 if url else 8, 'url' : url} @@ -158,6 +130,41 @@ class Hoster(Plugin): pass + def _setup(self): + if self.account: + self.chunk_limit = -1 #: -1 for unlimited + self.resume_download = True + self.premium = self.account.is_premium(self.user) + else: + self.chunk_limit = 1 + self.resume_download = False + self.premium = False + + + def load_account(self): + oldaccount = self.account + + if not self.account: + self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) + + if self.account: + if self.user: + self.account = self.account.is_logged(self.user, relogin=True) + else: + self.user, data = self.account.select() + if not self.user: + self.account = False + + if self.account == oldaccount: + self.req.clearCookies() + + elif self.account: + self.req = self.account.get_request(self.user) #: Browser instance, see `network.Browser` + + else: + self.req = self.pyload.requestFactory.getRequest(self.__name__) + + def preprocessing(self, thread): """ Handles important things to do before starting @@ -166,16 +173,13 @@ class Hoster(Plugin): self.req.renewHTTPRequest() - if self.account: - self.account.is_logged(self.user) - else: - self.req.clearCookies() - + self.load_account() + self._setup() self.setup() self.pyfile.setStatus("starting") - self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) + return self.process(self.pyfile) @@ -426,7 +430,7 @@ class Hoster(Plugin): return self.last_download - def check_download(self, rules, delete=False, file_size=0, size_tolerance=1024, read_size=100000): + def check_download(self, rules, delete=False, file_size=0, size_tolerance=1024, read_size=1048576): """ Checks the content of the last downloaded file, re match is saved to `lastCheck` diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index b64ff08c0..ea515d69a 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -11,7 +11,7 @@ from module.utils import decode, remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.52" + __version__ = "0.53" __status__ = "testing" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), @@ -84,8 +84,7 @@ class MultiHook(Hook): def load_account(self): self.account = self.pyload.accountManager.getAccountPlugin(self.pluginname) - if self.account and not self.account.can_use(): - self.account = None + self.load_account() if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: self.log_warning(_("Hook plugin will be deactivated due missing account reference")) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 33ebd8cac..20a03df9f 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.57" + __version__ = "0.58" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -59,14 +59,8 @@ class SimpleCrypter(Crypter, SimpleHoster): #@TODO: Remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") - account = self.pyload.accountManager.getAccountPlugin(account_name) - - if account and account.can_use(): - self.user, data = account.select() - self.req = account.get_request(self.user) - self.premium = account.is_premium(self.user) - - self.account = account + self.account = self.pyload.accountManager.getAccountPlugin(account_name) + self.load_account() def handle_direct(self, pyfile): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index f015ce9ff..a03e301b3 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -338,7 +338,7 @@ class SimpleHoster(Hoster): if self.CHECK_FILE: self.log_debug("Using custom check rules...") with open(fs_encode(self.last_download), "rb") as f: - self.html = f.read(50000) #@TODO: Recheck in 0.4.10 + self.html = f.read(1048576) #@TODO: Recheck in 0.4.10 self.check_errors() self.log_info(_("No errors found")) -- cgit v1.2.3 From 6108c417fa3eb049c839310d0894b11511d3f986 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 10:57:49 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1632 --- module/plugins/internal/Plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index fa0ccaadd..d3cc4bbc4 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -142,7 +142,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.22" + __version__ = "0.23" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -253,7 +253,7 @@ class Plugin(object): return self.pyload.config.getPlugin(plugin or self.__name__, option) except KeyError: - self.log_warning(_("Config option `%s` not found, use default `%s`") % (option, default or None)) + self.log_debug("Config option `%s` not found, use default `%s`" % (option, default or None)) #@TODO: Restore to `log_warning` in 0.4.10 return default -- cgit v1.2.3 From a9e1aacd52e53b734c1e18299141900c1c8c0c32 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 1 Aug 2015 02:36:15 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1636 --- module/plugins/internal/Hoster.py | 2 +- module/plugins/internal/MultiHook.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 1aea3804d..92afd3672 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -332,7 +332,7 @@ class Hoster(Plugin): return self.premium = False - self.account = None + self.account = False self.req = self.pyload.requestFactory.getRequest(self.__name__) raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index ea515d69a..42a1985b5 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -11,7 +11,7 @@ from module.utils import decode, remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.53" + __version__ = "0.54" __status__ = "testing" __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), @@ -84,7 +84,8 @@ class MultiHook(Hook): def load_account(self): self.account = self.pyload.accountManager.getAccountPlugin(self.pluginname) - self.load_account() + if self.account and not self.account.select()[0]: + self.account = False if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: self.log_warning(_("Hook plugin will be deactivated due missing account reference")) -- cgit v1.2.3 From b612fbda4fa6ed8cb0b4e2e3015e8127a20aaea2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 1 Aug 2015 22:34:51 +0200 Subject: Fix load_account routine --- module/plugins/internal/Hoster.py | 35 +++++++++++++------------------- module/plugins/internal/SimpleCrypter.py | 25 ++++++++++++++++++----- 2 files changed, 34 insertions(+), 26 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 92afd3672..ea54d9589 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.14" + __version__ = "0.15" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -142,27 +142,22 @@ class Hoster(Plugin): def load_account(self): - oldaccount = self.account + if self.req: + self.req.close() if not self.account: self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) if self.account: - if self.user: - self.account = self.account.is_logged(self.user, relogin=True) - else: + if not self.user: self.user, data = self.account.select() - if not self.user: - self.account = False - if self.account == oldaccount: - self.req.clearCookies() + if not self.user or not self.account.is_logged(self.user, relogin=True): + self.account = False - elif self.account: - self.req = self.account.get_request(self.user) #: Browser instance, see `network.Browser` - - else: - self.req = self.pyload.requestFactory.getRequest(self.__name__) + #: Browser instance, see `network.Browser` + self.req = self.pyload.requestFactory.getRequest(self.__name__, + self.user if self.account else None) def preprocessing(self, thread): @@ -171,8 +166,6 @@ class Hoster(Plugin): """ self.thread = thread - self.req.renewHTTPRequest() - self.load_account() self._setup() self.setup() @@ -516,10 +509,9 @@ class Hoster(Plugin): header = self.load(url, just_header=True) except Exception: #: Bad bad bad... rewrite this part in 0.4.10 - req = pyreq.getHTTPRequest() - res = self.load(url, just_header=True) - - req.close() + res = self.load(url, + just_header=True, + req=self.pyload.requestFactory.getRequest()) header = {'code': req.code} for line in res.splitlines(): @@ -657,7 +649,8 @@ class Hoster(Plugin): del self.pyfile if hasattr(self, "req"): - self.req.close() + if self.req: + self.req.close() del self.req if hasattr(self, "thread"): diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 20a03df9f..084bc133c 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.58" + __version__ = "0.59" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -57,10 +57,25 @@ class SimpleCrypter(Crypter, SimpleHoster): #@TODO: Remove in 0.4.10 - def init(self): - account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") - self.account = self.pyload.accountManager.getAccountPlugin(account_name) - self.load_account() + def load_account(self): + accountname = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") + + if self.req: + self.req.close() + + if not self.account: + self.account = self.pyload.accountManager.getAccountPlugin(accountname) + + if self.account: + if not self.user: + self.user, data = self.account.select() + + if not self.user or not self.account.is_logged(self.user, relogin=True): + self.account = False + + #: Browser instance, see `network.Browser` + self.req = self.pyload.requestFactory.getRequest(accountname, + self.user if self.account else None) def handle_direct(self, pyfile): -- cgit v1.2.3 From b0d9532f675854a6a32d22f7b6b7b4cd32683443 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 07:10:51 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1640 --- module/plugins/internal/Addon.py | 7 +++-- module/plugins/internal/Hoster.py | 64 ++++++++++++++++----------------------- module/plugins/internal/Plugin.py | 22 +++++++++++--- 3 files changed, 47 insertions(+), 46 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index ef2932b11..649eccbc1 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -25,7 +25,7 @@ def threaded(fn): class Addon(Plugin): __name__ = "Addon" __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 - __version__ = "0.02" + __version__ = "0.03" __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] @@ -166,8 +166,9 @@ class Addon(Plugin): #: Deprecated method, use `download_preparing` instead (Remove in 0.4.10) - def downloadPreparing(self, *args, **kwargs): - return self.download_preparing(*args, **kwargs) + def downloadPreparing(self, pyfile): + if pyfile.plugin.req is not None: #@TODO: Remove in 0.4.10 + return self.download_preparing(pyfile) def download_finished(self, pyfile): diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index ea54d9589..d707154bb 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.15" + __version__ = "0.16" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -75,7 +75,7 @@ class Hoster(Plugin): #: Account handler instance, see :py:class:`Account` self.account = None self.user = None - self.req = None + self.req = None #: Browser instance, see `network.Browser` #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile @@ -98,7 +98,8 @@ class Hoster(Plugin): self.html = None #: Dict of the amount of retries already made - self.retries = {} + self.retries = {} + self.retry_free = False #@TODO: Recheck in 0.4.10 self._setup() self.init() @@ -132,10 +133,12 @@ class Hoster(Plugin): def _setup(self): if self.account: + self.req = self.pyload.requestFactory.getRequest(self.__name__, self.user) self.chunk_limit = -1 #: -1 for unlimited self.resume_download = True self.premium = self.account.is_premium(self.user) else: + self.req = self.pyload.requestFactory.getRequest(self.__name__) self.chunk_limit = 1 self.resume_download = False self.premium = False @@ -150,15 +153,11 @@ class Hoster(Plugin): if self.account: if not self.user: - self.user, data = self.account.select() + self.user = self.account.select()[0] - if not self.user or not self.account.is_logged(self.user, relogin=True): + if not self.user or not self.account.is_logged(self.user, True): self.account = False - #: Browser instance, see `network.Browser` - self.req = self.pyload.requestFactory.getRequest(self.__name__, - self.user if self.account else None) - def preprocessing(self, thread): """ @@ -166,10 +165,20 @@ class Hoster(Plugin): """ self.thread = thread - self.load_account() + if self.retry_free: + self.account = False + else: + self.load_account() #@TODO: Move to PluginThread in 0.4.10 + self.retry_free = False + self._setup() self.setup() + self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 + + if self.pyfile.abort: + self.abort() + self.pyfile.setStatus("starting") self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) @@ -316,17 +325,15 @@ class Hoster(Plugin): raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 - def restart(self, reason=None, reset=False): + def restart(self, reason=None, nopremium=False): if not reason: - reason = _("Fallback to free download") if reset else _("Restart") - - if reset: - if not self.premium: - return + reason = _("Fallback to free download") if nopremium else _("Restart") - self.premium = False - self.account = False - self.req = self.pyload.requestFactory.getRequest(self.__name__) + if nopremium: + if self.premium: + self.retry_free = True + else: + self.fail(reason, _("Download was already free")) raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 @@ -639,22 +646,3 @@ class Hoster(Plugin): self.skip(pyfile[0]) self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name) - - - def clean(self): - """ - Clean everything and remove references - """ - if hasattr(self, "pyfile"): - del self.pyfile - - if hasattr(self, "req"): - if self.req: - self.req.close() - del self.req - - if hasattr(self, "thread"): - del self.thread - - if hasattr(self, "html"): - del self.html diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index d3cc4bbc4..3d53b7d2d 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -142,7 +142,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.23" + __version__ = "0.24" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -159,6 +159,7 @@ class Plugin(object): def __init__(self, core): self.pyload = core self.info = {} #: Provide information in dict here + self.req = None self.init() @@ -322,10 +323,7 @@ class Plugin(object): *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) if req is None: - if hasattr(self, "req"): - req = self.req - else: - req = self.pyload.requestFactory.getRequest(self.__name__) + req = self.req or self.pyload.requestFactory.getRequest(self.__name__) res = req.load(url, get, post, ref, cookies, just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 @@ -370,3 +368,17 @@ class Plugin(object): res = header return res + + + def clean(self): + """ + Clean everything and remove references + """ + for a in ("pyfile", "thread", "html"): + if hasattr(self, a): + setattr(self, a, None) + + try: + self.req.close() + finally: + self.req = None -- cgit v1.2.3 From 21cf50c60a794b5ca7d54408b590f74d4567ca79 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 07:15:28 +0200 Subject: Update some plugins --- module/plugins/internal/Account.py | 15 ++++----------- module/plugins/internal/MultiHoster.py | 4 ++-- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 6 +++--- 4 files changed, 10 insertions(+), 17 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index a402c4ad7..021dedf60 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -13,7 +13,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.12" + __version__ = "0.13" __status__ = "testing" __description__ = """Base account plugin""" @@ -73,7 +73,7 @@ class Account(Plugin): self.accounts[user]['valid'] = True #@TODO: Remove in 0.4.10 finally: - self.del_request() + self.clean() return res @@ -84,7 +84,7 @@ class Account(Plugin): req = self.get_request(user) if req: req.clearCookies() - self.del_request() + self.clean() if user in self.info: self.info[user]['login'].clear() @@ -254,7 +254,7 @@ class Account(Plugin): traceback.print_exc() finally: - self.del_request() + self.clean() self.info[user].update(info) return info @@ -288,13 +288,6 @@ class Account(Plugin): return self.pyload.requestFactory.getRequest(self.__name__, user) - def del_request(self): - try: - self.req.close() - finally: - self.req = None - - def get_cookies(self, user=None): if not user: user, info = self.select() diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index cdb668b82..a0ef3ff40 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.49" + __version__ = "0.50" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -100,7 +100,7 @@ class MultiHoster(SimpleHoster): except Fail, e: #@TODO: Move to PluginThread in 0.4.10 if self.premium: self.log_warning(_("Premium download failed")) - self.restart(reset=True) + self.restart(nopremium=True) elif self.get_config("revertfailed", True) \ and "new_module" in self.pyload.pluginManager.hosterPlugins[self.__name__]: diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 084bc133c..f044fbb50 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -68,7 +68,7 @@ class SimpleCrypter(Crypter, SimpleHoster): if self.account: if not self.user: - self.user, data = self.account.select() + self.user = self.account.select()[0] if not self.user or not self.account.is_logged(self.user, relogin=True): self.account = False diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index a03e301b3..8588fc5c5 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -23,7 +23,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.76" + __version__ = "1.77" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -210,7 +210,7 @@ class SimpleHoster(Hoster): self.leech_dl = False if not self.get_config('use_premium', True): - self.restart(reset=True) + self.restart(nopremium=True) if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) @@ -300,7 +300,7 @@ class SimpleHoster(Hoster): except Fail, e: #@TODO: Move to PluginThread in 0.4.10 if self.get_config('fallback', True) and self.premium: self.log_warning(_("Premium download failed"), e) - self.restart(reset=True) + self.restart(nopremium=True) else: raise Fail(encode(e)) #@TODO: Remove `encode` in 0.4.10 -- cgit v1.2.3 From 84f2193d76f7c6f47c834dc8902d8ead8e45a11a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 07:44:07 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1640 (2) --- module/plugins/internal/Account.py | 7 +++---- module/plugins/internal/Addon.py | 5 ++--- module/plugins/internal/Captcha.py | 6 +++--- module/plugins/internal/Extractor.py | 6 ++---- module/plugins/internal/Hoster.py | 5 ++--- module/plugins/internal/OCR.py | 8 ++------ module/plugins/internal/Plugin.py | 8 ++++++-- 7 files changed, 20 insertions(+), 25 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 021dedf60..1da31eede 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -13,7 +13,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.13" + __version__ = "0.14" __status__ = "testing" __description__ = """Base account plugin""" @@ -27,10 +27,9 @@ class Account(Plugin): def __init__(self, manager, accounts): - self.pyload = manager.core - self.info = {} #: Provide information in dict here + self._init(manager.core) + self.lock = threading.RLock() - self.req = None self.accounts = accounts #@TODO: Remove in 0.4.10 self.init() diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 649eccbc1..a2d51aa50 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -25,7 +25,7 @@ def threaded(fn): class Addon(Plugin): __name__ = "Addon" __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 - __version__ = "0.03" + __version__ = "0.04" __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] @@ -37,8 +37,7 @@ class Addon(Plugin): def __init__(self, core, manager): - self.pyload = core - self.info = {} #: Provide information in dict here + self._init(core) #: `HookManager` self.manager = manager diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 14308fbf5..3ebe42a89 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -12,7 +12,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.39" + __version__ = "0.40" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -21,9 +21,9 @@ class Captcha(Plugin): def __init__(self, plugin): #@TODO: Pass pyfile instead plugin, so store plugin's html in its associated pyfile as data - self.pyload = plugin.pyload + self._init(plugin.pyload) + self.plugin = plugin - self.info = {} #: Provide information in dict here self.task = None #: captchaManager task self.init() diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index c16b1a846..7d1f9ced5 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.31" + __version__ = "0.32" __status__ = "testing" __description__ = """Base extractor plugin""" @@ -86,9 +86,7 @@ class Extractor(Plugin): """ Initialize extractor for specific file """ - self.pyload = plugin.pyload - self.plugin = plugin - self.info = {} #: Provide information in dict here + self._init(plugin.pyload) self.filename = filename self.out = out diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index d707154bb..edc6f5d30 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.16" + __version__ = "0.17" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -58,8 +58,7 @@ class Hoster(Plugin): def __init__(self, pyfile): - self.pyload = pyfile.m.core - self.info = {} #: Provide information in dict here + self._init(pyfile.m.core) #: Engage wan reconnection self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10 diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 805755f39..a9982b076 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -21,7 +21,7 @@ from module.utils import save_join as fs_join class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" - __version__ = "0.17" + __version__ = "0.18" __status__ = "testing" __description__ = """OCR base plugin""" @@ -30,11 +30,7 @@ class OCR(Plugin): def __init__(self, plugin): - self.pyload = plugin.pyload - self.plugin = plugin - self.info = {} #: Provide information in dict here - - self.init() + super(OCR, self).__init__(plugin.pyload) def init(self): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 3d53b7d2d..601acd491 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -142,7 +142,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.24" + __version__ = "0.25" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -157,10 +157,14 @@ class Plugin(object): def __init__(self, core): + self._init(core) + self.init() + + + def _init(self, core): self.pyload = core self.info = {} #: Provide information in dict here self.req = None - self.init() def init(self): -- cgit v1.2.3 From fd2b928c08f0e1d1c54096696ef75bf899557db8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 08:56:00 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1649 --- module/plugins/internal/Extractor.py | 3 ++- module/plugins/internal/OCR.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 7d1f9ced5..7f5212090 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.32" + __version__ = "0.33" __status__ = "testing" __description__ = """Base extractor plugin""" @@ -88,6 +88,7 @@ class Extractor(Plugin): """ self._init(plugin.pyload) + self.plugin = plugin self.filename = filename self.out = out self.fullpath = fullpath diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index a9982b076..b24b3058b 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -21,7 +21,7 @@ from module.utils import save_join as fs_join class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" - __version__ = "0.18" + __version__ = "0.19" __status__ = "testing" __description__ = """OCR base plugin""" @@ -30,7 +30,9 @@ class OCR(Plugin): def __init__(self, plugin): - super(OCR, self).__init__(plugin.pyload) + self._init(plugin.pyload) + self.plugin = plugin + self.init() def init(self): -- cgit v1.2.3 From 04f4b7aa724454a69588ecc9fa46f7dd6e65c747 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 09:16:39 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1630 --- module/plugins/internal/Captcha.py | 50 +++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 3ebe42a89..8564d88d9 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -12,7 +12,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.40" + __version__ = "0.41" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -79,38 +79,38 @@ class Captcha(Plugin): with open(os.path.join("tmp", "captcha_image_%s_%s.%s" % (self.plugin.__name__, time_ref, input_type)), "wb") as tmp_img: tmp_img.write(raw) - if ocr: - if isinstance(ocr, basestring): - OCR = self.pyload.pluginManager.loadClass("captcha", ocr) #: Rename `captcha` to `ocr` in 0.4.10 - result = OCR(self.plugin).recognize(tmp_img.name) - else: - result = self.recognize(tmp_img.name) + if ocr: + if isinstance(ocr, basestring): + OCR = self.pyload.pluginManager.loadClass("captcha", ocr) #: Rename `captcha` to `ocr` in 0.4.10 + result = OCR(self.plugin).recognize(tmp_img.name) + else: + result = self.recognize(tmp_img.name) - if not result: - captchaManager = self.pyload.captchaManager + if not result: + captchaManager = self.pyload.captchaManager - try: - self.task = captchaManager.newTask(raw, input_type, tmp_img.name, output_type) + try: + self.task = captchaManager.newTask(raw, input_type, tmp_img.name, output_type) - captchaManager.handleCaptcha(self.task) + captchaManager.handleCaptcha(self.task) - self.task.setWaiting(max(timeout, 50)) #@TODO: Move to `CaptchaManager` in 0.4.10 - while self.task.isWaiting(): - if self.plugin.pyfile.abort: - self.plugin.abort() - time.sleep(1) + self.task.setWaiting(max(timeout, 50)) #@TODO: Move to `CaptchaManager` in 0.4.10 + while self.task.isWaiting(): + if self.plugin.pyfile.abort: + self.plugin.abort() + time.sleep(1) - finally: - captchaManager.removeTask(self.task) + finally: + captchaManager.removeTask(self.task) - if self.task.error: - self.fail(task.error) + if self.task.error: + self.fail(task.error) - elif not self.task.result: - self.invalid() - self.plugin.retry(reason=_("No captcha result obtained in appropiate time")) + elif not self.task.result: + self.invalid() + self.plugin.retry(reason=_("No captcha result obtained in appropiate time")) - result = self.task.result + result = self.task.result if not self.pyload.debug: try: -- cgit v1.2.3 From 539d2bc721a7363786da06b43de065f1405a61a0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 09:25:41 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1640 (3) --- module/plugins/internal/Plugin.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 601acd491..570c984ac 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -142,7 +142,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.25" + __version__ = "0.26" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -378,11 +378,11 @@ class Plugin(object): """ Clean everything and remove references """ - for a in ("pyfile", "thread", "html"): - if hasattr(self, a): - setattr(self, a, None) - try: self.req.close() - finally: - self.req = None + except Exception: + pass + + for a in ("pyfile", "thread", "html", "req"): + if hasattr(self, a): + setattr(self, a, None) -- cgit v1.2.3 From 8a97cd05eda1301dd2ac915f0eb9d8a57564fb70 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 10:22:15 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1650 --- module/plugins/internal/Captcha.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 8564d88d9..c08050ee8 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -12,7 +12,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.41" + __version__ = "0.42" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -104,7 +104,7 @@ class Captcha(Plugin): captchaManager.removeTask(self.task) if self.task.error: - self.fail(task.error) + self.fail(self.task.error) elif not self.task.result: self.invalid() -- cgit v1.2.3 From 5e15580202c44628f2fbfabad0c3f693975fb3c9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 17:13:17 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1663 --- module/plugins/internal/Account.py | 1 - module/plugins/internal/Addon.py | 2 +- module/plugins/internal/Crypter.py | 6 +++--- module/plugins/internal/Hoster.py | 15 ++++++++------- module/plugins/internal/Plugin.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 1da31eede..e9ba5e533 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -73,7 +73,6 @@ class Account(Plugin): finally: self.clean() - return res diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index a2d51aa50..45ca98eac 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -96,7 +96,7 @@ class Addon(Plugin): self.periodical() except Exception, e: - self.log_error(_("Error executing addon: %s") % e) + self.log_error(_("Error executing periodical task: %s") % e) if self.pyload.debug: traceback.print_exc() diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 0fdec559d..d0e8eb1b4 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -2,14 +2,14 @@ import urlparse -from module.plugins.internal.Hoster import Hoster +from module.plugins.internal.Hoster import Hoster, _fixurl from module.utils import save_path as safe_filename class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.06" + __version__ = "0.07" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -93,7 +93,7 @@ class Crypter(Hoster): elif not folder_per_package or name is not folder: if not folder: - folder = urlparse.urlparse(name).path.split("/")[-1] + folder = urlparse.urlparse(_fixurl(name)).path.split("/")[-1] set_folder(safe_filename(folder)) self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index edc6f5d30..67feec38c 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.17" + __version__ = "0.18" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -373,6 +373,9 @@ class Hoster(Plugin): self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + name = _fixurl(self.pyfile.name) + self.pyfile.name = urlparse.urlparse(name).path.split('/')[-1] or name + self.captcha.correct() self.check_for_same_files() @@ -384,15 +387,13 @@ class Hoster(Plugin): if not exists(download_location): try: os.makedirs(download_location) - except Exception, e: self.fail(e) self.set_permissions(download_location) location = fs_decode(download_location) - name = safe_filename(urlparse.urlparse(self.pyfile.name).path.split('/')[-1] or self.pyfile.name) - filename = os.path.join(location, name) + filename = os.path.join(location, safe_filename(self.pyfile.name)) #@TODO: Move `safe_filename` check to HTTPDownload in 0.4.10 self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename) @@ -407,10 +408,10 @@ class Hoster(Plugin): self.pyfile.size = self.req.size #@TODO: Recheck in 0.4.10 - if disposition: + if disposition and newname: finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] - if finalname != newname != name: + if finalname != newname != self.pyfile.name: try: os.rename(fs_join(location, newname), fs_join(location, finalname)) @@ -418,7 +419,7 @@ class Hoster(Plugin): self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e) finalname = newname - self.log_info(_("`%s` saved as `%s`") % (name, finalname)) + self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname)) self.pyfile.name = finalname filename = os.path.join(location, finalname) diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 570c984ac..ac6b4a57e 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -48,7 +48,7 @@ def exists(path): #@TODO: Move to utils in 0.4.10 def fixurl(url): - return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip() + return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip().rstrip('/') #@TODO: Move to utils in 0.4.10 @@ -142,7 +142,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.26" + __version__ = "0.27" __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 8588fc5c5..1d456a3ee 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -424,7 +424,7 @@ class SimpleHoster(Hoster): elif re.search('filename', errmsg, re.I): url_p = urlparse.urlparse(self.pyfile.url) - self.pyfile.url = "%s://%s/%s" % (url_p.scheme, url_p.netloc, url_p.path.strip('/').split('/')[0]) + self.pyfile.url = "%s://%s/%s" % (url_p.scheme, url_p.netloc, url_p.path.split('/')[0]) self.retry(1, reason=_("Wrong url")) elif re.search('premium', errmsg, re.I): -- cgit v1.2.3 From 9cb6b5d05ddcc15148bd5fab0fe02978159cfef5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 17:23:46 +0200 Subject: Tiny fixes --- module/plugins/internal/Plugin.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index ac6b4a57e..62eb17aa6 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -347,6 +347,7 @@ class Plugin(object): with open(framefile, "wb") as f: del frame #: Delete the frame or it wont be cleaned f.write(encode(res)) + except IOError, e: self.log_error(e) -- cgit v1.2.3 From d0acce3be51760fc97c6d8cf298ef37ba2f299ec Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 18:37:00 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1591 --- module/plugins/internal/SimpleCrypter.py | 28 +++++++++++++--------------- module/plugins/internal/XFSAccount.py | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index f044fbb50..bcbfd6488 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.59" + __version__ = "0.60" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -57,25 +57,23 @@ class SimpleCrypter(Crypter, SimpleHoster): #@TODO: Remove in 0.4.10 - def load_account(self): - accountname = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") + def _setup(self): + orig_name = self.__name__ + self.__name__ = (orig_name + ".py").replace("Folder.py", "").replace(".py", "") + + super(SimpleCrypter, self)._setup() - if self.req: - self.req.close() + self.__name__ = orig_name - if not self.account: - self.account = self.pyload.accountManager.getAccountPlugin(accountname) - if self.account: - if not self.user: - self.user = self.account.select()[0] + #@TODO: Remove in 0.4.10 + def load_account(self): + orig_name = self.__name__ + self.__name__ = (orig_name + ".py").replace("Folder.py", "").replace(".py", "") - if not self.user or not self.account.is_logged(self.user, relogin=True): - self.account = False + super(SimpleCrypter, self).load_account() - #: Browser instance, see `network.Browser` - self.req = self.pyload.requestFactory.getRequest(accountname, - self.user if self.account else None) + self.__name__ = orig_name def handle_direct(self, pyfile): diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 5bf959400..0cd723bab 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -152,7 +152,7 @@ class XFSAccount(Account): def login(self, user, password, data, req): if not self.HOSTER_URL: - self.login_fail(_("Missing HOSTER_DOMAIN")) + self.login_fail(_("Missing HOSTER_URL")) if not self.LOGIN_URL: self.LOGIN_URL = urlparse.urljoin(self.HOSTER_URL, "login.html") -- cgit v1.2.3 From 8180f932262473e4f6ab46ae65846c91ada6366d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 21:39:11 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1664 --- module/plugins/internal/Account.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index e9ba5e533..4553f3a79 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -13,13 +13,12 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.14" + __version__ = "0.15" __status__ = "testing" __description__ = """Base account plugin""" __license__ = "GPLv3" - __authors__ = [("mkaay" , "mkaay@mkaay.de" ), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] LOGIN_TIMEOUT = 10 * 60 #: After that time (in minutes) pyload will relogin the account @@ -141,7 +140,7 @@ class Account(Plugin): self.relogin(user) if options: - before = self.info[user]['data'][user]['options'] + before = self.info[user]['data']['options'] self.info[user]['data']['options'].update(options) return self.info[user]['data']['options'] != before -- cgit v1.2.3 From e5ce0acf056dc96c40d5616ab6d2b82f998eefbe Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Aug 2015 00:37:57 +0200 Subject: Use set_cookie instead cj.setCookie --- module/plugins/internal/Hoster.py | 2 +- module/plugins/internal/MultiHoster.py | 2 +- module/plugins/internal/Plugin.py | 22 +++++++++++++------ module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 2 +- module/plugins/internal/XFSAccount.py | 36 +++++++++++++++----------------- 6 files changed, 37 insertions(+), 29 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 67feec38c..87ee2188c 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -12,7 +12,7 @@ import urlparse from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, chunks, encode, exists, fixurl as _fixurl, replace_patterns, - seconds_to_midnight, set_cookies, parse_html_form, + seconds_to_midnight, set_cookie, set_cookies, parse_html_form, parse_html_tag_attr_value, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index a0ef3ff40..c0c928a45 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -3,7 +3,7 @@ import re from module.plugins.internal.Plugin import Fail, encode -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookie, set_cookies class MultiHoster(SimpleHoster): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 62eb17aa6..99b5751e5 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -81,11 +81,15 @@ def replace_patterns(string, ruleslist): return string +#@TODO: Remove in 0.4.10 and fix CookieJar.setCookie +def set_cookie(cj, domain, name, value): + return cj.setCookie(domain, name, encode(value)) + + def set_cookies(cj, cookies): for cookie in cookies: if isinstance(cookie, tuple) and len(cookie) == 3: - domain, name, value = cookie - cj.setCookie(domain, name, encode(value)) #@TODO: Remove `encode` in 0.4.10 + set_cookie(cj, *cookie) def parse_html_tag_attr_value(attr_name, tag): @@ -142,7 +146,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.27" + __version__ = "0.28" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -329,12 +333,18 @@ class Plugin(object): if req is None: req = self.req or self.pyload.requestFactory.getRequest(self.__name__) - res = req.load(url, get, post, ref, cookies, just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 + #@TODO: Move to network in 0.4.10 + if isinstance(self.COOKIES, list): + set_cookies(req.cj, cookies) + + res = req.load(url, get, post, ref, bool(cookies), just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 - if decode: #@TODO: Move to network in 0.4.10 + #@TODO: Move to network in 0.4.10 + if decode: res = html_unescape(res) - if isinstance(decode, basestring): #@TODO: Move to network in 0.4.10 + #@TODO: Move to network in 0.4.10 + if isinstance(decode, basestring): res = decode(res, decode) if self.pyload.debug: diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index bcbfd6488..6a3f91a5b 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -3,7 +3,7 @@ import re from module.plugins.internal.Crypter import Crypter -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookie, set_cookies from module.utils import fixup, html_unescape diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1d456a3ee..e27f9a231 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -12,7 +12,7 @@ from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, encode, fixurl, replace_patterns, seconds_to_midnight, set_cookies +from module.plugins.internal.Plugin import Fail, encode, fixurl, replace_patterns, seconds_to_midnight, set_cookie, set_cookies from module.utils import fixup, fs_encode, parseFileSize as parse_size diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 0cd723bab..efb84de35 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -5,13 +5,13 @@ import time import urlparse from module.plugins.internal.Account import Account -from module.plugins.internal.Plugin import parse_html_form, set_cookies +from module.plugins.internal.Plugin import parse_html_form, set_cookie class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.40" + __version__ = "0.41" __status__ = "testing" __description__ = """XFileSharing account plugin""" @@ -39,20 +39,6 @@ class XFSAccount(Account): LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<' - def init(self): - if not self.HOSTER_DOMAIN: - self.log_error(_("Missing HOSTER_DOMAIN")) - self.COOKIES = False - - else: - if not self.HOSTER_URL: - self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN - - if isinstance(self.COOKIES, list): - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) - set_cookies(req.cj, self.COOKIES) - - def parse_info(self, user, password, data, req): validuntil = None trafficleft = None @@ -65,7 +51,9 @@ class XFSAccount(Account): 'leechtraffic': leechtraffic, 'premium' : premium} - html = self.load(self.HOSTER_URL, get={'op': "my_account"}) + html = self.load(self.HOSTER_URL, + get={'op': "my_account"}, + cookies=self.COOKIES) premium = True if re.search(self.PREMIUM_PATTERN, html) else False @@ -151,12 +139,22 @@ class XFSAccount(Account): def login(self, user, password, data, req): + if self.HOSTER_DOMAIN: + if not self.HOSTER_URL: + self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN + + if isinstance(self.COOKIES, list): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + else: + set_cookie(req.cj, self.HOSTER_DOMAIN, "lang", "english") + if not self.HOSTER_URL: self.login_fail(_("Missing HOSTER_URL")) if not self.LOGIN_URL: self.LOGIN_URL = urlparse.urljoin(self.HOSTER_URL, "login.html") - html = self.load(self.LOGIN_URL) + + html = self.load(self.LOGIN_URL, cookies=self.COOKIES) action, inputs = parse_html_form('name="FL"', html) if not inputs: @@ -171,7 +169,7 @@ class XFSAccount(Account): else: url = self.HOSTER_URL - html = self.load(url, post=inputs) + html = self.load(url, post=inputs, cookies=self.COOKIES) if re.search(self.LOGIN_FAIL_PATTERN, html): self.login_fail() -- cgit v1.2.3 From f2416468c89af6064b07ee51384818502ae54b05 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Mon, 3 Aug 2015 21:27:55 +0300 Subject: Update Plugin.py --- module/plugins/internal/Plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 99b5751e5..7d1fea1e3 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -146,7 +146,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.28" + __version__ = "0.29" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -334,7 +334,7 @@ class Plugin(object): req = self.req or self.pyload.requestFactory.getRequest(self.__name__) #@TODO: Move to network in 0.4.10 - if isinstance(self.COOKIES, list): + if hasattr(self, COOKIES) and isinstance(self.COOKIES, list): set_cookies(req.cj, cookies) res = req.load(url, get, post, ref, bool(cookies), just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 -- cgit v1.2.3 From 05a9b379f63c16cf1b320d81ab49abf8ba167a84 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Mon, 3 Aug 2015 21:30:18 +0300 Subject: Update XFSAccount.py --- module/plugins/internal/XFSAccount.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index efb84de35..7093ccd27 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -143,10 +143,11 @@ class XFSAccount(Account): if not self.HOSTER_URL: self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN - if isinstance(self.COOKIES, list): - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) - else: - set_cookie(req.cj, self.HOSTER_DOMAIN, "lang", "english") + if hasattr(self, 'COOKIES'): + if isinstance(self.COOKIES, list): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + else: + set_cookie(req.cj, self.HOSTER_DOMAIN, "lang", "english") if not self.HOSTER_URL: self.login_fail(_("Missing HOSTER_URL")) -- cgit v1.2.3 From 967f6d917fa1c7a70958528d0de918a13f91d16b Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Mon, 3 Aug 2015 21:35:31 +0300 Subject: Update Plugin.py --- module/plugins/internal/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 7d1fea1e3..971bf7bed 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -334,7 +334,7 @@ class Plugin(object): req = self.req or self.pyload.requestFactory.getRequest(self.__name__) #@TODO: Move to network in 0.4.10 - if hasattr(self, COOKIES) and isinstance(self.COOKIES, list): + if hasattr(self, 'COOKIES') and isinstance(self.COOKIES, list): set_cookies(req.cj, cookies) res = req.load(url, get, post, ref, bool(cookies), just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 -- cgit v1.2.3 From c8a4a19c35e1ec1f7afcacc88e223160ac89659f Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Mon, 3 Aug 2015 21:58:24 +0300 Subject: Update SimpleHoster.py --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e27f9a231..e2cc21ed8 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -23,7 +23,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.77" + __version__ = "1.78" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -221,7 +221,7 @@ class SimpleHoster(Hoster): self.req.setOption("timeout", 120) - if isinstance(self.COOKIES, list): + if hasattr(self, 'COOKIES') and isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) if self.LINK_PATTERN: -- cgit v1.2.3 From 459bd7482a67334e91a753adba1741181d634291 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Mon, 3 Aug 2015 23:22:15 +0300 Subject: Update Plugin.py --- module/plugins/internal/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 971bf7bed..5f70a292d 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -335,7 +335,7 @@ class Plugin(object): #@TODO: Move to network in 0.4.10 if hasattr(self, 'COOKIES') and isinstance(self.COOKIES, list): - set_cookies(req.cj, cookies) + set_cookies(req.cj, self.COOKIES) res = req.load(url, get, post, ref, bool(cookies), just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 -- cgit v1.2.3 From f46daaab4c262b4a727525c70d7b470af05d343c Mon Sep 17 00:00:00 2001 From: TodsDeath Date: Tue, 4 Aug 2015 00:12:05 +0200 Subject: Fix #1603 --- module/plugins/internal/UnRar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 92128c77f..0386991d9 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.24" + __version__ = "1.25" __status__ = "testing" __description__ = """Rar extractor plugin""" @@ -177,7 +177,7 @@ class UnRar(Extractor): #: eventually Multipart Files files.extend(fs_join(dir, os.path.basename(file)) for file in filter(self.is_multipart, os.listdir(dir)) - if re.sub(self.re_multipart, ".rar", name) is re.sub(self.re_multipart, ".rar", file)) + if re.sub(self.re_multipart, ".rar", name) == re.sub(self.re_multipart, ".rar", file)) return files -- cgit v1.2.3 From 2cf928db10224b5327f918dceaa13273753620ac Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Aug 2015 18:06:42 +0200 Subject: Some fixes --- module/plugins/internal/Account.py | 15 +++++++-------- module/plugins/internal/Plugin.py | 6 +++--- module/plugins/internal/SimpleHoster.py | 7 ++----- module/plugins/internal/XFSAccount.py | 8 ++++---- module/plugins/internal/XFSCrypter.py | 9 ++++++--- module/plugins/internal/XFSHoster.py | 9 ++++++--- 6 files changed, 28 insertions(+), 26 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 4553f3a79..ec89caa00 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -13,7 +13,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.15" + __version__ = "0.16" __status__ = "testing" __description__ = """Base account plugin""" @@ -190,7 +190,6 @@ class Account(Plugin): return data - @lock def get_info(self, user, reload=False): """ Retrieve account infos for an user, do **not** overwrite this method!\\ @@ -213,13 +212,13 @@ class Account(Plugin): safe_info['data']['password'] = "**********" #@TODO: Remove in 0.4.10 self.log_debug("Account info for user `%s`: %s" % (user, safe_info)) + elif self.INFO_THRESHOLD > 0 and self.info[user]['login']['timestamp'] + self.INFO_THRESHOLD < time.time(): + self.log_debug("Reached data timeout for %s" % user) + info = self.get_info(user, True) + else: info = self.info[user] - if self.INFO_THRESHOLD > 0 and info['login']['timestamp'] + self.INFO_THRESHOLD < time.time(): - self.log_debug("Reached data timeout for %s" % user) - self.schedule_refresh(user) - return info @@ -371,12 +370,12 @@ class Account(Plugin): self.schedule_refresh(user, 60 * 60) - def schedule_refresh(self, user, time=0, reload=True): + def schedule_refresh(self, user, time=0): """ Add task to refresh account info to sheduler """ self.log_debug("Scheduled refresh for user `%s` in %s seconds" % (user, time)) - self.pyload.scheduler.addJob(time, self.get_info, [user, reload]) + self.pyload.scheduler.addJob(time, self.get_info, [user, True]) #: Deprecated method, use `schedule_refresh` instead (Remove in 0.4.10) diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 5f70a292d..7b45c40a8 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -146,7 +146,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.29" + __version__ = "0.30" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -334,8 +334,8 @@ class Plugin(object): req = self.req or self.pyload.requestFactory.getRequest(self.__name__) #@TODO: Move to network in 0.4.10 - if hasattr(self, 'COOKIES') and isinstance(self.COOKIES, list): - set_cookies(req.cj, self.COOKIES) + if isinstance(cookies, list): + set_cookies(req.cj, cookies) res = req.load(url, get, post, ref, bool(cookies), just_header, multipart, decode is True) #@TODO: Fix network multipart in 0.4.10 diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e2cc21ed8..8d4100026 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -23,7 +23,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.78" + __version__ = "1.79" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -221,9 +221,6 @@ class SimpleHoster(Hoster): self.req.setOption("timeout", 120) - if hasattr(self, 'COOKIES') and isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) - if self.LINK_PATTERN: if not hasattr(self, 'LINK_FREE_PATTERN'): self.LINK_FREE_PATTERN = self.LINK_PATTERN @@ -250,7 +247,7 @@ class SimpleHoster(Hoster): def preload(self): self.html = self.load(self.pyfile.url, - cookies=bool(self.COOKIES), + cookies=self.COOKIES, ref=False, decode=self.TEXT_ENCODING) diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 7093ccd27..e0f6b1ee8 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -11,7 +11,7 @@ from module.plugins.internal.Plugin import parse_html_form, set_cookie class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.41" + __version__ = "0.42" __status__ = "testing" __description__ = """XFileSharing account plugin""" @@ -143,11 +143,11 @@ class XFSAccount(Account): if not self.HOSTER_URL: self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN - if hasattr(self, 'COOKIES'): - if isinstance(self.COOKIES, list): + if self.COOKIES: + if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")): self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) else: - set_cookie(req.cj, self.HOSTER_DOMAIN, "lang", "english") + set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") if not self.HOSTER_URL: self.login_fail(_("Missing HOSTER_URL")) diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index fcb017466..eaf295bf2 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.11" + __version__ = "0.12" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -40,7 +40,10 @@ class XFSCrypter(SimpleCrypter): else: self.fail(_("Missing HOSTER_DOMAIN")) - if isinstance(self.COOKIES, list): - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + if self.COOKIES: + if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + else: + set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") return super(XFSCrypter, self).prepare() diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 962cffb06..715ef93ca 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -13,7 +13,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.55" + __version__ = "0.56" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -71,8 +71,11 @@ class XFSHoster(SimpleHoster): else: self.fail(_("Missing HOSTER_DOMAIN")) - if isinstance(self.COOKIES, list): - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + if self.COOKIES: + if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + else: + set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") if not self.LINK_PATTERN: pattern = r'(?:file: "(.+?)"|(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<])' -- cgit v1.2.3 From 34728780b6f4f560dbb033e8955b7c2f1dec02e5 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 4 Aug 2015 22:48:26 +0300 Subject: [Account] fix relogin not working --- module/plugins/internal/Account.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index ec89caa00..2713e8da4 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -13,7 +13,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.16" + __version__ = "0.17" __status__ = "testing" __description__ = """Base account plugin""" @@ -83,9 +83,6 @@ class Account(Plugin): req.clearCookies() self.clean() - if user in self.info: - self.info[user]['login'].clear() - return self._login(user) -- cgit v1.2.3 From f80576bd8c809932dd9f62483402ec18532699b2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Aug 2015 22:08:42 +0200 Subject: Some fixes (2) --- module/plugins/internal/XFSCrypter.py | 3 ++- module/plugins/internal/XFSHoster.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index eaf295bf2..4c059d647 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- +from module.plugins.internal.Plugin import set_cookie from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.12" + __version__ = "0.13" __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 715ef93ca..5e0830dc6 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -6,6 +6,7 @@ import re from module.plugins.captcha.ReCaptcha import ReCaptcha from module.plugins.captcha.SolveMedia import SolveMedia +from module.plugins.internal.Plugin import set_cookie from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, seconds_to_midnight from module.utils import html_unescape @@ -13,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.56" + __version__ = "0.57" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 0a5daec6812f27e68673c7ef5842f701b4ed1594 Mon Sep 17 00:00:00 2001 From: estaban Date: Tue, 4 Aug 2015 23:11:43 +0200 Subject: Download deleted because of file size mismatch One some hosters like OneFichier for example, the webpage displays a file size of 1.46GB. However the real file size can be 1.4649853GB, which makes a difference between the downloaded file size and the expected file size. size_tolerance is meant for that, but its value is currently set to 1MB. This is an issue in the above case where the difference between expected size and downloaded size is almost 10MB. In many cases, this causes the downloaded file to simply be deleted, even though it has been downloaded without any issue. The biggest possible size difference is 10MB, at least if the website displays 2 decimals and that the biggest unit is GB. Therefore I propose to change the max tolerance from 1MB to 10MB. FYI the error message in the log looks like that: Fail: File size mismatch | Expected file size: 1492501135.36 | Downloaded file size: 1488025852 --- module/plugins/internal/SimpleHoster.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 8d4100026..5c2bafe8e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -310,9 +310,11 @@ class SimpleHoster(Hoster): self.captcha.invalid() self.retry(10, reason=_("Wrong captcha")) + # 10485760 is 10MB, tolerance is used when comparing displayed size to real size + # For example displayed size can be 1.46GB for example, but real size can be 1.4649853 elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, file_size=self.info['size'] if 'size' in self.info else 0, - size_tolerance=1048576, + size_tolerance=10485760, delete=True): self.error(_("Empty file")) -- cgit v1.2.3 From 0019494ce0c844952f1acb5cd4b661e256839baa Mon Sep 17 00:00:00 2001 From: estaban Date: Tue, 4 Aug 2015 23:16:56 +0200 Subject: Update SimpleHoster.py --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 5c2bafe8e..91d5fe4b3 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -310,8 +310,8 @@ class SimpleHoster(Hoster): self.captcha.invalid() self.retry(10, reason=_("Wrong captcha")) - # 10485760 is 10MB, tolerance is used when comparing displayed size to real size - # For example displayed size can be 1.46GB for example, but real size can be 1.4649853 + # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size + # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, file_size=self.info['size'] if 'size' in self.info else 0, size_tolerance=10485760, -- cgit v1.2.3 From 4323507752c54c03cf109ee4c5b74d11bae753dd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Aug 2015 23:57:23 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1707 --- module/plugins/internal/Hoster.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 87ee2188c..a0cdb1e2e 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.18" + __version__ = "0.19" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -332,7 +332,7 @@ class Hoster(Plugin): if self.premium: self.retry_free = True else: - self.fail(reason, _("Download was already free")) + self.fail("%s | %s" % (reason, _("Download was already free"))) raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 91d5fe4b3..9c310ca27 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -23,7 +23,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.79" + __version__ = "1.80" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3