From 3af8bfcf685f82de097ef6b110dffa77fd529eaf Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 10:01:24 +0200 Subject: [SimpleHoster] Improve logging and file checking --- module/plugins/accounts/ShareonlineBiz.py | 2 +- module/plugins/crypter/Dereferer.py | 6 +- module/plugins/crypter/XFileSharingProFolder.py | 6 +- module/plugins/hoster/SolidfilesCom.py | 2 +- module/plugins/hoster/XFileSharingPro.py | 6 +- module/plugins/internal/MultiHoster.py | 8 +- module/plugins/internal/SimpleCrypter.py | 13 ++- module/plugins/internal/SimpleHoster.py | 124 ++++++++++++++++-------- 8 files changed, 100 insertions(+), 67 deletions(-) (limited to 'module') diff --git a/module/plugins/accounts/ShareonlineBiz.py b/module/plugins/accounts/ShareonlineBiz.py index b9509b466..9269cf2a1 100644 --- a/module/plugins/accounts/ShareonlineBiz.py +++ b/module/plugins/accounts/ShareonlineBiz.py @@ -64,5 +64,5 @@ class ShareonlineBiz(Account): html = self.api_response(user, req) err = re.search(r'\*\*(.+?)\*\*', html) if err: - self.logError(err.group(1)) + self.logError(err.group(1).strip()) self.wrongPassword() diff --git a/module/plugins/crypter/Dereferer.py b/module/plugins/crypter/Dereferer.py index 85b84a242..cdc8d2d53 100644 --- a/module/plugins/crypter/Dereferer.py +++ b/module/plugins/crypter/Dereferer.py @@ -8,7 +8,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter class Dereferer(SimpleCrypter): __name__ = "Dereferer" __type__ = "crypter" - __version__ = "0.14" + __version__ = "0.15" __pattern__ = r'https?://(?:www\.)?(?:\w+\.)*?(?P(?:[\d.]+|[\w\-]{3,}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)/.*?(?P(?:ht|f)tps?://.+)' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), @@ -24,9 +24,7 @@ class Dereferer(SimpleCrypter): def _log(self, type, args): - msg = " | ".join(str(a).strip() for a in args if a) - logger = getattr(self.log, type) - logger("%s: %s: %s" % (self.__name__, self.HOSTER_NAME, msg or _("%s MARK" % type.upper()))) + return super(Dereferer, self)._log(type, [self.HOSTER_NAME] + args) def init(self): diff --git a/module/plugins/crypter/XFileSharingProFolder.py b/module/plugins/crypter/XFileSharingProFolder.py index 2e1a2f022..8043a1183 100644 --- a/module/plugins/crypter/XFileSharingProFolder.py +++ b/module/plugins/crypter/XFileSharingProFolder.py @@ -8,7 +8,7 @@ from module.plugins.internal.XFSCrypter import XFSCrypter, create_getInfo class XFileSharingProFolder(XFSCrypter): __name__ = "XFileSharingProFolder" __type__ = "crypter" - __version__ = "0.06" + __version__ = "0.07" __pattern__ = r'https?://(?:www\.)?(?:\w+\.)*?(?P(?:[\d.]+|[\w\-^_]{3,}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)/(?:user|folder)s?/\w+' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), @@ -20,9 +20,7 @@ class XFileSharingProFolder(XFSCrypter): def _log(self, type, args): - msg = " | ".join(str(a).strip() for a in args if a) - logger = getattr(self.log, type) - logger("%s: %s: %s" % (self.__name__, self.HOSTER_NAME, msg or _("%s MARK" % type.upper()))) + return super(XFileSharingProFolder, self)._log(type, [self.HOSTER_NAME] + args) def init(self): diff --git a/module/plugins/hoster/SolidfilesCom.py b/module/plugins/hoster/SolidfilesCom.py index d359577d6..184329a40 100644 --- a/module/plugins/hoster/SolidfilesCom.py +++ b/module/plugins/hoster/SolidfilesCom.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Test links: -# http://www.solidfiles.com/d/609cdb4b1b +# http://www.solidfiles.com/d/609cdb4b1b from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/module/plugins/hoster/XFileSharingPro.py b/module/plugins/hoster/XFileSharingPro.py index 169fbc8d3..c62a42779 100644 --- a/module/plugins/hoster/XFileSharingPro.py +++ b/module/plugins/hoster/XFileSharingPro.py @@ -8,7 +8,7 @@ from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo class XFileSharingPro(XFSHoster): __name__ = "XFileSharingPro" __type__ = "hoster" - __version__ = "0.46" + __version__ = "0.47" __pattern__ = r'https?://(?:www\.)?(?:\w+\.)*?(?P(?:[\d.]+|[\w\-^_]{3,}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)/(?:embed-)?\w{12}(?:\W|$)' @@ -21,9 +21,7 @@ class XFileSharingPro(XFSHoster): def _log(self, type, args): - msg = " | ".join(str(a).strip() for a in args if a) - logger = getattr(self.log, type) - logger("%s: %s: %s" % (self.__name__, self.HOSTER_NAME, msg or _("%s MARK" % type.upper()))) + return super(XFileSharingPro, self)._log(type, [self.HOSTER_NAME] + args) def init(self): diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 2b4ceb9b0..2b3eb8941 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.41" + __version__ = "0.42" __pattern__ = r'^unmatchable$' __config__ = [("use_premium" , "bool", "Use premium account if available" , True), @@ -32,8 +32,8 @@ class MultiHoster(SimpleHoster): def prepare(self): self.info = {} self.html = "" - self.link = "" #@TODO: Move to hoster class in 0.4.10 - self.directDL = False #@TODO: Move to hoster class in 0.4.10 + self.link = "" #@TODO: Move to Hoster in 0.4.10 + self.directDL = False #@TODO: Move to Hoster in 0.4.10 if not self.getConfig('use_premium', True): self.retryFree() @@ -77,7 +77,7 @@ class MultiHoster(SimpleHoster): self.logDebug("Handled as free download") self.handleFree(pyfile) - self.downloadLink(self.link, True) + self.download(self.link, ref=False, disposition=True) self.checkFile() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 6279be1bd..00b872b69 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -11,7 +11,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.50" + __version__ = "0.51" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] @@ -74,8 +74,8 @@ class SimpleCrypter(Crypter, SimpleHoster): self.info = {} self.html = "" - self.link = "" #@TODO: Move to hoster class in 0.4.10 - self.links = [] #@TODO: Move to hoster class in 0.4.10 + self.link = "" #@TODO: Move to Hoster in 0.4.10 + self.links = [] #@TODO: Move to Hoster in 0.4.10 if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) @@ -121,6 +121,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.logDebug("Package has %d links" % len(self.links)) if self.links: + self.links = [html_unescape(l.decode('unicode-escape').strip()) for l in links] #@TODO: Move to Crypter in 0.4.10 self.packages = [(self.info['name'], self.links, self.info['folder'])] elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 @@ -160,10 +161,8 @@ class SimpleCrypter(Crypter, SimpleHoster): url_p = urlparse.urlparse(self.pyfile.url) baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - links = [urlparse.urljoin(baseurl, link) if not urlparse.urlparse(link).scheme else link \ - for link in re.findall(self.LINK_PATTERN, self.html)] - - return [html_unescape(l.decode('unicode-escape').strip()) for l in links] + return [urlparse.urljoin(baseurl, link) if not urlparse.urlparse(link).scheme else link \ + for link in re.findall(self.LINK_PATTERN, self.html)] def handlePages(self, pyfile): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index da477ad79..a1a1a4a67 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -133,7 +133,7 @@ def timestamp(): return int(time.time() * 1000) -#@TODO: Move to hoster class in 0.4.10 +#@TODO: Move to Hoster in 0.4.10 def getFileURL(self, url, follow_location=None): link = "" redirect = 1 @@ -239,7 +239,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.56" + __version__ = "1.57" __pattern__ = r'^unmatchable$' __config__ = [("use_premium", "bool", "Use premium account if available" , True), @@ -308,6 +308,11 @@ class SimpleHoster(Hoster): SIZE_REPLACEMENTS = [] URL_REPLACEMENTS = [] + FILE_ERRORS = [('Html error' , r'\A(?:\s*<.+>)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)'), + ('Request error', r'([Aa]n error occured while processing your request)' ), + ('Html file' , r'\A\s*)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)')}) - - if not errmsg: - for r, p in [('Html file' , re.compile(r'\A\s*