diff options
author | Walter Purcaro <vuolter@users.noreply.github.com> | 2015-03-26 17:04:38 +0100 |
---|---|---|
committer | Walter Purcaro <vuolter@users.noreply.github.com> | 2015-03-26 17:04:38 +0100 |
commit | ccb6aaee9d987b56faf19fb48dd06a04ff8ca721 (patch) | |
tree | 6a385417ff77f3ed17329e2d606f80b673bdb3c0 /pyload/plugin/internal | |
parent | Prepare to merging (diff) | |
parent | Tiny code cosmetics (diff) | |
download | pyload-ccb6aaee9d987b56faf19fb48dd06a04ff8ca721.tar.xz |
Merge branch 'stable' into 0.4.10
Conflicts:
module/plugins/accounts/Keep2ShareCc.py
module/plugins/captcha/OCR.py
module/plugins/captcha/captcha.py
module/plugins/crypter/DailymotionBatch.py
module/plugins/crypter/DailymotionComFolder.py
module/plugins/crypter/YoutubeBatch.py
module/plugins/crypter/YoutubeComFolder.py
module/plugins/internal/CaptchaService.py
pyload/plugin/Extractor.py
pyload/plugin/OCR.py
pyload/plugin/account/DebridItaliaCom.py
pyload/plugin/account/MegaRapidCz.py
pyload/plugin/account/NoPremiumPl.py
pyload/plugin/account/RapideoPl.py
pyload/plugin/addon/AndroidPhoneNotify.py
pyload/plugin/addon/Checksum.py
pyload/plugin/addon/ClickAndLoad.py
pyload/plugin/addon/DeleteFinished.py
pyload/plugin/addon/DownloadScheduler.py
pyload/plugin/addon/ExternalScripts.py
pyload/plugin/addon/ExtractArchive.py
pyload/plugin/addon/HotFolder.py
pyload/plugin/addon/IRCInterface.py
pyload/plugin/addon/JustPremium.py
pyload/plugin/addon/MergeFiles.py
pyload/plugin/addon/MultiHome.py
pyload/plugin/addon/RestartFailed.py
pyload/plugin/addon/RestartSlow.py
pyload/plugin/addon/SkipRev.py
pyload/plugin/addon/UnSkipOnFail.py
pyload/plugin/addon/UpdateManager.py
pyload/plugin/addon/WindowsPhoneNotify.py
pyload/plugin/crypter/DDLMusicOrg.py
pyload/plugin/crypter/DailymotionBatch.py
pyload/plugin/crypter/DevhostSt.py
pyload/plugin/crypter/EmbeduploadCom.py
pyload/plugin/crypter/MultiloadCz.py
pyload/plugin/crypter/YoutubeBatch.py
pyload/plugin/extractor/SevenZip.py
pyload/plugin/extractor/UnRar.py
pyload/plugin/extractor/UnZip.py
pyload/plugin/hook/BypassCaptcha.py
pyload/plugin/hook/Captcha9Kw.py
pyload/plugin/hook/CaptchaBrotherhood.py
pyload/plugin/hook/DeathByCaptcha.py
pyload/plugin/hook/ExpertDecoders.py
pyload/plugin/hook/ImageTyperz.py
pyload/plugin/hook/XFileSharingPro.py
pyload/plugin/hoster/AlldebridCom.py
pyload/plugin/hoster/CzshareCom.py
pyload/plugin/hoster/EuroshareEu.py
pyload/plugin/hoster/FastixRu.py
pyload/plugin/hoster/FastshareCz.py
pyload/plugin/hoster/GooIm.py
pyload/plugin/hoster/MediafireCom.py
pyload/plugin/hoster/MegaDebridEu.py
pyload/plugin/hoster/NitroflareCom.py
pyload/plugin/hoster/OverLoadMe.py
pyload/plugin/hoster/PremiumTo.py
pyload/plugin/hoster/PremiumizeMe.py
pyload/plugin/hoster/RapidgatorNet.py
pyload/plugin/hoster/RealdebridCom.py
pyload/plugin/hoster/SimplyPremiumCom.py
pyload/plugin/hoster/SimplydebridCom.py
pyload/plugin/hoster/SmoozedCom.py
pyload/plugin/hoster/SoundcloudCom.py
pyload/plugin/hoster/UlozTo.py
pyload/plugin/hoster/UploadableCh.py
pyload/plugin/hoster/UploadedTo.py
pyload/plugin/hoster/UploadheroCom.py
pyload/plugin/hoster/VeehdCom.py
pyload/plugin/hoster/VimeoCom.py
pyload/plugin/hoster/ZeveraCom.py
pyload/plugin/hoster/ZippyshareCom.py
pyload/plugin/internal/BasePlugin.py
pyload/plugin/internal/MultiHoster.py
pyload/plugin/internal/SimpleDereferer.py
pyload/plugin/internal/SimpleHoster.py
pyload/plugin/internal/XFSHoster.py
pyload/plugin/ocr/GigasizeCom.py
pyload/plugin/ocr/LinksaveIn.py
pyload/plugin/ocr/NetloadIn.py
pyload/plugin/ocr/ShareonlineBiz.py
Diffstat (limited to 'pyload/plugin/internal')
-rw-r--r-- | pyload/plugin/internal/BasePlugin.py | 28 | ||||
-rw-r--r-- | pyload/plugin/internal/MultiHook.py | 126 | ||||
-rw-r--r-- | pyload/plugin/internal/MultiHoster.py | 67 | ||||
-rw-r--r-- | pyload/plugin/internal/SimpleCrypter.py | 8 | ||||
-rw-r--r-- | pyload/plugin/internal/SimpleDereferer.py | 10 | ||||
-rw-r--r-- | pyload/plugin/internal/SimpleHoster.py | 173 | ||||
-rw-r--r-- | pyload/plugin/internal/XFSAccount.py | 10 | ||||
-rw-r--r-- | pyload/plugin/internal/XFSHoster.py | 14 |
8 files changed, 265 insertions, 171 deletions
diff --git a/pyload/plugin/internal/BasePlugin.py b/pyload/plugin/internal/BasePlugin.py index 103e0d5cb..ed4da72a1 100644 --- a/pyload/plugin/internal/BasePlugin.py +++ b/pyload/plugin/internal/BasePlugin.py @@ -6,14 +6,14 @@ from urllib import unquote from urlparse import urljoin, urlparse from pyload.network.HTTPRequest import BadHeader -from pyload.plugin.internal.SimpleHoster import fileUrl +from pyload.plugin.internal.SimpleHoster import create_getInfo, getFileURL from pyload.plugin.Hoster import Hoster class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.34" + __version__ = "0.38" __pattern__ = r'^unmatchable$' @@ -51,7 +51,7 @@ class BasePlugin(Hoster): for _i in xrange(5): try: - link = fileUrl(self, unquote(pyfile.url)) + link = getFileURL(self, unquote(pyfile.url)) if link: self.download(link, ref=False, disposition=True) @@ -85,8 +85,20 @@ class BasePlugin(Hoster): else: self.fail(_("No file downloaded")) #@TODO: Move to hoster class in 0.4.10 - check = self.checkDownload({'empty file': re.compile(r'\A\Z'), - 'html file' : re.compile(r'\A\s*<!DOCTYPE html'), - 'html error': re.compile(r'\A\s*(<.+>)?\d{3}(\Z|\s+)')}) - if check: - self.fail(check.capitalize()) + errmsg = self.checkDownload({'Empty file' : re.compile(r'\A\s*\Z'), + 'Html error' : re.compile(r'\A(?:\s*<.+>)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)'), + 'Html file' : re.compile(r'\A\s*<!DOCTYPE html'), + 'Request error': re.compile(r'([Aa]n error occured while processing your request)')}) + if not errmsg: + return + + try: + errmsg += " | " + self.lastCheck.group(1).strip() + except Exception: + pass + + self.logWarning("Check result: " + errmsg, "Waiting 1 minute and retry") + self.retry(3, 60, errmsg) + + +getInfo = create_getInfo(BasePlugin) diff --git a/pyload/plugin/internal/MultiHook.py b/pyload/plugin/internal/MultiHook.py index 2beccfcc5..d43956691 100644 --- a/pyload/plugin/internal/MultiHook.py +++ b/pyload/plugin/internal/MultiHook.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import re - -from time import sleep +import time from pyload.plugin.Hook import Hook from pyload.utils import decode, remove_chars @@ -11,28 +10,22 @@ from pyload.utils import decode, remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.37" + __version__ = "0.40" - __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), - ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), - ("revertfailed" , "bool" , "Revert to standard download if fails", True ), - ("retry" , "int" , "Number of retries before revert" , 10 ), - ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), - ("reload" , "bool" , "Reload plugin list" , True ), - ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] __description__ = """Hook plugin for multi hoster/crypter""" __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org"), + __authors__ = [("pyLoad Team" , "admin@pyload.org" ), ("Walter Purcaro", "vuolter@gmail.com")] - MIN_INTERVAL = 1 * 60 * 60 + MIN_RELOAD_INTERVAL = 1 * 60 * 60 #: 1 hour DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"), - (r'1fichier\.com' , "onefichier.com" ), - (r'2shared\.com' , "twoshared.com" ), - (r'4shared\.com' , "fourshared.com" ), (r'bayfiles\.net' , "bayfiles.com" ), (r'cloudnator\.com' , "shragle.com" ), (r'dfiles\.eu' , "depositfiles.com" ), @@ -48,10 +41,21 @@ class MultiHook(Hook): (r'uploaded\.net' , "uploaded.to" ), (r'uploadhero\.co' , "uploadhero.com" ), (r'zshares\.net' , "zshare.net" ), - (r'(\d+.+)' , "X\1" )] + (r'^1' , "one" ), + (r'^2' , "two" ), + (r'^3' , "three" ), + (r'^4' , "four" ), + (r'^5' , "five" ), + (r'^6' , "six" ), + (r'^7' , "seven" ), + (r'^8' , "eight" ), + (r'^9' , "nine" ), + (r'^0' , "zero" )] def setup(self): + self.info = {} #@TODO: Remove in 0.4.10 + self.plugins = [] self.supported = [] self.new_supported = [] @@ -106,7 +110,7 @@ class MultiHook(Hook): return rep - def getConfig(self, option, default=''): + def getConfig(self, option, default=''): #@TODO: Remove in 0.4.10 """getConfig with default value - sublass may not implements all config options""" try: return self.getConf(option) @@ -119,17 +123,17 @@ class MultiHook(Hook): if self.plugins: return self.plugins - for _i in xrange(3): + for _i in xrange(2): try: pluginset = self._pluginSet(self.getHosters() if self.plugintype == "hoster" else self.getCrypters()) + break except Exception, e: - self.logError(e, "Waiting 1 minute and retry") - sleep(60) - - else: - break + self.logDebug(e, "Waiting 1 minute and retry") + time.sleep(60) else: + self.logWarning(_("Fallback to default reload interval due plugin parse error")) + self.interval = self.MIN_RELOAD_INTERVAL return list() try: @@ -152,17 +156,15 @@ class MultiHook(Hook): def _pluginSet(self, plugins): - plugins = set((decode(x).strip().lower() for x in plugins if '.' in x)) + regexp = re.compile(r'^[\w\-.^_]{3,63}\.[a-zA-Z]{2,}$', re.U) + plugins = [decode(p.strip()).lower() for p in plugins if regexp.match(p.strip())] - for rf, rt in self.DOMAIN_REPLACEMENTS: - regex = re.compile(rf) - for p in filter(lambda x: regex.match(x), plugins): - plugins.remove(p) - plugins.add(re.sub(rf, rt, p)) + for r in self.DOMAIN_REPLACEMENTS: + rf, rt = r + repr = re.compile(rf, re.I|re.U) + plugins = [re.sub(rf, rt, p) if repr.match(p) else p for p in plugins] - plugins.discard('') - - return plugins + return set(plugins) def getHosters(self): @@ -181,8 +183,28 @@ class MultiHook(Hook): raise NotImplementedError + #: Threaded _periodical, remove in 0.4.10 and use built-in flag for that + def _periodical(self): + try: + if self.isActivated(): + self.periodical() + + except Exception, e: + self.core.log.error(_("Error executing hooks: %s") % str(e)) + if self.core.debug: + print_exc() + + self.cb = self.core.scheduler.addJob(self.interval, self._periodical) + + def periodical(self): """reload plugin list periodically""" + if self.getConfig("reload", True): + self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_RELOAD_INTERVAL) + else: + self.core.scheduler.removeJob(self.cb) + self.cb = None + self.logInfo(_("Reloading supported %s list") % self.plugintype) old_supported = self.supported @@ -200,12 +222,6 @@ class MultiHook(Hook): for plugin in old_supported: self.unloadPlugin(plugin) - if self.getConfig("reload", True): - self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_INTERVAL) - else: - self.core.scheduler.removeJob(self.cb) - self.cb = None - def overridePlugins(self): excludedList = [] @@ -249,7 +265,7 @@ class MultiHook(Hook): self.logDebug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) # create new regexp - regexp = r'.*(?P<DOMAIN>%s).*' % "|".join([x.replace(".", "\.") for x in plugins]) + regexp = r'.*(?P<DOMAIN>%s).*' % "|".join(x.replace('.', '\.') for x in plugins) if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and '://' in self.pluginclass.__pattern__: regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) @@ -263,11 +279,11 @@ class MultiHook(Hook): def unloadPlugin(self, plugin): hdict = self.core.pluginManager.plugins[self.plugintype][plugin] if "module" in hdict: - del hdict['module'] + hdict.pop('module', None) if "new_module" in hdict: - del hdict['new_module'] - del hdict['new_name'] + hdict.pop('new_module', None) + hdict.pop('new_name', None) def deactivate(self): @@ -280,29 +296,3 @@ class MultiHook(Hook): hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') hdict['re'] = re.compile(hdict['pattern']) - - - def downloadFailed(self, pyfile): - """remove plugin override if download fails but not if file is offline/temp.offline""" - if pyfile.status != 8 or not self.getConfig("revertfailed", True): - return - - hdict = self.core.pluginManager.plugins[self.plugintype][pyfile.pluginname] - if "new_name" in hdict and hdict['new_name'] == self.pluginname: - if pyfile.error == "MultiHook": - self.logDebug("Unload MultiHook", pyfile.pluginname, hdict) - self.unloadPlugin(pyfile.pluginname) - pyfile.setStatus("queued") - pyfile.sync() - else: - retries = max(self.getConfig("retry", 10), 0) - wait_time = max(self.getConfig("retryinterval", 1), 0) - - if 0 < retries > pyfile.plugin.retries: - self.logInfo(_("Retrying: %s") % pyfile.name) - pyfile.setCustomStatus("MultiHook", "queued") - pyfile.sync() - - pyfile.plugin.retries += 1 - pyfile.plugin.setWait(wait_time) - pyfile.plugin.wait() diff --git a/pyload/plugin/internal/MultiHoster.py b/pyload/plugin/internal/MultiHoster.py index ed425ffaa..036570805 100644 --- a/pyload/plugin/internal/MultiHoster.py +++ b/pyload/plugin/internal/MultiHoster.py @@ -2,15 +2,18 @@ import re +from pyload.plugin.Plugin import Fail, Retry from pyload.plugin.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.37" + __version__ = "0.39" __pattern__ = r'^unmatchable$' + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("revertfailed", "bool", "Revert to standard download if fails", True)] __description__ = """Multi hoster plugin""" __license__ = "GPLv3" @@ -32,6 +35,9 @@ class MultiHoster(SimpleHoster): self.link = "" #@TODO: Move to hoster class in 0.4.10 self.directDL = False #@TODO: Move to hoster class in 0.4.10 + if not self.getConfig('use_premium', True): + self.retryFree() + if self.LOGIN_ACCOUNT and not self.account: self.fail(_("Required account not found")) @@ -49,29 +55,54 @@ class MultiHoster(SimpleHoster): def process(self, pyfile): - self.prepare() + try: + self.prepare() + + if self.directDL: + self.checkInfo() + self.logDebug("Looking for direct download link...") + self.handleDirect(pyfile) + + if not self.link and not self.lastDownload: + self.preload() + + self.checkErrors() + self.checkStatus(getinfo=False) + + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as premium download") + self.handlePremium(pyfile) + + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as free download") + self.handleFree(pyfile) + + self.downloadLink(self.link, True) + self.checkFile() + + except Fail, e: #@TODO: Move to PluginThread in 0.4.10 + if self.premium: + self.logWarning(_("Premium download failed")) + self.retryFree() - if self.directDL: - self.checkInfo() - self.logDebug("Looking for direct download link...") - self.handleDirect(pyfile) + elif self.getConfig("revertfailed", True) \ + and "new_module" in self.core.pluginManager.hosterPlugins[self.__name__]: + hdict = self.core.pluginManager.hosterPlugins[self.__name__] - if not self.link and not self.lastDownload: - self.preload() + tmp_module = hdict['new_module'] + tmp_name = hdict['new_name'] + hdict.pop('new_module', None) + hdict.pop('new_name', None) - self.checkErrors() - self.checkStatus(getinfo=False) + pyfile.initPlugin() - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as premium download") - self.handlePremium(pyfile) + hdict['new_module'] = tmp_module + hdict['new_name'] = tmp_name - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as free download") - self.handleFree(pyfile) + raise Retry(_("Revert to original hoster plugin")) - self.downloadLink(self.link, True) - self.checkFile() + else: + raise Fail(e) def handlePremium(self, pyfile): diff --git a/pyload/plugin/internal/SimpleCrypter.py b/pyload/plugin/internal/SimpleCrypter.py index e4b8874f3..472488268 100644 --- a/pyload/plugin/internal/SimpleCrypter.py +++ b/pyload/plugin/internal/SimpleCrypter.py @@ -15,14 +15,12 @@ class SimpleCrypter(Crypter, SimpleHoster): __version__ = "0.43" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package'] + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Simple decrypter plugin""" __license__ = "GPLv3" - __authors__ = [("stickell", "l.stickell@yahoo.it"), - ("zoidberg", "zoidberg@mujmail.cz"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com" )] """ diff --git a/pyload/plugin/internal/SimpleDereferer.py b/pyload/plugin/internal/SimpleDereferer.py index 6d323b4b0..e24a7b836 100644 --- a/pyload/plugin/internal/SimpleDereferer.py +++ b/pyload/plugin/internal/SimpleDereferer.py @@ -5,17 +5,17 @@ import re from urllib import unquote from pyload.plugin.Crypter import Crypter -from pyload.plugin.internal.SimpleHoster import fileUrl, set_cookies +from pyload.plugin.internal.SimpleHoster import getFileURL, set_cookies class SimpleDereferer(Crypter): __name__ = "SimpleDereferer" __type__ = "crypter" - __version__ = "0.07" + __version__ = "0.08" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), + ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Simple dereferer plugin""" __license__ = "GPLv3" @@ -45,7 +45,7 @@ class SimpleDereferer(Crypter): def decrypt(self, pyfile): - link = fileUrl(self, pyfile.url) + link = getFileURL(self, pyfile.url) if not link: try: diff --git a/pyload/plugin/internal/SimpleHoster.py b/pyload/plugin/internal/SimpleHoster.py index ac7f5aa4d..75f54c767 100644 --- a/pyload/plugin/internal/SimpleHoster.py +++ b/pyload/plugin/internal/SimpleHoster.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- +import datetime import mimetypes import os import re +import time -from datetime import datetime, timedelta from inspect import isclass -from time import time from urllib import unquote from urlparse import urljoin, urlparse @@ -15,7 +15,7 @@ from pyload.network.CookieJar import CookieJar from pyload.network.HTTPRequest import BadHeader from pyload.network.RequestFactory import getURL from pyload.plugin.Hoster import Hoster -from pyload.plugin.Plugin import Fail +from pyload.plugin.Plugin import Fail, Retry from pyload.utils import fixup, fs_encode, parseFileSize @@ -137,27 +137,27 @@ def parseFileInfo(plugin, url="", html=""): def timestamp(): - return int(time() * 1000) + return int(time.time() * 1000) #@TODO: Move to hoster class in 0.4.10 -def fileUrl(self, url, follow_location=None): +def getFileURL(self, url, follow_location=None): link = "" redirect = 1 if type(follow_location) is int: redirect = max(follow_location, 1) else: - redirect = 5 + redirect = 10 for i in xrange(redirect): try: self.logDebug("Redirect #%d to: %s" % (i, url)) - header = self.load(url, ref=True, cookies=True, just_header=True, decode=True) + header = self.load(url, just_header=True, decode=True) except Exception: #: Bad bad bad... req = pyreq.getHTTPRequest() - res = req.load(url, cookies=True, just_header=True, decode=True) + res = req.load(url, just_header=True, decode=True) req.close() @@ -226,18 +226,18 @@ def fileUrl(self, url, follow_location=None): def secondsToMidnight(gmt=0): - now = datetime.utcnow() + timedelta(hours=gmt) + now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) if now.hour is 0 and now.minute < 10: midnight = now else: - midnight = now + timedelta(days=1) + midnight = now + datetime.timedelta(days=1) td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now if hasattr(td, 'total_seconds'): res = td.total_seconds() - else: #@NOTE: work-around for python 2.5 and 2.6 missing timedelta.total_seconds + else: #: work-around for python 2.5 and 2.6 missing datetime.timedelta.total_seconds res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 return int(res) @@ -246,15 +246,14 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.15" + __version__ = "1.31" __pattern__ = r'^unmatchable$' + __config__ = [("use_premium", "bool", "Use premium account if available", True)] __description__ = """Simple hoster plugin""" __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("stickell", "l.stickell@yahoo.it"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com" )] """ @@ -311,7 +310,7 @@ class SimpleHoster(Hoster): LOGIN_ACCOUNT = False #: Set to True to require account login DISPOSITION = True #: Work-around to `filename*=UTF-8` bug; remove in 0.4.10 - directLink = fileUrl #@TODO: Remove in 0.4.10 + directLink = getFileURL #@TODO: Remove in 0.4.10 @classmethod @@ -349,7 +348,7 @@ class SimpleHoster(Hoster): info['error'] = "missing url" info['status'] = 1 - elif info['status'] is 3 and not fileUrl(None, url): + elif info['status'] is 3 and not getFileURL(None, url): try: html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING) @@ -426,6 +425,9 @@ class SimpleHoster(Hoster): self.directDL = False #@TODO: Move to hoster class in 0.4.10 self.multihost = False #@TODO: Move to hoster class in 0.4.10 + if not self.getConfig('use_premium', True): + self.retryFree() + if self.LOGIN_ACCOUNT and not self.account: self.fail(_("Required account not found")) @@ -456,35 +458,43 @@ class SimpleHoster(Hoster): def process(self, pyfile): - self.prepare() - self.checkInfo() + try: + self.prepare() + self.checkInfo() - if self.directDL: - self.logDebug("Looking for direct download link...") - self.handleDirect(pyfile) + if self.directDL: + self.logDebug("Looking for direct download link...") + self.handleDirect(pyfile) - if self.multihost and not self.link and not self.lastDownload: - self.logDebug("Looking for leeched download link...") - self.handleMulti(pyfile) + if self.multihost and not self.link and not self.lastDownload: + self.logDebug("Looking for leeched download link...") + self.handleMulti(pyfile) + + if not self.link and not self.lastDownload: + self.MULTI_HOSTER = False + self.retry(1, reason="Multi hoster fails") if not self.link and not self.lastDownload: - self.MULTI_HOSTER = False - self.retry(1, reason="Multi hoster fails") + self.preload() + self.checkInfo() - if not self.link and not self.lastDownload: - self.preload() - self.checkInfo() + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as premium download") + self.handlePremium(pyfile) - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as premium download") - self.handlePremium(pyfile) + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as free download") + self.handleFree(pyfile) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as free download") - self.handleFree(pyfile) + self.downloadLink(self.link, self.DISPOSITION) #: Remove `self.DISPOSITION` in 0.4.10 + self.checkFile() - self.downloadLink(self.link, self.DISPOSITION) #: Remove `self.DISPOSITION` in 0.4.10 - self.checkFile() + except Fail, e: #@TODO: Move to PluginThread in 0.4.10 + if self.premium: + self.logWarning(_("Premium download failed")) + self.retryFree() + else: + raise Fail(e) def downloadLink(self, link, disposition=True): @@ -499,7 +509,7 @@ class SimpleHoster(Hoster): self.download(link, ref=False, disposition=disposition) - def checkFile(self): + def checkFile(self, rules={}): if self.cTask and not self.lastDownload: self.invalidCaptcha() self.retry(10, reason=_("Wrong captcha")) @@ -509,21 +519,35 @@ class SimpleHoster(Hoster): self.error(self.pyfile.error or _("No file downloaded")) else: - rules = {'empty file': re.compile(r'\A\Z'), - 'html file' : re.compile(r'\A\s*<!DOCTYPE html'), - 'html error': re.compile(r'\A\s*(<.+>)?\d{3}(\Z|\s+)')} + errmsg = self.checkDownload({'Empty file': re.compile(r'\A\s*\Z'), + 'Html error': re.compile(r'\A(?:\s*<.+>)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)')}) - if hasattr(self, 'ERROR_PATTERN'): - rules['error'] = re.compile(self.ERROR_PATTERN) + if not errmsg: + for r, p in [('Html file' , re.compile(r'\A\s*<!DOCTYPE html') ), + ('Request error', re.compile(r'([Aa]n error occured while processing your request)'))]: + if r not in rules: + rules[r] = p - check = self.checkDownload(rules) - if check: #@TODO: Move to hoster in 0.4.10 - errmsg = check.strip().capitalize() - if self.lastCheck: - errmsg += " | " + self.lastCheck.group(0).strip() + for r, a in [('Error' , "ERROR_PATTERN" ), + ('Premium only', "PREMIUM_ONLY_PATTERN"), + ('Wait error' , "WAIT_PATTERN" )]: + if r not in rules and hasattr(self, a): + rules[r] = getattr(self, a) - self.lastDownload = "" - self.retry(10, 60, errmsg) + errmsg = self.checkDownload(rules) + + if not errmsg: + return + + errmsg = errmsg.strip().capitalize() + + try: + errmsg += " | " + self.lastCheck.group(1).strip() + except Exception: + pass + + self.logWarning("Check result: " + errmsg, "Waiting 1 minute and retry") + self.retry(3, 60, errmsg) def checkErrors(self): @@ -537,16 +561,36 @@ class SimpleHoster(Hoster): elif hasattr(self, 'ERROR_PATTERN'): m = re.search(self.ERROR_PATTERN, self.html) if m: - errmsg = self.info['error'] = m.group(1) - self.error(errmsg) + try: + errmsg = m.group(1).strip() + except Exception: + errmsg = m.group(0).strip() + + self.info['error'] = errmsg + + if "hour" in errmsg: + self.wait(1 * 60 * 60, True) + + elif re.search("da(il)?y|today", errmsg): + self.wait(secondsToMidnight(gmt=2), True) + + elif "minute" in errmsg: + self.wait(1 * 60) + + else: + self.error(errmsg) elif hasattr(self, 'WAIT_PATTERN'): m = re.search(self.WAIT_PATTERN, self.html) if m: + try: + waitmsg = m.group(1).strip() + except Exception: + waitmsg = m.group(0).strip() + wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I)) + re.findall(r'(\d+)\s*(hr|hour|min|sec)', waitmsg, re.I)) self.wait(wait_time, wait_time > 300) - return self.info.pop('error', None) @@ -692,6 +736,25 @@ class SimpleHoster(Hoster): return size <= traffic + def getConfig(self, option, default=''): #@TODO: Remove in 0.4.10 + """getConfig with default value - sublass may not implements all config options""" + try: + return self.getConf(option) + + except KeyError: + return default + + + def retryFree(self): + if not self.premium: + return + self.premium = False + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retries = 0 + raise Retry(_("Fallback to free download")) + + #@TODO: Remove in 0.4.10 def wait(self, seconds=0, reconnect=None): return _wait(self, seconds, reconnect) diff --git a/pyload/plugin/internal/XFSAccount.py b/pyload/plugin/internal/XFSAccount.py index 2e6b7dc50..1f2d2b180 100644 --- a/pyload/plugin/internal/XFSAccount.py +++ b/pyload/plugin/internal/XFSAccount.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import re +import time -from time import gmtime, mktime, strptime from urlparse import urljoin from pyload.plugin.Account import Account @@ -16,8 +16,8 @@ class XFSAccount(Account): __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("zoidberg" , "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com" )] HOSTER_DOMAIN = None @@ -75,7 +75,7 @@ class XFSAccount(Account): self.logDebug("Expire date: " + expiredate) try: - validuntil = mktime(strptime(expiredate, "%d %B %Y")) + validuntil = time.mktime(time.strptime(expiredate, "%d %B %Y")) except Exception, e: self.logError(e) @@ -83,7 +83,7 @@ class XFSAccount(Account): else: self.logDebug("Valid until: %s" % validuntil) - if validuntil > mktime(gmtime()): + if validuntil > time.mktime(time.gmtime()): premium = True trafficleft = -1 else: diff --git a/pyload/plugin/internal/XFSHoster.py b/pyload/plugin/internal/XFSHoster.py index b0a5aff0f..fc48d6229 100644 --- a/pyload/plugin/internal/XFSHoster.py +++ b/pyload/plugin/internal/XFSHoster.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- import re +import time from random import random -from time import sleep from urlparse import urljoin, urlparse from pyload.plugin.internal.captcha import ReCaptcha, SolveMedia @@ -14,15 +14,15 @@ from pyload.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.44" + __version__ = "0.45" __pattern__ = r'^unmatchable$' __description__ = """XFileSharing hoster plugin""" __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("stickell", "l.stickell@yahoo.it"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("zoidberg" , "zoidberg@mujmail.cz"), + ("stickell" , "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com" )] HOSTER_DOMAIN = None @@ -99,7 +99,7 @@ class XFSHoster(SimpleHoster): data = self.getPostParameters() - self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True, follow_location=False) + self.html = self.load(pyfile.url, post=data, ref=True, decode=True, follow_location=False) m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) if m and not "op=" in m.group(1): @@ -189,7 +189,7 @@ class XFSHoster(SimpleHoster): if 'wait' in self.errmsg: wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec)', self.errmsg, re.I)) - self.wait(wait_time, True) + self.wait(wait_time, wait_time > 300) elif 'country' in self.errmsg: self.fail(_("Downloads are disabled for your country")) |