summaryrefslogtreecommitdiffstats
path: root/pyload/plugin/internal
diff options
context:
space:
mode:
authorGravatar Walter Purcaro <vuolter@users.noreply.github.com> 2015-03-26 17:04:38 +0100
committerGravatar Walter Purcaro <vuolter@users.noreply.github.com> 2015-03-26 17:04:38 +0100
commitccb6aaee9d987b56faf19fb48dd06a04ff8ca721 (patch)
tree6a385417ff77f3ed17329e2d606f80b673bdb3c0 /pyload/plugin/internal
parentPrepare to merging (diff)
parentTiny code cosmetics (diff)
downloadpyload-ccb6aaee9d987b56faf19fb48dd06a04ff8ca721.tar.xz
Merge branch 'stable' into 0.4.10
Conflicts: module/plugins/accounts/Keep2ShareCc.py module/plugins/captcha/OCR.py module/plugins/captcha/captcha.py module/plugins/crypter/DailymotionBatch.py module/plugins/crypter/DailymotionComFolder.py module/plugins/crypter/YoutubeBatch.py module/plugins/crypter/YoutubeComFolder.py module/plugins/internal/CaptchaService.py pyload/plugin/Extractor.py pyload/plugin/OCR.py pyload/plugin/account/DebridItaliaCom.py pyload/plugin/account/MegaRapidCz.py pyload/plugin/account/NoPremiumPl.py pyload/plugin/account/RapideoPl.py pyload/plugin/addon/AndroidPhoneNotify.py pyload/plugin/addon/Checksum.py pyload/plugin/addon/ClickAndLoad.py pyload/plugin/addon/DeleteFinished.py pyload/plugin/addon/DownloadScheduler.py pyload/plugin/addon/ExternalScripts.py pyload/plugin/addon/ExtractArchive.py pyload/plugin/addon/HotFolder.py pyload/plugin/addon/IRCInterface.py pyload/plugin/addon/JustPremium.py pyload/plugin/addon/MergeFiles.py pyload/plugin/addon/MultiHome.py pyload/plugin/addon/RestartFailed.py pyload/plugin/addon/RestartSlow.py pyload/plugin/addon/SkipRev.py pyload/plugin/addon/UnSkipOnFail.py pyload/plugin/addon/UpdateManager.py pyload/plugin/addon/WindowsPhoneNotify.py pyload/plugin/crypter/DDLMusicOrg.py pyload/plugin/crypter/DailymotionBatch.py pyload/plugin/crypter/DevhostSt.py pyload/plugin/crypter/EmbeduploadCom.py pyload/plugin/crypter/MultiloadCz.py pyload/plugin/crypter/YoutubeBatch.py pyload/plugin/extractor/SevenZip.py pyload/plugin/extractor/UnRar.py pyload/plugin/extractor/UnZip.py pyload/plugin/hook/BypassCaptcha.py pyload/plugin/hook/Captcha9Kw.py pyload/plugin/hook/CaptchaBrotherhood.py pyload/plugin/hook/DeathByCaptcha.py pyload/plugin/hook/ExpertDecoders.py pyload/plugin/hook/ImageTyperz.py pyload/plugin/hook/XFileSharingPro.py pyload/plugin/hoster/AlldebridCom.py pyload/plugin/hoster/CzshareCom.py pyload/plugin/hoster/EuroshareEu.py pyload/plugin/hoster/FastixRu.py pyload/plugin/hoster/FastshareCz.py pyload/plugin/hoster/GooIm.py pyload/plugin/hoster/MediafireCom.py pyload/plugin/hoster/MegaDebridEu.py pyload/plugin/hoster/NitroflareCom.py pyload/plugin/hoster/OverLoadMe.py pyload/plugin/hoster/PremiumTo.py pyload/plugin/hoster/PremiumizeMe.py pyload/plugin/hoster/RapidgatorNet.py pyload/plugin/hoster/RealdebridCom.py pyload/plugin/hoster/SimplyPremiumCom.py pyload/plugin/hoster/SimplydebridCom.py pyload/plugin/hoster/SmoozedCom.py pyload/plugin/hoster/SoundcloudCom.py pyload/plugin/hoster/UlozTo.py pyload/plugin/hoster/UploadableCh.py pyload/plugin/hoster/UploadedTo.py pyload/plugin/hoster/UploadheroCom.py pyload/plugin/hoster/VeehdCom.py pyload/plugin/hoster/VimeoCom.py pyload/plugin/hoster/ZeveraCom.py pyload/plugin/hoster/ZippyshareCom.py pyload/plugin/internal/BasePlugin.py pyload/plugin/internal/MultiHoster.py pyload/plugin/internal/SimpleDereferer.py pyload/plugin/internal/SimpleHoster.py pyload/plugin/internal/XFSHoster.py pyload/plugin/ocr/GigasizeCom.py pyload/plugin/ocr/LinksaveIn.py pyload/plugin/ocr/NetloadIn.py pyload/plugin/ocr/ShareonlineBiz.py
Diffstat (limited to 'pyload/plugin/internal')
-rw-r--r--pyload/plugin/internal/BasePlugin.py28
-rw-r--r--pyload/plugin/internal/MultiHook.py126
-rw-r--r--pyload/plugin/internal/MultiHoster.py67
-rw-r--r--pyload/plugin/internal/SimpleCrypter.py8
-rw-r--r--pyload/plugin/internal/SimpleDereferer.py10
-rw-r--r--pyload/plugin/internal/SimpleHoster.py173
-rw-r--r--pyload/plugin/internal/XFSAccount.py10
-rw-r--r--pyload/plugin/internal/XFSHoster.py14
8 files changed, 265 insertions, 171 deletions
diff --git a/pyload/plugin/internal/BasePlugin.py b/pyload/plugin/internal/BasePlugin.py
index 103e0d5cb..ed4da72a1 100644
--- a/pyload/plugin/internal/BasePlugin.py
+++ b/pyload/plugin/internal/BasePlugin.py
@@ -6,14 +6,14 @@ from urllib import unquote
from urlparse import urljoin, urlparse
from pyload.network.HTTPRequest import BadHeader
-from pyload.plugin.internal.SimpleHoster import fileUrl
+from pyload.plugin.internal.SimpleHoster import create_getInfo, getFileURL
from pyload.plugin.Hoster import Hoster
class BasePlugin(Hoster):
__name__ = "BasePlugin"
__type__ = "hoster"
- __version__ = "0.34"
+ __version__ = "0.38"
__pattern__ = r'^unmatchable$'
@@ -51,7 +51,7 @@ class BasePlugin(Hoster):
for _i in xrange(5):
try:
- link = fileUrl(self, unquote(pyfile.url))
+ link = getFileURL(self, unquote(pyfile.url))
if link:
self.download(link, ref=False, disposition=True)
@@ -85,8 +85,20 @@ class BasePlugin(Hoster):
else:
self.fail(_("No file downloaded")) #@TODO: Move to hoster class in 0.4.10
- check = self.checkDownload({'empty file': re.compile(r'\A\Z'),
- 'html file' : re.compile(r'\A\s*<!DOCTYPE html'),
- 'html error': re.compile(r'\A\s*(<.+>)?\d{3}(\Z|\s+)')})
- if check:
- self.fail(check.capitalize())
+ errmsg = self.checkDownload({'Empty file' : re.compile(r'\A\s*\Z'),
+ 'Html error' : re.compile(r'\A(?:\s*<.+>)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)'),
+ 'Html file' : re.compile(r'\A\s*<!DOCTYPE html'),
+ 'Request error': re.compile(r'([Aa]n error occured while processing your request)')})
+ if not errmsg:
+ return
+
+ try:
+ errmsg += " | " + self.lastCheck.group(1).strip()
+ except Exception:
+ pass
+
+ self.logWarning("Check result: " + errmsg, "Waiting 1 minute and retry")
+ self.retry(3, 60, errmsg)
+
+
+getInfo = create_getInfo(BasePlugin)
diff --git a/pyload/plugin/internal/MultiHook.py b/pyload/plugin/internal/MultiHook.py
index 2beccfcc5..d43956691 100644
--- a/pyload/plugin/internal/MultiHook.py
+++ b/pyload/plugin/internal/MultiHook.py
@@ -1,8 +1,7 @@
# -*- coding: utf-8 -*-
import re
-
-from time import sleep
+import time
from pyload.plugin.Hook import Hook
from pyload.utils import decode, remove_chars
@@ -11,28 +10,22 @@ from pyload.utils import decode, remove_chars
class MultiHook(Hook):
__name__ = "MultiHook"
__type__ = "hook"
- __version__ = "0.37"
+ __version__ = "0.40"
- __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"),
- ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ),
- ("revertfailed" , "bool" , "Revert to standard download if fails", True ),
- ("retry" , "int" , "Number of retries before revert" , 10 ),
- ("retryinterval" , "int" , "Retry interval in minutes" , 1 ),
- ("reload" , "bool" , "Reload plugin list" , True ),
- ("reloadinterval", "int" , "Reload interval in hours" , 12 )]
+ __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"),
+ ("pluginlist" , "str" , "Plugin list (comma separated)", "" ),
+ ("reload" , "bool" , "Reload plugin list" , True ),
+ ("reloadinterval", "int" , "Reload interval in hours" , 12 )]
__description__ = """Hook plugin for multi hoster/crypter"""
__license__ = "GPLv3"
- __authors__ = [("pyLoad Team", "admin@pyload.org"),
+ __authors__ = [("pyLoad Team" , "admin@pyload.org" ),
("Walter Purcaro", "vuolter@gmail.com")]
- MIN_INTERVAL = 1 * 60 * 60
+ MIN_RELOAD_INTERVAL = 1 * 60 * 60 #: 1 hour
DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"),
- (r'1fichier\.com' , "onefichier.com" ),
- (r'2shared\.com' , "twoshared.com" ),
- (r'4shared\.com' , "fourshared.com" ),
(r'bayfiles\.net' , "bayfiles.com" ),
(r'cloudnator\.com' , "shragle.com" ),
(r'dfiles\.eu' , "depositfiles.com" ),
@@ -48,10 +41,21 @@ class MultiHook(Hook):
(r'uploaded\.net' , "uploaded.to" ),
(r'uploadhero\.co' , "uploadhero.com" ),
(r'zshares\.net' , "zshare.net" ),
- (r'(\d+.+)' , "X\1" )]
+ (r'^1' , "one" ),
+ (r'^2' , "two" ),
+ (r'^3' , "three" ),
+ (r'^4' , "four" ),
+ (r'^5' , "five" ),
+ (r'^6' , "six" ),
+ (r'^7' , "seven" ),
+ (r'^8' , "eight" ),
+ (r'^9' , "nine" ),
+ (r'^0' , "zero" )]
def setup(self):
+ self.info = {} #@TODO: Remove in 0.4.10
+
self.plugins = []
self.supported = []
self.new_supported = []
@@ -106,7 +110,7 @@ class MultiHook(Hook):
return rep
- def getConfig(self, option, default=''):
+ def getConfig(self, option, default=''): #@TODO: Remove in 0.4.10
"""getConfig with default value - sublass may not implements all config options"""
try:
return self.getConf(option)
@@ -119,17 +123,17 @@ class MultiHook(Hook):
if self.plugins:
return self.plugins
- for _i in xrange(3):
+ for _i in xrange(2):
try:
pluginset = self._pluginSet(self.getHosters() if self.plugintype == "hoster" else self.getCrypters())
+ break
except Exception, e:
- self.logError(e, "Waiting 1 minute and retry")
- sleep(60)
-
- else:
- break
+ self.logDebug(e, "Waiting 1 minute and retry")
+ time.sleep(60)
else:
+ self.logWarning(_("Fallback to default reload interval due plugin parse error"))
+ self.interval = self.MIN_RELOAD_INTERVAL
return list()
try:
@@ -152,17 +156,15 @@ class MultiHook(Hook):
def _pluginSet(self, plugins):
- plugins = set((decode(x).strip().lower() for x in plugins if '.' in x))
+ regexp = re.compile(r'^[\w\-.^_]{3,63}\.[a-zA-Z]{2,}$', re.U)
+ plugins = [decode(p.strip()).lower() for p in plugins if regexp.match(p.strip())]
- for rf, rt in self.DOMAIN_REPLACEMENTS:
- regex = re.compile(rf)
- for p in filter(lambda x: regex.match(x), plugins):
- plugins.remove(p)
- plugins.add(re.sub(rf, rt, p))
+ for r in self.DOMAIN_REPLACEMENTS:
+ rf, rt = r
+ repr = re.compile(rf, re.I|re.U)
+ plugins = [re.sub(rf, rt, p) if repr.match(p) else p for p in plugins]
- plugins.discard('')
-
- return plugins
+ return set(plugins)
def getHosters(self):
@@ -181,8 +183,28 @@ class MultiHook(Hook):
raise NotImplementedError
+ #: Threaded _periodical, remove in 0.4.10 and use built-in flag for that
+ def _periodical(self):
+ try:
+ if self.isActivated():
+ self.periodical()
+
+ except Exception, e:
+ self.core.log.error(_("Error executing hooks: %s") % str(e))
+ if self.core.debug:
+ print_exc()
+
+ self.cb = self.core.scheduler.addJob(self.interval, self._periodical)
+
+
def periodical(self):
"""reload plugin list periodically"""
+ if self.getConfig("reload", True):
+ self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_RELOAD_INTERVAL)
+ else:
+ self.core.scheduler.removeJob(self.cb)
+ self.cb = None
+
self.logInfo(_("Reloading supported %s list") % self.plugintype)
old_supported = self.supported
@@ -200,12 +222,6 @@ class MultiHook(Hook):
for plugin in old_supported:
self.unloadPlugin(plugin)
- if self.getConfig("reload", True):
- self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_INTERVAL)
- else:
- self.core.scheduler.removeJob(self.cb)
- self.cb = None
-
def overridePlugins(self):
excludedList = []
@@ -249,7 +265,7 @@ class MultiHook(Hook):
self.logDebug("New %ss: %s" % (self.plugintype, ", ".join(plugins)))
# create new regexp
- regexp = r'.*(?P<DOMAIN>%s).*' % "|".join([x.replace(".", "\.") for x in plugins])
+ regexp = r'.*(?P<DOMAIN>%s).*' % "|".join(x.replace('.', '\.') for x in plugins)
if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and '://' in self.pluginclass.__pattern__:
regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp)
@@ -263,11 +279,11 @@ class MultiHook(Hook):
def unloadPlugin(self, plugin):
hdict = self.core.pluginManager.plugins[self.plugintype][plugin]
if "module" in hdict:
- del hdict['module']
+ hdict.pop('module', None)
if "new_module" in hdict:
- del hdict['new_module']
- del hdict['new_name']
+ hdict.pop('new_module', None)
+ hdict.pop('new_name', None)
def deactivate(self):
@@ -280,29 +296,3 @@ class MultiHook(Hook):
hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$')
hdict['re'] = re.compile(hdict['pattern'])
-
-
- def downloadFailed(self, pyfile):
- """remove plugin override if download fails but not if file is offline/temp.offline"""
- if pyfile.status != 8 or not self.getConfig("revertfailed", True):
- return
-
- hdict = self.core.pluginManager.plugins[self.plugintype][pyfile.pluginname]
- if "new_name" in hdict and hdict['new_name'] == self.pluginname:
- if pyfile.error == "MultiHook":
- self.logDebug("Unload MultiHook", pyfile.pluginname, hdict)
- self.unloadPlugin(pyfile.pluginname)
- pyfile.setStatus("queued")
- pyfile.sync()
- else:
- retries = max(self.getConfig("retry", 10), 0)
- wait_time = max(self.getConfig("retryinterval", 1), 0)
-
- if 0 < retries > pyfile.plugin.retries:
- self.logInfo(_("Retrying: %s") % pyfile.name)
- pyfile.setCustomStatus("MultiHook", "queued")
- pyfile.sync()
-
- pyfile.plugin.retries += 1
- pyfile.plugin.setWait(wait_time)
- pyfile.plugin.wait()
diff --git a/pyload/plugin/internal/MultiHoster.py b/pyload/plugin/internal/MultiHoster.py
index ed425ffaa..036570805 100644
--- a/pyload/plugin/internal/MultiHoster.py
+++ b/pyload/plugin/internal/MultiHoster.py
@@ -2,15 +2,18 @@
import re
+from pyload.plugin.Plugin import Fail, Retry
from pyload.plugin.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies
class MultiHoster(SimpleHoster):
__name__ = "MultiHoster"
__type__ = "hoster"
- __version__ = "0.37"
+ __version__ = "0.39"
__pattern__ = r'^unmatchable$'
+ __config__ = [("use_premium" , "bool", "Use premium account if available" , True),
+ ("revertfailed", "bool", "Revert to standard download if fails", True)]
__description__ = """Multi hoster plugin"""
__license__ = "GPLv3"
@@ -32,6 +35,9 @@ class MultiHoster(SimpleHoster):
self.link = "" #@TODO: Move to hoster class in 0.4.10
self.directDL = False #@TODO: Move to hoster class in 0.4.10
+ if not self.getConfig('use_premium', True):
+ self.retryFree()
+
if self.LOGIN_ACCOUNT and not self.account:
self.fail(_("Required account not found"))
@@ -49,29 +55,54 @@ class MultiHoster(SimpleHoster):
def process(self, pyfile):
- self.prepare()
+ try:
+ self.prepare()
+
+ if self.directDL:
+ self.checkInfo()
+ self.logDebug("Looking for direct download link...")
+ self.handleDirect(pyfile)
+
+ if not self.link and not self.lastDownload:
+ self.preload()
+
+ self.checkErrors()
+ self.checkStatus(getinfo=False)
+
+ if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
+ self.logDebug("Handled as premium download")
+ self.handlePremium(pyfile)
+
+ elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
+ self.logDebug("Handled as free download")
+ self.handleFree(pyfile)
+
+ self.downloadLink(self.link, True)
+ self.checkFile()
+
+ except Fail, e: #@TODO: Move to PluginThread in 0.4.10
+ if self.premium:
+ self.logWarning(_("Premium download failed"))
+ self.retryFree()
- if self.directDL:
- self.checkInfo()
- self.logDebug("Looking for direct download link...")
- self.handleDirect(pyfile)
+ elif self.getConfig("revertfailed", True) \
+ and "new_module" in self.core.pluginManager.hosterPlugins[self.__name__]:
+ hdict = self.core.pluginManager.hosterPlugins[self.__name__]
- if not self.link and not self.lastDownload:
- self.preload()
+ tmp_module = hdict['new_module']
+ tmp_name = hdict['new_name']
+ hdict.pop('new_module', None)
+ hdict.pop('new_name', None)
- self.checkErrors()
- self.checkStatus(getinfo=False)
+ pyfile.initPlugin()
- if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
- self.logDebug("Handled as premium download")
- self.handlePremium(pyfile)
+ hdict['new_module'] = tmp_module
+ hdict['new_name'] = tmp_name
- elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
- self.logDebug("Handled as free download")
- self.handleFree(pyfile)
+ raise Retry(_("Revert to original hoster plugin"))
- self.downloadLink(self.link, True)
- self.checkFile()
+ else:
+ raise Fail(e)
def handlePremium(self, pyfile):
diff --git a/pyload/plugin/internal/SimpleCrypter.py b/pyload/plugin/internal/SimpleCrypter.py
index e4b8874f3..472488268 100644
--- a/pyload/plugin/internal/SimpleCrypter.py
+++ b/pyload/plugin/internal/SimpleCrypter.py
@@ -15,14 +15,12 @@ class SimpleCrypter(Crypter, SimpleHoster):
__version__ = "0.43"
__pattern__ = r'^unmatchable$'
- __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package']
- ("subfolder_per_package", "bool", "Create a subfolder for each package", True)]
+ __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides core.config['general']['folder_per_package']
+ ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)]
__description__ = """Simple decrypter plugin"""
__license__ = "GPLv3"
- __authors__ = [("stickell", "l.stickell@yahoo.it"),
- ("zoidberg", "zoidberg@mujmail.cz"),
- ("Walter Purcaro", "vuolter@gmail.com")]
+ __authors__ = [("Walter Purcaro", "vuolter@gmail.com" )]
"""
diff --git a/pyload/plugin/internal/SimpleDereferer.py b/pyload/plugin/internal/SimpleDereferer.py
index 6d323b4b0..e24a7b836 100644
--- a/pyload/plugin/internal/SimpleDereferer.py
+++ b/pyload/plugin/internal/SimpleDereferer.py
@@ -5,17 +5,17 @@ import re
from urllib import unquote
from pyload.plugin.Crypter import Crypter
-from pyload.plugin.internal.SimpleHoster import fileUrl, set_cookies
+from pyload.plugin.internal.SimpleHoster import getFileURL, set_cookies
class SimpleDereferer(Crypter):
__name__ = "SimpleDereferer"
__type__ = "crypter"
- __version__ = "0.07"
+ __version__ = "0.08"
__pattern__ = r'^unmatchable$'
- __config__ = [("use_subfolder", "bool", "Save package to subfolder", True),
- ("subfolder_per_package", "bool", "Create a subfolder for each package", True)]
+ __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True),
+ ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)]
__description__ = """Simple dereferer plugin"""
__license__ = "GPLv3"
@@ -45,7 +45,7 @@ class SimpleDereferer(Crypter):
def decrypt(self, pyfile):
- link = fileUrl(self, pyfile.url)
+ link = getFileURL(self, pyfile.url)
if not link:
try:
diff --git a/pyload/plugin/internal/SimpleHoster.py b/pyload/plugin/internal/SimpleHoster.py
index ac7f5aa4d..75f54c767 100644
--- a/pyload/plugin/internal/SimpleHoster.py
+++ b/pyload/plugin/internal/SimpleHoster.py
@@ -1,12 +1,12 @@
# -*- coding: utf-8 -*-
+import datetime
import mimetypes
import os
import re
+import time
-from datetime import datetime, timedelta
from inspect import isclass
-from time import time
from urllib import unquote
from urlparse import urljoin, urlparse
@@ -15,7 +15,7 @@ from pyload.network.CookieJar import CookieJar
from pyload.network.HTTPRequest import BadHeader
from pyload.network.RequestFactory import getURL
from pyload.plugin.Hoster import Hoster
-from pyload.plugin.Plugin import Fail
+from pyload.plugin.Plugin import Fail, Retry
from pyload.utils import fixup, fs_encode, parseFileSize
@@ -137,27 +137,27 @@ def parseFileInfo(plugin, url="", html=""):
def timestamp():
- return int(time() * 1000)
+ return int(time.time() * 1000)
#@TODO: Move to hoster class in 0.4.10
-def fileUrl(self, url, follow_location=None):
+def getFileURL(self, url, follow_location=None):
link = ""
redirect = 1
if type(follow_location) is int:
redirect = max(follow_location, 1)
else:
- redirect = 5
+ redirect = 10
for i in xrange(redirect):
try:
self.logDebug("Redirect #%d to: %s" % (i, url))
- header = self.load(url, ref=True, cookies=True, just_header=True, decode=True)
+ header = self.load(url, just_header=True, decode=True)
except Exception: #: Bad bad bad...
req = pyreq.getHTTPRequest()
- res = req.load(url, cookies=True, just_header=True, decode=True)
+ res = req.load(url, just_header=True, decode=True)
req.close()
@@ -226,18 +226,18 @@ def fileUrl(self, url, follow_location=None):
def secondsToMidnight(gmt=0):
- now = datetime.utcnow() + timedelta(hours=gmt)
+ now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt)
if now.hour is 0 and now.minute < 10:
midnight = now
else:
- midnight = now + timedelta(days=1)
+ midnight = now + datetime.timedelta(days=1)
td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now
if hasattr(td, 'total_seconds'):
res = td.total_seconds()
- else: #@NOTE: work-around for python 2.5 and 2.6 missing timedelta.total_seconds
+ else: #: work-around for python 2.5 and 2.6 missing datetime.timedelta.total_seconds
res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
return int(res)
@@ -246,15 +246,14 @@ def secondsToMidnight(gmt=0):
class SimpleHoster(Hoster):
__name__ = "SimpleHoster"
__type__ = "hoster"
- __version__ = "1.15"
+ __version__ = "1.31"
__pattern__ = r'^unmatchable$'
+ __config__ = [("use_premium", "bool", "Use premium account if available", True)]
__description__ = """Simple hoster plugin"""
__license__ = "GPLv3"
- __authors__ = [("zoidberg", "zoidberg@mujmail.cz"),
- ("stickell", "l.stickell@yahoo.it"),
- ("Walter Purcaro", "vuolter@gmail.com")]
+ __authors__ = [("Walter Purcaro", "vuolter@gmail.com" )]
"""
@@ -311,7 +310,7 @@ class SimpleHoster(Hoster):
LOGIN_ACCOUNT = False #: Set to True to require account login
DISPOSITION = True #: Work-around to `filename*=UTF-8` bug; remove in 0.4.10
- directLink = fileUrl #@TODO: Remove in 0.4.10
+ directLink = getFileURL #@TODO: Remove in 0.4.10
@classmethod
@@ -349,7 +348,7 @@ class SimpleHoster(Hoster):
info['error'] = "missing url"
info['status'] = 1
- elif info['status'] is 3 and not fileUrl(None, url):
+ elif info['status'] is 3 and not getFileURL(None, url):
try:
html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING)
@@ -426,6 +425,9 @@ class SimpleHoster(Hoster):
self.directDL = False #@TODO: Move to hoster class in 0.4.10
self.multihost = False #@TODO: Move to hoster class in 0.4.10
+ if not self.getConfig('use_premium', True):
+ self.retryFree()
+
if self.LOGIN_ACCOUNT and not self.account:
self.fail(_("Required account not found"))
@@ -456,35 +458,43 @@ class SimpleHoster(Hoster):
def process(self, pyfile):
- self.prepare()
- self.checkInfo()
+ try:
+ self.prepare()
+ self.checkInfo()
- if self.directDL:
- self.logDebug("Looking for direct download link...")
- self.handleDirect(pyfile)
+ if self.directDL:
+ self.logDebug("Looking for direct download link...")
+ self.handleDirect(pyfile)
- if self.multihost and not self.link and not self.lastDownload:
- self.logDebug("Looking for leeched download link...")
- self.handleMulti(pyfile)
+ if self.multihost and not self.link and not self.lastDownload:
+ self.logDebug("Looking for leeched download link...")
+ self.handleMulti(pyfile)
+
+ if not self.link and not self.lastDownload:
+ self.MULTI_HOSTER = False
+ self.retry(1, reason="Multi hoster fails")
if not self.link and not self.lastDownload:
- self.MULTI_HOSTER = False
- self.retry(1, reason="Multi hoster fails")
+ self.preload()
+ self.checkInfo()
- if not self.link and not self.lastDownload:
- self.preload()
- self.checkInfo()
+ if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
+ self.logDebug("Handled as premium download")
+ self.handlePremium(pyfile)
- if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
- self.logDebug("Handled as premium download")
- self.handlePremium(pyfile)
+ elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
+ self.logDebug("Handled as free download")
+ self.handleFree(pyfile)
- elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()):
- self.logDebug("Handled as free download")
- self.handleFree(pyfile)
+ self.downloadLink(self.link, self.DISPOSITION) #: Remove `self.DISPOSITION` in 0.4.10
+ self.checkFile()
- self.downloadLink(self.link, self.DISPOSITION) #: Remove `self.DISPOSITION` in 0.4.10
- self.checkFile()
+ except Fail, e: #@TODO: Move to PluginThread in 0.4.10
+ if self.premium:
+ self.logWarning(_("Premium download failed"))
+ self.retryFree()
+ else:
+ raise Fail(e)
def downloadLink(self, link, disposition=True):
@@ -499,7 +509,7 @@ class SimpleHoster(Hoster):
self.download(link, ref=False, disposition=disposition)
- def checkFile(self):
+ def checkFile(self, rules={}):
if self.cTask and not self.lastDownload:
self.invalidCaptcha()
self.retry(10, reason=_("Wrong captcha"))
@@ -509,21 +519,35 @@ class SimpleHoster(Hoster):
self.error(self.pyfile.error or _("No file downloaded"))
else:
- rules = {'empty file': re.compile(r'\A\Z'),
- 'html file' : re.compile(r'\A\s*<!DOCTYPE html'),
- 'html error': re.compile(r'\A\s*(<.+>)?\d{3}(\Z|\s+)')}
+ errmsg = self.checkDownload({'Empty file': re.compile(r'\A\s*\Z'),
+ 'Html error': re.compile(r'\A(?:\s*<.+>)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)')})
- if hasattr(self, 'ERROR_PATTERN'):
- rules['error'] = re.compile(self.ERROR_PATTERN)
+ if not errmsg:
+ for r, p in [('Html file' , re.compile(r'\A\s*<!DOCTYPE html') ),
+ ('Request error', re.compile(r'([Aa]n error occured while processing your request)'))]:
+ if r not in rules:
+ rules[r] = p
- check = self.checkDownload(rules)
- if check: #@TODO: Move to hoster in 0.4.10
- errmsg = check.strip().capitalize()
- if self.lastCheck:
- errmsg += " | " + self.lastCheck.group(0).strip()
+ for r, a in [('Error' , "ERROR_PATTERN" ),
+ ('Premium only', "PREMIUM_ONLY_PATTERN"),
+ ('Wait error' , "WAIT_PATTERN" )]:
+ if r not in rules and hasattr(self, a):
+ rules[r] = getattr(self, a)
- self.lastDownload = ""
- self.retry(10, 60, errmsg)
+ errmsg = self.checkDownload(rules)
+
+ if not errmsg:
+ return
+
+ errmsg = errmsg.strip().capitalize()
+
+ try:
+ errmsg += " | " + self.lastCheck.group(1).strip()
+ except Exception:
+ pass
+
+ self.logWarning("Check result: " + errmsg, "Waiting 1 minute and retry")
+ self.retry(3, 60, errmsg)
def checkErrors(self):
@@ -537,16 +561,36 @@ class SimpleHoster(Hoster):
elif hasattr(self, 'ERROR_PATTERN'):
m = re.search(self.ERROR_PATTERN, self.html)
if m:
- errmsg = self.info['error'] = m.group(1)
- self.error(errmsg)
+ try:
+ errmsg = m.group(1).strip()
+ except Exception:
+ errmsg = m.group(0).strip()
+
+ self.info['error'] = errmsg
+
+ if "hour" in errmsg:
+ self.wait(1 * 60 * 60, True)
+
+ elif re.search("da(il)?y|today", errmsg):
+ self.wait(secondsToMidnight(gmt=2), True)
+
+ elif "minute" in errmsg:
+ self.wait(1 * 60)
+
+ else:
+ self.error(errmsg)
elif hasattr(self, 'WAIT_PATTERN'):
m = re.search(self.WAIT_PATTERN, self.html)
if m:
+ try:
+ waitmsg = m.group(1).strip()
+ except Exception:
+ waitmsg = m.group(0).strip()
+
wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in
- re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I))
+ re.findall(r'(\d+)\s*(hr|hour|min|sec)', waitmsg, re.I))
self.wait(wait_time, wait_time > 300)
- return
self.info.pop('error', None)
@@ -692,6 +736,25 @@ class SimpleHoster(Hoster):
return size <= traffic
+ def getConfig(self, option, default=''): #@TODO: Remove in 0.4.10
+ """getConfig with default value - sublass may not implements all config options"""
+ try:
+ return self.getConf(option)
+
+ except KeyError:
+ return default
+
+
+ def retryFree(self):
+ if not self.premium:
+ return
+ self.premium = False
+ self.account = None
+ self.req = self.core.requestFactory.getRequest(self.__name__)
+ self.retries = 0
+ raise Retry(_("Fallback to free download"))
+
+
#@TODO: Remove in 0.4.10
def wait(self, seconds=0, reconnect=None):
return _wait(self, seconds, reconnect)
diff --git a/pyload/plugin/internal/XFSAccount.py b/pyload/plugin/internal/XFSAccount.py
index 2e6b7dc50..1f2d2b180 100644
--- a/pyload/plugin/internal/XFSAccount.py
+++ b/pyload/plugin/internal/XFSAccount.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
import re
+import time
-from time import gmtime, mktime, strptime
from urlparse import urljoin
from pyload.plugin.Account import Account
@@ -16,8 +16,8 @@ class XFSAccount(Account):
__description__ = """XFileSharing account plugin"""
__license__ = "GPLv3"
- __authors__ = [("zoidberg", "zoidberg@mujmail.cz"),
- ("Walter Purcaro", "vuolter@gmail.com")]
+ __authors__ = [("zoidberg" , "zoidberg@mujmail.cz"),
+ ("Walter Purcaro", "vuolter@gmail.com" )]
HOSTER_DOMAIN = None
@@ -75,7 +75,7 @@ class XFSAccount(Account):
self.logDebug("Expire date: " + expiredate)
try:
- validuntil = mktime(strptime(expiredate, "%d %B %Y"))
+ validuntil = time.mktime(time.strptime(expiredate, "%d %B %Y"))
except Exception, e:
self.logError(e)
@@ -83,7 +83,7 @@ class XFSAccount(Account):
else:
self.logDebug("Valid until: %s" % validuntil)
- if validuntil > mktime(gmtime()):
+ if validuntil > time.mktime(time.gmtime()):
premium = True
trafficleft = -1
else:
diff --git a/pyload/plugin/internal/XFSHoster.py b/pyload/plugin/internal/XFSHoster.py
index b0a5aff0f..fc48d6229 100644
--- a/pyload/plugin/internal/XFSHoster.py
+++ b/pyload/plugin/internal/XFSHoster.py
@@ -1,9 +1,9 @@
# -*- coding: utf-8 -*-
import re
+import time
from random import random
-from time import sleep
from urlparse import urljoin, urlparse
from pyload.plugin.internal.captcha import ReCaptcha, SolveMedia
@@ -14,15 +14,15 @@ from pyload.utils import html_unescape
class XFSHoster(SimpleHoster):
__name__ = "XFSHoster"
__type__ = "hoster"
- __version__ = "0.44"
+ __version__ = "0.45"
__pattern__ = r'^unmatchable$'
__description__ = """XFileSharing hoster plugin"""
__license__ = "GPLv3"
- __authors__ = [("zoidberg", "zoidberg@mujmail.cz"),
- ("stickell", "l.stickell@yahoo.it"),
- ("Walter Purcaro", "vuolter@gmail.com")]
+ __authors__ = [("zoidberg" , "zoidberg@mujmail.cz"),
+ ("stickell" , "l.stickell@yahoo.it"),
+ ("Walter Purcaro", "vuolter@gmail.com" )]
HOSTER_DOMAIN = None
@@ -99,7 +99,7 @@ class XFSHoster(SimpleHoster):
data = self.getPostParameters()
- self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True, follow_location=False)
+ self.html = self.load(pyfile.url, post=data, ref=True, decode=True, follow_location=False)
m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I)
if m and not "op=" in m.group(1):
@@ -189,7 +189,7 @@ class XFSHoster(SimpleHoster):
if 'wait' in self.errmsg:
wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in
re.findall(r'(\d+)\s*(hr|hour|min|sec)', self.errmsg, re.I))
- self.wait(wait_time, True)
+ self.wait(wait_time, wait_time > 300)
elif 'country' in self.errmsg:
self.fail(_("Downloads are disabled for your country"))