diff options
| author |  GammaC0de <GammaC0de@users.noreply.github.com>
| 2015-05-29 23:33:10 +0200 |
|---|---|---|
| committer | GammaC0de <GammaC0de@users.noreply.github.com>
| 2015-05-29 23:33:10 +0200 |
| commit | 844dfd92f590e531ca2f7fd86305fcbc13a03721 (patch) | |
| tree | 5303bd07749b362dab071ada6197fe37dda85b27 /module/plugins/hoster/BasePlugin.py | |
| parent | [BitshareCom] Code cosmetics (diff) | |
| parent | [SimpleHoster] Fix DB error (diff) | |
| download | pyload-844dfd92f590e531ca2f7fd86305fcbc13a03721.tar.xz | |
Merge pull request #1 from pyload/stable
sync stable
Diffstat (limited to 'module/plugins/hoster/BasePlugin.py')
| -rw-r--r-- | module/plugins/hoster/BasePlugin.py | 73 |
1 files changed, 34 insertions, 39 deletions
diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index d0d8e7cc8..2228516aa 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -1,19 +1,18 @@ # -*- coding: utf-8 -*- import re - -from urllib import unquote -from urlparse import urljoin, urlparse +import urllib +import urlparse from module.network.HTTPRequest import BadHeader -from module.plugins.internal.SimpleHoster import create_getInfo +from module.plugins.internal.SimpleHoster import create_getInfo, getFileURL from module.plugins.Hoster import Hoster class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.26" + __version__ = "0.43" __pattern__ = r'^unmatchable$' @@ -25,11 +24,19 @@ class BasePlugin(Hoster): @classmethod def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10 - return {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3 if url else 1, 'url': unquote(url) or ""} + url = urllib.unquote(url) + url_p = urlparse.urlparse(url) + return {'name' : (url_p.path.split('/')[-1] + or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] + or url_p.netloc.split('.', 1)[0]), + 'size' : 0, + 'status': 3 if url else 8, + 'url' : url} def setup(self): - self.chunkLimit = -1 + self.chunkLimit = -1 + self.multiDL = True self.resumeDownload = True @@ -43,7 +50,12 @@ class BasePlugin(Hoster): for _i in xrange(5): try: - self.downloadFile(pyfile) + link = getFileURL(self, urllib.unquote(pyfile.url)) + + if link: + self.download(link, ref=False, disposition=True) + else: + self.fail(_("File not found")) except BadHeader, e: if e.code is 404: @@ -54,11 +66,11 @@ class BasePlugin(Hoster): account = self.core.accountManager.getAccountPlugin('Http') servers = [x['login'] for x in account.getAllAccounts()] - server = urlparse(pyfile.url).netloc + server = urlparse.urlparse(pyfile.url).netloc if server in servers: self.logDebug("Logging on to %s" % server) - self.req.addAuth(account.accounts[server]['password']) + self.req.addAuth(account.getAccountData(server)['password']) else: pwd = self.getPassword() if ':' in pwd: @@ -72,37 +84,20 @@ class BasePlugin(Hoster): else: self.fail(_("No file downloaded")) #@TODO: Move to hoster class in 0.4.10 - if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": #@TODO: Move to hoster in 0.4.10 - self.fail(_("Empty file")) - - - def downloadFile(self, pyfile): - url = pyfile.url - - for i in xrange(1, 7): #@TODO: retrieve the pycurl.MAXREDIRS value set by req - header = self.load(url, ref=True, cookies=True, just_header=True, decode=True) + errmsg = self.checkDownload({'Empty file' : re.compile(r'\A\s*\Z'), + 'Html error' : re.compile(r'\A(?:\s*<.+>)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)'), + 'Html file' : re.compile(r'\A\s*<!DOCTYPE html'), + 'Request error': re.compile(r'([Aa]n error occured while processing your request)')}) + if not errmsg: + return - if 'location' not in header or not header['location']: - if 'code' in header and header['code'] not in (200, 201, 203, 206): - self.logDebug("Received HTTP status code: %d" % header['code']) - self.fail(_("File not found")) - else: - break - - location = header['location'] - - self.logDebug("Redirect #%d to: %s" % (i, location)) - - if urlparse(location).scheme: - url = location - else: - p = urlparse(url) - base = "%s://%s" % (p.scheme, p.netloc) - url = urljoin(base, location) - else: - self.fail(_("Too many redirects")) + try: + errmsg += " | " + self.lastCheck.group(1).strip() + except Exception: + pass - self.download(unquote(url), disposition=True) + self.logWarning("Check result: " + errmsg, "Waiting 1 minute and retry") + self.retry(3, 60, errmsg) getInfo = create_getInfo(BasePlugin) |