diff options
| -rw-r--r-- | module/network/Browser.py | 22 | ||||
| -rw-r--r-- | module/network/HTTPChunk.py | 2 | ||||
| -rw-r--r-- | module/network/HTTPRequest.py | 10 | ||||
| -rw-r--r-- | module/network/RequestFactory.py | 6 | ||||
| -rw-r--r-- | module/plugins/hooks/ExtractArchive.py | 2 | ||||
| -rw-r--r-- | module/plugins/hoster/BasePlugin.py | 78 | ||||
| -rwxr-xr-x | module/remote/thriftbackend/thriftgen/pyload/Pyload-remote | 2 | ||||
| -rw-r--r-- | module/remote/thriftbackend/thriftgen/pyload/Pyload.py | 17 |
8 files changed, 96 insertions, 43 deletions
diff --git a/module/network/Browser.py b/module/network/Browser.py index 822e2ed6d..23cf7666b 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -8,7 +8,6 @@ from HTTPDownload import HTTPDownload class Browser(object): - __slots__ = ("log", "options", "bucket", "cj", "_size", "http", "dl") def __init__(self, bucket=None, options={}): @@ -20,9 +19,14 @@ class Browser(object): self.cj = None # needs to be setted later self._size = 0 - self.http = HTTPRequest(self.cj, options) + self.renewHTTPRequest() self.dl = None + + def renewHTTPRequest(self): + if hasattr(self, "http"): self.http.close() + self.http = HTTPRequest(self.cj, self.options) + def setLastURL(self, val): self.http.lastURL = val @@ -80,7 +84,7 @@ class Browser(object): """ this can also download ftp """ self._size = 0 self.dl = HTTPDownload(url, filename, get, post, self.lastEffectiveURL if ref else None, - self.cj if cookies else None, self.bucket, self.options, progressNotify, disposition) + self.cj if cookies else None, self.bucket, self.options, progressNotify, disposition) name = self.dl.download(chunks, resume) self._size = self.dl.size @@ -96,6 +100,18 @@ class Browser(object): """ add a header to the request """ self.http.putHeader(name, value) + def addAuth(self, pwd): + """Adds user and pw for http auth + + :param pwd: string, user:password + """ + self.options["auth"] = pwd + self.renewHTTPRequest() #we need a new request + + def removeAuth(self): + if "auth" in self.options: del self.options["auth"] + self.renewHTTPRequest() + def clearHeaders(self): self.http.clearHeaders() diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index 680b982d3..69eedb19c 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -137,7 +137,7 @@ class HTTPChunk(HTTPRequest): self.fp = None #file handle self.initHandle() - self.setInterface(self.p.options["interface"], self.p.options["proxies"], self.p.options["ipv6"]) + self.setInterface(self.p.options) self.BOMChecked = False # check and remove byte order mark diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index ffe5d1873..6672a58e6 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -55,7 +55,7 @@ class HTTPRequest(): self.headers = [] #temporary request header self.initHandle() - self.setInterface(options["interface"], options["proxies"], options["ipv6"]) + self.setInterface(options) self.c.setopt(pycurl.WRITEFUNCTION, self.write) self.c.setopt(pycurl.HEADERFUNCTION, self.writeHeader) @@ -89,7 +89,10 @@ class HTTPRequest(): "Keep-Alive: 300", "Expect:"]) - def setInterface(self, interface, proxy, ipv6=False): + def setInterface(self, options): + + interface, proxy, ipv6 = options["interface"], options["proxies"], options["ipv6"] + if interface and interface.lower() != "none": self.c.setopt(pycurl.INTERFACE, str(interface)) @@ -112,6 +115,9 @@ class HTTPRequest(): else: self.c.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) + if "auth" in options: + self.c.setopt(pycurl.USERPWD, str(options["auth"])) + def addCookies(self): """ put cookies from curl handle to cj """ if self.cj: diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 774249a70..5b1528281 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -54,9 +54,11 @@ class RequestFactory(): self.lock.release() return req - def getHTTPRequest(self): + def getHTTPRequest(self, **kwargs): """ returns a http request, dont forget to close it ! """ - return HTTPRequest(CookieJar(None), self.getOptions()) + options = self.getOptions() + options.update(kwargs) # submit kwargs as additional options + return HTTPRequest(CookieJar(None), options) def getURL(self, *args, **kwargs): """ see HTTPRequest for argument list """ diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 54dd6ff2d..23ae6c543 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -151,7 +151,7 @@ class ExtractArchive(Hook): #iterate packages -> plugins -> targets for pid in ids: p = self.core.files.getPackage(pid) - self.logInfo(_("Extract package %s") % p.name) + self.logInfo(_("Check package %s") % p.name) if not p: continue # determine output folder diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index ed11c378b..15e35ce24 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - -from module.plugins.Hoster import Hoster -from module.utils import html_unescape - from urlparse import urlparse from re import search from urllib import unquote +from module.network.HTTPRequest import BadHeader +from module.plugins.Hoster import Hoster +from module.utils import html_unescape, removeChars + class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" @@ -41,31 +41,49 @@ class BasePlugin(Hoster): # # return if pyfile.url.startswith("http"): - header = self.load(pyfile.url, just_header = True) - #self.logDebug(header) - - if 'location' in header: - self.logDebug("Location: " + header['location']) - url = unquote(header['location']) - else: - url = pyfile.url - - name = html_unescape(urlparse(url).path.split("/")[-1]) - - if 'content-disposition' in header: - self.logDebug("Content-Disposition: " + header['content-disposition']) - m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", header['content-disposition']) - if m: - disp = m.groupdict() - self.logDebug(disp) - if not disp['enc']: disp['enc'] = 'utf-8' - name = disp['name'].replace('"', "").replace("'", "").replace(";", "").strip() - name = unicode(unquote(name), disp['enc']) - - if not name: self.offline() - pyfile.name = name - self.logDebug("Filename: %s" % pyfile.name) - self.download(url, disposition=True) + + try: + self.downloadFile(pyfile) + except BadHeader, e: + if e.code in (401, 403): + self.logDebug("Auth required") + + pwd = pyfile.package().password.strip() + if ":" not in pwd: + self.fail(_("Authorization required (username:password)")) + + self.req.addAuth(pwd) + self.downloadFile(pyfile) + else: + raise else: - self.fail("No Plugin matched and not a downloadable url.")
\ No newline at end of file + self.fail("No Plugin matched and not a downloadable url.") + + + def downloadFile(self, pyfile): + header = self.load(pyfile.url, just_header = True) + #self.logDebug(header) + + if 'location' in header: + self.logDebug("Location: " + header['location']) + url = unquote(header['location']) + else: + url = pyfile.url + + name = html_unescape(urlparse(url).path.split("/")[-1]) + + if 'content-disposition' in header: + self.logDebug("Content-Disposition: " + header['content-disposition']) + m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", header['content-disposition']) + if m: + disp = m.groupdict() + self.logDebug(disp) + if not disp['enc']: disp['enc'] = 'utf-8' + name = removeChars(disp['name'], "\"';").strip() + name = unicode(unquote(name), disp['enc']) + + if not name: name = url + pyfile.name = name + self.logDebug("Filename: %s" % pyfile.name) + self.download(url, disposition=True)
\ No newline at end of file diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote index e2e56d5ca..854b1589e 100755 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote @@ -74,7 +74,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print ' void orderPackage(PackageID pid, i16 position)' print ' void orderFile(FileID fid, i16 position)' print ' void setPackageData(PackageID pid, data)' - print ' void deleteFinished()' + print ' deleteFinished()' print ' void restartFailed()' print ' bool isCaptchaWaiting()' print ' CaptchaTask getCaptchaTask(bool exclusive)' diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py index 677e4afe2..a1bc63f75 100644 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py @@ -1873,7 +1873,7 @@ class Client(Iface): def deleteFinished(self, ): self.send_deleteFinished() - self.recv_deleteFinished() + return self.recv_deleteFinished() def send_deleteFinished(self, ): self._oprot.writeMessageBegin('deleteFinished', TMessageType.CALL, self._seqid) @@ -1892,7 +1892,9 @@ class Client(Iface): result = deleteFinished_result() result.read(self._iprot) self._iprot.readMessageEnd() - return + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "deleteFinished failed: unknown result"); def restartFailed(self, ): self.send_restartFailed() @@ -3085,7 +3087,7 @@ class Processor(Iface, TProcessor): args.read(iprot) iprot.readMessageEnd() result = deleteFinished_result() - self._handler.deleteFinished() + result.success = self._handler.deleteFinished() oprot.writeMessageBegin("deleteFinished", TMessageType.REPLY, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -4904,13 +4906,22 @@ class deleteFinished_args(TBase): class deleteFinished_result(TBase): + """ + Attributes: + - success + """ __slots__ = [ + 'success', ] thrift_spec = ( + (0, TType.LIST, 'success', (TType.I32,None), None, ), # 0 ) + def __init__(self, success=None,): + self.success = success + class restartFailed_args(TBase): |