diff options
Diffstat (limited to 'module/network')
| -rw-r--r-- | module/network/Browser.py | 14 | ||||
| -rw-r--r-- | module/network/Bucket.py | 6 | ||||
| -rw-r--r-- | module/network/CookieJar.py | 2 | ||||
| -rw-r--r-- | module/network/HTTPChunk.py | 11 | ||||
| -rw-r--r-- | module/network/HTTPDownload.py | 25 | ||||
| -rw-r--r-- | module/network/HTTPRequest.py | 23 | ||||
| -rw-r--r-- | module/network/RequestFactory.py | 6 | ||||
| -rw-r--r-- | module/network/XDCCRequest.py | 55 |
8 files changed, 52 insertions, 90 deletions
diff --git a/module/network/Browser.py b/module/network/Browser.py index d68a23687..e78d24688 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- from logging import getLogger @@ -131,16 +130,3 @@ class Browser(object): del self.dl if hasattr(self, "cj"): del self.cj - -if __name__ == "__main__": - browser = Browser()#proxies={"socks5": "localhost:5000"}) - ip = "http://www.whatismyip.com/automation/n09230945.asp" - #browser.getPage("http://google.com/search?q=bar") - #browser.getPage("https://encrypted.google.com/") - #print browser.getPage(ip) - #print browser.getRedirectLocation("http://google.com/") - #browser.getPage("https://encrypted.google.com/") - #browser.getPage("http://google.com/search?q=bar") - - browser.httpDownload("http://speedtest.netcologne.de/test_10mb.bin", "test_10mb.bin") - diff --git a/module/network/Bucket.py b/module/network/Bucket.py index 69da277ae..a096d644a 100644 --- a/module/network/Bucket.py +++ b/module/network/Bucket.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ This program is free software; you can redistribute it and/or modify @@ -13,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - + @author: RaNaN """ @@ -47,7 +46,7 @@ class Bucket: time = -self.tokens/float(self.rate) else: time = 0 - + self.lock.release() return time @@ -58,4 +57,3 @@ class Bucket: delta = self.rate * (now - self.timestamp) self.tokens = min(self.rate, self.tokens + delta) self.timestamp = now - diff --git a/module/network/CookieJar.py b/module/network/CookieJar.py index c05812334..a6ae090bc 100644 --- a/module/network/CookieJar.py +++ b/module/network/CookieJar.py @@ -19,7 +19,7 @@ from time import time -class CookieJar(): +class CookieJar: def __init__(self, pluginname, account=None): self.cookies = {} self.plugin = pluginname diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index b637aef32..719c3ed0b 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ This program is free software; you can redistribute it and/or modify @@ -13,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - + @author: RaNaN """ from os import remove, stat, fsync @@ -30,7 +29,7 @@ class WrongFormat(Exception): pass -class ChunkInfo(): +class ChunkInfo: def __init__(self, name): self.name = unicode(name) self.size = 0 @@ -208,7 +207,7 @@ class HTTPChunk(HTTPRequest): # as first chunk, we will parse the headers if not self.range and self.header.endswith("\r\n\r\n"): self.parseHeader() - elif not self.range and buf.startswith("150") and "data connection" in buf: #ftp file size parsing + elif not self.range and buf.startswith("150") and "data connection" in buf.lower(): #: ftp file size parsing size = search(r"(\d+) bytes", buf) if size: self.p.size = int(size.group(1)) @@ -269,7 +268,7 @@ class HTTPChunk(HTTPRequest): def stop(self): """The download will not proceed after next call of writeBody""" - self.range = [0,0] + self.range = [0, 0] self.size = 0 def resetRange(self): @@ -290,4 +289,4 @@ class HTTPChunk(HTTPRequest): """ closes everything, unusable after this """ if self.fp: self.fp.close() self.c.close() - if hasattr(self, "p"): del self.p
\ No newline at end of file + if hasattr(self, "p"): del self.p diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index fe8075539..20ac00e48 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ This program is free software; you can redistribute it and/or modify @@ -13,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - + @author: RaNaN """ @@ -29,9 +28,9 @@ from HTTPChunk import ChunkInfo, HTTPChunk from HTTPRequest import BadHeader from module.plugins.Plugin import Abort -from module.utils import save_join, fs_encode +from module.utils import safe_join, fs_encode -class HTTPDownload(): +class HTTPDownload: """ loads a url http + ftp """ def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, @@ -114,7 +113,7 @@ class HTTPDownload(): fo.close() if self.nameDisposition and self.disposition: - self.filename = save_join(dirname(self.filename), self.nameDisposition) + self.filename = safe_join(dirname(self.filename), self.nameDisposition) move(init, fs_encode(self.filename)) self.info.remove() #remove info file @@ -286,7 +285,7 @@ class HTTPDownload(): if self.abort: raise Abort() - #sleep(0.003) #supress busy waiting - limits dl speed to (1 / x) * buffersize + #sleep(0.003) #supress busy waiting - limits dl speed to (1 / x) * buffersize self.m.select(1) for chunk in self.chunks: @@ -324,17 +323,3 @@ class HTTPDownload(): del self.cj if hasattr(self, "info"): del self.info - -if __name__ == "__main__": - url = "http://speedtest.netcologne.de/test_100mb.bin" - - from Bucket import Bucket - - bucket = Bucket() - bucket.setRate(200 * 1024) - bucket = None - - print "starting" - - dwnld = HTTPDownload(url, "test_100mb.bin", bucket=bucket) - dwnld.download(chunks=3, resume=True) diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 4747d937f..67635f944 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ This program is free software; you can redistribute it and/or modify @@ -13,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - + @author: RaNaN """ @@ -29,7 +28,7 @@ from module.plugins.Plugin import Abort def myquote(url): return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") - + def myurlencode(data): data = dict(data) return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ @@ -44,7 +43,7 @@ class BadHeader(Exception): self.content = content -class HTTPRequest(): +class HTTPRequest: def __init__(self, cookies=None, options=None): self.c = pycurl.Curl() self.rep = StringIO() @@ -89,8 +88,8 @@ class HTTPRequest(): if pycurl.version_info()[7]: self.c.setopt(pycurl.ENCODING, "gzip, deflate") self.c.setopt(pycurl.HTTPHEADER, ["Accept: */*", - "Accept-Language: en-US,en", - "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language: en-US, en", + "Accept-Charset: ISO-8859-1, utf-8;q=0.7,*;q=0.7", "Connection: keep-alive", "Keep-Alive: 300", "Expect:"]) @@ -193,6 +192,10 @@ class HTTPRequest(): if just_header: self.c.setopt(pycurl.FOLLOWLOCATION, 0) self.c.setopt(pycurl.NOBODY, 1) + if post: + self.c.setopt(pycurl.POST, 1) + else: + self.c.setopt(pycurl.HTTPGET, 1) self.c.perform() rep = self.header @@ -255,7 +258,7 @@ class HTTPRequest(): #self.log.debug("Decoded %s" % encoding ) if lookup(encoding).name == 'utf-8' and rep.startswith(BOM_UTF8): encoding = 'utf-8-sig' - + decoder = getincrementaldecoder(encoding)("replace") rep = decoder.decode(rep, True) @@ -298,9 +301,3 @@ class HTTPRequest(): if hasattr(self, "c"): self.c.close() del self.c - -if __name__ == "__main__": - url = "http://pyload.org" - c = HTTPRequest() - print c.load(url) - diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 5b1528281..6811b11d8 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - + @author: mkaay, RaNaN """ @@ -26,7 +26,7 @@ from CookieJar import CookieJar from XDCCRequest import XDCCRequest -class RequestFactory(): +class RequestFactory: def __init__(self, core): self.lock = Lock() self.core = core @@ -67,7 +67,7 @@ class RequestFactory(): rep = h.load(*args, **kwargs) finally: h.close() - + return rep def getCookieJar(self, pluginName, account=None): diff --git a/module/network/XDCCRequest.py b/module/network/XDCCRequest.py index f03798c17..74769da96 100644 --- a/module/network/XDCCRequest.py +++ b/module/network/XDCCRequest.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ This program is free software; you can redistribute it and/or modify @@ -13,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - + @author: jeix """ @@ -31,19 +30,18 @@ from select import select from module.plugins.Plugin import Abort -class XDCCRequest(): +class XDCCRequest: def __init__(self, timeout=30, proxies={}): - + self.proxies = proxies self.timeout = timeout - + self.filesize = 0 self.recv = 0 self.speed = 0 - + self.abort = False - def createSocket(self): # proxytype = None # proxy = None @@ -60,33 +58,33 @@ class XDCCRequest(): # else: # sock = socket.socket() # return sock - + return socket.socket() - + def download(self, ip, port, filename, irc, progressNotify=None): ircbuffer = "" lastUpdate = time() cumRecvLen = 0 - + dccsock = self.createSocket() - + dccsock.settimeout(self.timeout) dccsock.connect((ip, port)) - + if exists(filename): i = 0 nameParts = filename.rpartition(".") while True: newfilename = "%s-%d%s%s" % (nameParts[0], i, nameParts[1], nameParts[2]) i += 1 - + if not exists(newfilename): filename = newfilename break - + fh = open(filename, "wb") - + # recv loop for dcc socket while True: if self.abort: @@ -94,44 +92,43 @@ class XDCCRequest(): fh.close() remove(filename) raise Abort() - + self._keepAlive(irc, ircbuffer) - + data = dccsock.recv(4096) dataLen = len(data) self.recv += dataLen - + cumRecvLen += dataLen - + now = time() timespan = now - lastUpdate if timespan > 1: self.speed = cumRecvLen / timespan cumRecvLen = 0 lastUpdate = now - + if progressNotify: progressNotify(self.percent) - - + if not data: break - + fh.write(data) - + # acknowledge data by sending number of recceived bytes dccsock.send(struct.pack('!I', self.recv)) - + dccsock.close() fh.close() - + return filename - + def _keepAlive(self, sock, readbuffer): fdset = select([sock], [], [], 0) if sock not in fdset[0]: return - + readbuffer += sock.recv(1024) temp = readbuffer.split("\n") readbuffer = temp.pop() @@ -144,7 +141,7 @@ class XDCCRequest(): def abortDownloads(self): self.abort = True - + @property def size(self): return self.filesize |