summaryrefslogtreecommitdiffstats
path: root/module/network/HTTPBase.py
diff options
context:
space:
mode:
Diffstat (limited to 'module/network/HTTPBase.py')
-rw-r--r--module/network/HTTPBase.py396
1 files changed, 0 insertions, 396 deletions
diff --git a/module/network/HTTPBase.py b/module/network/HTTPBase.py
deleted file mode 100644
index f5cd7afcc..000000000
--- a/module/network/HTTPBase.py
+++ /dev/null
@@ -1,396 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, see <http://www.gnu.org/licenses/>.
-
- @author: mkaay
-"""
-
-from urllib import urlencode
-
-from urllib2 import Request
-from urllib2 import OpenerDirector
-from urllib2 import HTTPHandler
-from urllib2 import HTTPSHandler
-from urllib2 import HTTPDefaultErrorHandler
-from urllib2 import HTTPErrorProcessor
-from urllib2 import ProxyHandler
-from urllib2 import URLError
-from urllib2 import _parse_proxy
-
-from httplib import HTTPConnection
-from httplib import HTTPResponse
-from httplib import responses as HTTPStatusCodes
-from httplib import ResponseNotReady
-from httplib import BadStatusLine
-from httplib import CannotSendRequest
-
-from CookieJar import CookieJar
-from CookieRedirectHandler import CookieRedirectHandler
-
-import socket
-import socks
-
-from MultipartPostHandler import MultipartPostHandler
-
-DEBUG = 0
-HANDLE_ERRORS = 1
-
-class PyLoadHTTPResponse(HTTPResponse):
- def __init__(self, sock, debuglevel=0, strict=0, method=None):
- if method: # the httplib in python 2.3 uses the method arg
- HTTPResponse.__init__(self, sock, debuglevel, method)
- else: # 2.2 doesn't
- HTTPResponse.__init__(self, sock, debuglevel)
- self.fileno = sock.fileno
- self._rbuf = ''
- self._rbufsize = 8096
- self._handler = None # inserted by the handler later
- self._host = None # (same)
- self._url = None # (same)
-
- _raw_read = HTTPResponse.read
-
- def close_connection(self):
- self.close()
- self._handler._remove_connection(self._host, close=1)
-
- def info(self):
- return self.msg
-
- def geturl(self):
- return self._url
-
- def read(self, amt=None):
- # the _rbuf test is only in this first if for speed. It's not
- # logically necessary
- if self._rbuf and not amt is None:
- L = len(self._rbuf)
- if amt > L:
- amt -= L
- else:
- s = self._rbuf[:amt]
- self._rbuf = self._rbuf[amt:]
- return s
-
- s = self._rbuf + self._raw_read(amt)
- self._rbuf = ''
- return s
-
- def readline(self, limit=-1):
- i = self._rbuf.find('\n')
- while i < 0 and not (0 < limit <= len(self._rbuf)):
- new = self._raw_read(self._rbufsize)
- if not new: break
- i = new.find('\n')
- if i >= 0: i = i + len(self._rbuf)
- self._rbuf = self._rbuf + new
- if i < 0: i = len(self._rbuf)
- else: i += 1
- if 0 <= limit < len(self._rbuf): i = limit
- data, self._rbuf = self._rbuf[:i], self._rbuf[i:]
- return data
-
- def readlines(self, sizehint = 0):
- total = 0
- list = []
- while 1:
- line = self.readline()
- if not line: break
- list.append(line)
- total += len(line)
- if sizehint and total >= sizehint:
- break
- return list
-
- @property
- def code(self):
- return self.status
-
- def getcode(self):
- return self.status
-
-class PyLoadHTTPConnection(HTTPConnection):
- sourceAddress = ('', 0)
- socksProxy = None
- response_class = PyLoadHTTPResponse
-
- def connect(self):
- if self.socksProxy:
- self.sock = socks.socksocket()
- t = _parse_proxy(self.socksProxy[1])
- self.sock.setproxy(self.socksProxy[0], addr=t[3].split(":")[0], port=int(t[3].split(":")[1]), username=t[1], password=t[2])
- else:
- self.sock = socket.socket()
- self.sock.settimeout(30)
- self.sock.bind(self.sourceAddress)
- self.sock.connect((self.host, self.port))
-
- try:
- if self._tunnel_host:
- self._tunnel()
- except: #python2.5
- pass
-
-class PyLoadHTTPHandler(HTTPHandler):
- sourceAddress = ('', 0)
- socksProxy = None
-
- def __init__(self):
- self._connections = {}
-
- def setInterface(self, interface):
- if interface is None:
- interface = ""
- self.sourceAddress = (interface, 0)
-
- def setSocksProxy(self, *t):
- self.socksProxy = t
-
- def close_connection(self, host):
- """close connection to <host>
- host is the host:port spec, as in 'www.cnn.com:8080' as passed in.
- no error occurs if there is no connection to that host."""
- self._remove_connection(host, close=1)
-
- def open_connections(self):
- """return a list of connected hosts"""
- return self._connections.keys()
-
- def close_all(self):
- """close all open connections"""
- for host, conn in self._connections.items():
- conn.close()
- self._connections = {}
-
- def _remove_connection(self, host, close=0):
- if self._connections.has_key(host):
- if close: self._connections[host].close()
- del self._connections[host]
-
- def _start_connection(self, h, req):
- data = ""
-
- if req.has_data():
- data = req.get_data()
- h.putrequest('POST', req.get_selector(), skip_accept_encoding=1)
- if not req.headers.has_key('Content-type'):
- h.putheader('Content-type',
- 'application/x-www-form-urlencoded')
- if not req.headers.has_key('Content-length'):
- h.putheader('Content-length', '%d' % len(data))
- else:
- h.putrequest('GET', req.get_selector(), skip_accept_encoding=1)
-
- for args in self.parent.addheaders:
- h.putheader(*args)
- for k, v in req.headers.items():
- h.putheader(k, v)
- h.endheaders()
- if req.has_data():
- h.send(data)
-
- def do_open(self, http_class, req):
- host = req.get_host()
- if not host:
- raise URLError('no host given')
-
- need_new_connection = 1
- h = self._connections.get(host)
- if not h is None:
- try:
- self._start_connection(h, req)
- except socket.error:
- r = None
- except BadStatusLine:
- r = None
- except CannotSendRequest:
- r = None
- else:
- try: r = h.getresponse()
- except ResponseNotReady: r = None
- except BadStatusLine: r = None
-
- if r is None or r.version == 9:
- # httplib falls back to assuming HTTP 0.9 if it gets a
- # bad header back. This is most likely to happen if
- # the socket has been closed by the server since we
- # last used the connection.
- if DEBUG: print "failed to re-use connection to %s" % host
- h.close()
- else:
- if DEBUG: print "re-using connection to %s" % host
- need_new_connection = 0
- if need_new_connection:
- if DEBUG: print "creating new connection to %s" % host
- h = http_class(host)
- h.sourceAddress = self.sourceAddress
- h.socksProxy = self.socksProxy
- self._connections[host] = h
- self._start_connection(h, req)
- r = h.getresponse()
-
-
- # if not a persistent connection, don't try to reuse it
- if r.will_close: self._remove_connection(host)
-
- if DEBUG:
- print "STATUS: %s, %s" % (r.status, r.reason)
- r._handler = self
- r._host = host
- r._url = req.get_full_url()
-
- if r.status in (200, 206) or not HANDLE_ERRORS:
- return r
- else:
- return self.parent.error('http', req, r, r.status, r.reason, r.msg)
-
- def http_open(self, req):
- return self.do_open(PyLoadHTTPConnection, req)
-
-class HTTPBase():
- def __init__(self, interface=None, proxies={}):
- self.followRedirect = True
- self.interface = interface
- self.proxies = proxies
-
- self.size = None
-
- self.referer = None
-
- self.userAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.10"
-
- self.handler = PyLoadHTTPHandler()
- self.handler.setInterface(interface)
- if proxies.has_key("socks5"):
- self.handler.setSocksProxy(socks.PROXY_TYPE_SOCKS5, proxies["socks5"])
- elif proxies.has_key("socks4"):
- self.handler.setSocksProxy(socks.PROXY_TYPE_SOCKS4, proxies["socks4"])
-
- self.cookieJar = CookieJar()
-
- self.opener = None
-
- self.debug = DEBUG
-
- def getOpener(self, cookies=True):
- if not self.opener:
- self.opener = self.createOpener(cookies)
-
- return self.opener
-
- def createOpener(self, cookies=True):
- opener = OpenerDirector()
- opener.add_handler(self.handler)
- opener.add_handler(MultipartPostHandler())
- opener.add_handler(HTTPSHandler())
- opener.add_handler(HTTPDefaultErrorHandler())
- opener.add_handler(HTTPErrorProcessor())
- opener.add_handler(CookieRedirectHandler(self.cookieJar, self.followRedirect))
- if self.proxies.has_key("http") or self.proxies.has_key("https"):
- opener.add_handler(ProxyHandler(self.proxies))
- opener.version = self.userAgent
- opener.addheaders[0] = ("User-Agent", self.userAgent)
- opener.addheaders.append(("Accept", "*/*"))
- opener.addheaders.append(("Accept-Language", "en-US,en"))
- opener.addheaders.append(("Accept-Encoding", "gzip, deflate"))
- opener.addheaders.append(("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7"))
- return opener
-
- def createRequest(self, url, get={}, post={}, referer=None, customHeaders={}):
- if get:
- if isinstance(get, dict):
- get = urlencode(get)
- url = "%s?%s" % (url, get)
-
- req = Request(url)
-
- if post:
- if isinstance(post, dict):
- post = urlencode(post)
- req.add_data(post)
-
- if referer:
- req.add_header("Referer", referer)
-
- for key, val in customHeaders.iteritems():
- req.add_header(key, val)
-
- return req
-
- def getResponse(self, url, get={}, post={}, referer=None, cookies=True, customHeaders={}):
- req = self.createRequest(url, get, post, referer, customHeaders)
- opener = self.getOpener(cookies)
-
- if self.debug:
- print "[HTTP] ----"
- print "[HTTP] creating request"
- print "[HTTP] URL:", url
- print "[HTTP] GET"
- if get:
- for key, value in get.iteritems():
- print "[HTTP] \t", key, ":", value
- if post:
- print "[HTTP] POST"
- for key, value in post.iteritems():
- print "[HTTP] \t", key, ":", value
- print "[HTTP] headers"
- for key, value in opener.addheaders:
- print "[HTTP] \t", key, ":", value
- for key, value in req.headers.iteritems():
- print "[HTTP] \t", key, ":", value
- print "[HTTP] cookies"
- if self.cookieJar:
- from pprint import pprint
- pprint(self.cookieJar._cookies)
- print "[HTTP] ----"
-
- resp = opener.open(req)
- resp.getcode = lambda: resp.code
-
- if self.debug:
- print "[HTTP] ----"
- print "[HTTP] got response"
- print "[HTTP] status:", resp.getcode()
- print "[HTTP] headers"
- for key, value in resp.info().dict.iteritems():
- print "[HTTP] \t", key, ":", value
- print "[HTTP] cookies"
- if self.cookieJar:
- from pprint import pprint
- pprint(self.cookieJar._cookies)
- print "[HTTP] ----"
- try:
- self.size = int(resp.info()["Content-Length"])
- except: #chunked transfer
- pass
- return resp
-
- def closeAll(self):
- """ closes all connections """
- if hasattr(self, "handler"):
- self.handler.close_all()
-
- def clean(self):
- """ cleanup """
- self.closeAll()
- if hasattr(self, "opener"):
- del self.opener
- if hasattr(self, "handler"):
- del self.handler
-
-if __name__ == "__main__":
- base = HTTPBase()
- resp = base.getResponse("http://python.org/")
- print resp.read()
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-03 01:18:16 +0100