From dcd493c8c029a45e7e1e87fbbcae69f5385d5f9e Mon Sep 17 00:00:00 2001
From: RaNaN <Mast3rRaNaN@hotmail.de>
Date: Tue, 21 Dec 2010 22:42:57 +0100
Subject: changed request files
---
module/network/Browser.py | 9 +-
module/network/NewRequest.py | 185 ------------------
module/network/OldRequest.py | 435 +++++++++++++++++++++++++++++++++++++++++++
module/network/Request.py | 342 +++++-----------------------------
4 files changed, 487 insertions(+), 484 deletions(-)
delete mode 100755 module/network/NewRequest.py
create mode 100755 module/network/OldRequest.py
diff --git a/module/network/Browser.py b/module/network/Browser.py
index c0742c4ff..d70cb3aba 100644
--- a/module/network/Browser.py
+++ b/module/network/Browser.py
@@ -78,13 +78,16 @@ class Browser():
return d
if __name__ == "__main__":
- browser = Browser(proxies={"socks5": "localhost:5000"})
+ browser = Browser()#proxies={"socks5": "localhost:5000"})
ip = "http://www.whatismyip.com/automation/n09230945.asp"
#browser.getPage("http://google.com/search?q=bar")
#browser.getPage("https://encrypted.google.com/")
- print browser.getPage(ip)
+ #print browser.getPage(ip)
#print browser.getRedirectLocation("http://google.com/")
#browser.getPage("https://encrypted.google.com/")
#browser.getPage("http://google.com/search?q=bar")
- #browser.downloadFile("http://speedtest.netcologne.de/test_100mb.bin", "test_100mb.bin")
+ browser.httpDownload("http://speedtest.netcologne.de/test_100mb.bin", "test_100mb.bin")
+ from time import sleep
+ while True:
+ sleep(1)
\ No newline at end of file
diff --git a/module/network/NewRequest.py b/module/network/NewRequest.py
deleted file mode 100755
index 9ac7d54aa..000000000
--- a/module/network/NewRequest.py
+++ /dev/null
@@ -1,185 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, see <http://www.gnu.org/licenses/>.
-
- @author: spoob
- @author: RaNaN
- @author: mkaay
-"""
-
-import time
-from os.path import exists, join
-from shutil import move
-import urllib
-
-from module.plugins.Plugin import Abort
-
-from module.network.Browser import Browser
-from module.network.helper import waitFor
-
-class Request:
- def __init__(self, interface=None):
- self.browser = Browser(interface=interface)
- self.d = None
-
- self.dl_time = 0
- self.dl_finished = 0
- self.dl_size = 0
- self.dl_arrived = 0
- self.dl = False
-
- self.abort = False
-
- self.lastEffectiveURL = self.lastURL = property(lambda: self.browser.lastUrl)
- self.auth = False
-
- self.canContinue = False
-
- self.dl_speed = 0.0
-
- self.cookieJar = None
- self.interface = interface
- self.progressNotify = None
-
- # change this for connection information
- self.debug = False
-
- def set_timeout(self, timeout):
- self.timeout = int(timeout)
-
- def setCookieJar(self, j):
- self.cookieJar = j
-
- def addCookies(self):
- #@TODO
- pass
-
- def getCookies(self):
- #@TODO
- pass
-
- def getCookie(self, name):
- #@TODO
- pass
-
- def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, no_post_encode=False, raw_cookies={}):
- url = self.__myquote(str(url))
-
- #@TODO: cookies
- #@TODO: auth
-
- if not ref:
- self.browser.clearReferer()
-
- return self.browser.getPage(url, get=get, post=post, cookies=cookies)
-
- def add_auth(self, user, pw):
- #@TODO
- pass
-
- def clearCookies(self):
- #@TODO
- pass
-
- def add_proxy(self, protocol, adress):
- #@TODO
- pass
-
- def download(self, url, file_name, folder, get={}, post={}, ref=True, cookies=True, no_post_encode=False):
- url = self.__myquote(str(url))
-
- file_temp = self.get_free_name(folder,file_name)
-
- #@TODO: cookies
- #@TODO: auth
-
- if not ref:
- self.browser.clearReferer()
-
- self.d = self.browser.httpDownload(url, file_temp, get=get, post=post, cookies=cookies, chunks=1, resume=self.canContinue)
- self.dl_time = property(lambda: self.d.startTime)
- self.dl_finished = property(lambda: self.d.endTime)
- self.dl_speed = property(lambda: self.d.speed)
- self.dl_size = property(lambda: self.d.size)
- self.dl = property(lambda: True if self.d.startTime and not self.d.endTime else False)
- self.abort = property(self.d.getAbort, self.d.setAbort)
-
- waitFor(self.d)
-
- if self.abort: raise Abort
-
- free_name = self.get_free_name(folder, file_name)
- move(file_temp, free_name)
-
- self.dl_time = 0
- self.dl_finished = 0
- self.dl_size = 0
- self.dl_arrived = 0
- self.dl = False
- self.dl_speed = 0.0
-
- return free_name
-
- def get_speed(self):
- try:
- return self.dl_speed
- except:
- return 0
-
- def get_ETA(self):
- try:
- return (self.dl_size - self.dl_arrived) / (self.dl_arrived / (time.time() - self.dl_time))
- except:
- return 0
-
- def bytes_left(self):
- return (self.dl_size - self.dl_arrived)
-
- def progress(self):
- if self.progressNotify:
- try:
- progress = int(float(self.dl_arrived)/self.dl_size*100)
- self.progressNotify(progress)
- except:
- pass
-
- def get_free_name(self, folder, file_name):
- file_count = 0
- file_name = join(folder, file_name)
- while exists(file_name):
- file_count += 1
- if "." in file_name:
- file_split = file_name.split(".")
- temp_name = "%s-%i.%s" % (".".join(file_split[:-1]), file_count, file_split[-1])
- else:
- temp_name = "%s-%i" % (file_name, file_count)
- if not exists(temp_name):
- file_name = temp_name
- return file_name
-
- def __myquote(self, url):
- return urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
-
-
-def getURL(url, get={}, post={}):
- """
- currently used for update check
- """
- b = Browser()
- return b.getPage(url, get=get, post=post)
-
-if __name__ == "__main__":
- import doctest
- doctest.testmod()
diff --git a/module/network/OldRequest.py b/module/network/OldRequest.py
new file mode 100755
index 000000000..5314f97ab
--- /dev/null
+++ b/module/network/OldRequest.py
@@ -0,0 +1,435 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+ @author: spoob
+ @author: RaNaN
+ @author: mkaay
+"""
+
+import base64
+import time
+from os.path import exists, join
+from shutil import move
+import urllib
+from cStringIO import StringIO
+import pycurl
+
+from module.plugins.Plugin import Abort
+
+class Request:
+ def __init__(self, interface=None):
+
+ self.dl_time = 0
+ self.dl_finished = 0
+ self.dl_size = 0
+ self.dl_arrived = 0
+ self.dl = False
+
+ self.abort = False
+
+ self.lastEffectiveURL = None
+ self.lastURL = None
+ self.auth = False
+
+ self.timeout = 5
+
+ bufferBase = 1024
+ bufferMulti = 4
+ self.bufferSize = bufferBase*bufferMulti
+ self.canContinue = False
+ self.offset = 0
+
+ self.dl_speed = 0.0
+ self.averageSpeed = 0.0
+ self.averageSpeeds = []
+ self.averageSpeedTime = 0.0
+ self.averageSpeedCount = 0.0
+
+ self.speedLimitActive = False
+ self.maxSpeed = 0
+ self.isSlow = False
+ self.cookieJar = None
+ self.interface = interface
+ self.progressNotify = None
+
+ # change this for connection information
+ self.debug = False
+
+ self.init_curl()
+
+ def set_timeout(self, timeout):
+ self.timeout = int(timeout)
+
+ def init_curl(self):
+ self.rep = StringIO()
+ self.header = ""
+
+ self.pycurl = pycurl.Curl()
+ self.pycurl.setopt(pycurl.FOLLOWLOCATION, 1)
+ self.pycurl.setopt(pycurl.MAXREDIRS, 5)
+ self.pycurl.setopt(pycurl.TIMEOUT, (self.timeout*3600))
+ self.pycurl.setopt(pycurl.CONNECTTIMEOUT, 30)
+ self.pycurl.setopt(pycurl.NOSIGNAL, 1)
+ self.pycurl.setopt(pycurl.NOPROGRESS, 0)
+ self.pycurl.setopt(pycurl.PROGRESSFUNCTION, self.progress)
+ if hasattr(pycurl, "AUTOREFERER"):
+ self.pycurl.setopt(pycurl.AUTOREFERER, 1)
+ self.pycurl.setopt(pycurl.HEADERFUNCTION, self.write_header)
+ #self.pycurl.setopt(pycurl.BUFFERSIZE, self.bufferSize)
+ self.pycurl.setopt(pycurl.SSL_VERIFYPEER, 0)
+ self.pycurl.setopt(pycurl.LOW_SPEED_TIME, 30)
+ self.pycurl.setopt(pycurl.LOW_SPEED_LIMIT, 20)
+
+ if self.debug:
+ self.pycurl.setopt(pycurl.VERBOSE, 1)
+ if self.interface and self.interface.lower() != "none":
+ self.pycurl.setopt(pycurl.INTERFACE, self.interface)
+
+
+ self.pycurl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.2.10) Gecko/20100916 Firefox/3.6.10")
+ if pycurl.version_info()[7]:
+ self.pycurl.setopt(pycurl.ENCODING, "gzip, deflate")
+ self.pycurl.setopt(pycurl.HTTPHEADER, ["Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7",
+ "Connection: keep-alive",
+ "Keep-Alive: 300"])
+
+ def setCookieJar(self, j):
+ self.cookieJar = j
+
+ def addCookies(self):
+ if self.cookieJar:
+ self.cookieJar.addCookies(self.pycurl.getinfo(pycurl.INFO_COOKIELIST))
+ return
+
+ def getCookies(self):
+ if self.cookieJar:
+ for c in self.cookieJar.getCookies():
+ self.pycurl.setopt(pycurl.COOKIELIST, c)
+ return
+
+ def getCookie(self, name):
+ if self.cookieJar:
+ return self.cookieJar.getCookie(name)
+ return None
+
+ def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, no_post_encode=False, raw_cookies={}):
+
+ self.pycurl.setopt(pycurl.NOPROGRESS, 1)
+
+ url = self.__myquote(str(url))
+
+ if post:
+ if not no_post_encode:
+ post = urllib.urlencode(post)
+ else:
+ post = None
+
+ if get:
+ get = urllib.urlencode(get)
+ url = "%s?%s" % (url, get)
+ else:
+ get = ""
+
+ self.pycurl.setopt(pycurl.URL, url)
+ self.pycurl.setopt(pycurl.WRITEFUNCTION, self.write_rep)
+
+ if cookies:
+ self.curl_enable_cookies()
+ self.getCookies()
+ if raw_cookies:
+ self.pycurl.setopt(pycurl.COOKIELIST, "Set-Cookie: %s;"% "; ".join(["%s=%s"%(x,y) for x,y in raw_cookies.items()]))
+
+ if post:
+ self.pycurl.setopt(pycurl.POSTFIELDS, post)
+
+ if ref and self.lastURL is not None:
+ self.pycurl.setopt(pycurl.REFERER, self.lastURL)
+
+ if just_header:
+ self.pycurl.setopt(pycurl.NOBODY, 1)
+ self.pycurl.perform()
+ self.lastEffectiveURL = self.pycurl.getinfo(pycurl.EFFECTIVE_URL)
+ self.pycurl.setopt(pycurl.NOPROGRESS, 0)
+ self.pycurl.setopt(pycurl.NOBODY, 0)
+ return self.header
+
+ self.pycurl.perform()
+
+ self.lastEffectiveURL = self.pycurl.getinfo(pycurl.EFFECTIVE_URL)
+ self.addCookies()
+
+ #reset progress
+
+ self.dl_time = 0
+ self.dl_finished = 0
+ self.dl_size = 0
+ self.dl_arrived = 0
+
+ self.lastURL = url
+ header = self.get_header()
+
+ return self.get_rep()
+
+ def curl_enable_cookies(self):
+ self.pycurl.setopt(pycurl.COOKIEFILE, "")
+ self.pycurl.setopt(pycurl.COOKIEJAR, "")
+
+ def add_auth(self, user, pw):
+
+ self.auth = True
+ self.user = user
+ self.pw = pw
+
+ upwstr = str("%s:%s" % (user,pw))
+ self.pycurl.setopt(pycurl.HTTPHEADER, ['Authorization: Basic ' + base64.encodestring(upwstr)[:-1]])
+ self.pycurl.setopt(pycurl.USERPWD, upwstr)
+ self.pycurl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_ANY)
+
+ def clearCookies(self):
+ self.pycurl.setopt(pycurl.COOKIELIST, "")
+
+ def add_proxy(self, protocol, adress):
+ # @TODO: pycurl proxy protocol selection
+ self.pycurl.setopt(pycurl.PROXY, adress.split(":")[0])
+ self.pycurl.setopt(pycurl.PROXYPORT, adress.split(":")[1])
+
+ def download(self, url, file_name, folder, get={}, post={}, ref=True, cookies=True, no_post_encode=False):
+
+ url = self.__myquote(str(url))
+
+ self.pycurl.setopt(pycurl.NOPROGRESS, 0)
+
+ if post:
+ if not no_post_encode:
+ post = urllib.urlencode(post)
+ else:
+ post = None
+
+ if get:
+ get = urllib.urlencode(get)
+ url = "%s?%s" % (url, get)
+ else:
+ get = ""
+
+ file_temp = self.get_free_name(folder,file_name) + ".part"
+
+ self.fp = open(file_temp, 'wb' if not self.canContinue else 'ab')
+
+ partSize = self.fp.tell()
+
+ self.init_curl()
+
+ self.pycurl.setopt(pycurl.URL, url)
+
+ if self.canContinue:
+ self.offset = partSize
+ self.pycurl.setopt(pycurl.RESUME_FROM, self.offset)
+
+ self.dl_arrived = self.offset
+
+ if cookies:
+ self.curl_enable_cookies()
+ self.getCookies()
+
+ if post:
+ self.pycurl.setopt(pycurl.POSTFIELDS, post)
+
+ if self.auth:
+ self.add_auth(self.user, self.pw)
+
+ if ref and self.lastURL is not None:
+ self.pycurl.setopt(pycurl.REFERER, self.lastURL)
+
+ self.dl_time = time.time()
+ self.dl = True
+
+ self.chunkSize = 0
+ self.chunkRead = 0
+ self.subStartTime = 0
+ self.maxChunkSize = 0
+
+ def restLimit():
+ subTime = time.time() - self.subStartTime
+ if subTime <= 1:
+ if self.speedLimitActive:
+ return self.maxChunkSize
+ else:
+ return -1
+ else:
+ self.updateCurrentSpeed(float(self.chunkRead/1024) / subTime)
+
+ self.subStartTime += subTime # time.time()
+ self.chunkRead = 0
+ if self.maxSpeed > 0:
+ self.maxChunkSize = self.maxSpeed
+ else:
+ self.maxChunkSize = 0
+ return 0
+
+ def writefunc(buf):
+ if self.abort:
+ return False
+ chunkSize = len(buf)
+ while chunkSize > restLimit() > -1:
+ time.sleep(0.05)
+ self.maxChunkSize -= chunkSize
+ self.fp.write(buf)
+ self.chunkRead += chunkSize
+ self.dl_arrived += chunkSize
+
+ self.pycurl.setopt(pycurl.WRITEFUNCTION, writefunc)
+ #self.pycurl.setopt(pycurl.WRITEDATA, self.fp)
+
+ try:
+ self.pycurl.perform()
+ except Exception, e:
+ code, msg = e
+ if not code == 23:
+ raise Exception, e
+ finally:
+ self.dl = False
+ self.dl_finished = time.time()
+
+ self.addCookies()
+ self.fp.close()
+
+ if self.abort: raise Abort
+
+ free_name = self.get_free_name(folder, file_name)
+ move(file_temp, free_name)
+
+ #@TODO content disposition
+
+ return free_name
+
+ def updateCurrentSpeed(self, speed):
+ self.dl_speed = speed
+ if self.averageSpeedTime + 10 < time.time():
+ self.averageSpeeds = []
+ self.averageSpeeds.append(self.averageSpeed)
+ self.averageSpeeds.append(speed)
+ self.averageSpeed = (speed + self.averageSpeed)/2
+ self.averageSpeedTime = time.time()
+ self.averageSpeedCount = 2
+ else:
+ self.averageSpeeds.append(speed)
+ self.averageSpeedCount += 1
+ allspeed = 0.0
+ for s in self.averageSpeeds:
+ allspeed += s
+ self.averageSpeed = allspeed / self.averageSpeedCount
+
+ def write_header(self, string):
+ self.header += string
+
+ def write_rep(self, buf):
+ if self.rep.tell() > 400000 or self.abort:
+ rep = self.get_rep()
+ if self.abort: raise Abort
+ f = open("response.dump", "wb")
+ f.write(rep)
+ f.close()
+ raise Exception("Loaded Url exceeded limit")
+
+ self.rep.write(buf)
+
+ def get_rep(self):
+ value = self.rep.getvalue()
+ self.rep.close()
+ self.rep = StringIO()
+ return value
+
+ def get_header(self):
+ h = self.header
+ self.header = ""
+ return h
+
+ def get_speed(self):
+ try:
+ return self.dl_speed
+ #return (self.dl_arrived / (time.time()- self.dl_time)) / 1024
+ except:
+ return 0
+
+ def get_ETA(self):
+ try:
+ return (self.dl_size - self.dl_arrived) / (self.dl_arrived / (time.time() - self.dl_time))
+ except:
+ return 0
+
+ def bytes_left(self):
+ return (self.dl_size - self.dl_arrived)
+
+ def progress(self, dl_t, dl_d, up_t, up_d):
+ if self.abort:
+ return False
+ self.dl_arrived = int(dl_d)+self.offset
+ self.dl_size = int(dl_t)+self.offset
+
+ if self.progressNotify:
+ try:
+ progress = int(float(self.dl_arrived)/self.dl_size*100)
+ self.progressNotify(progress)
+ except:
+ pass
+
+ def get_free_name(self, folder, file_name):
+ file_count = 0
+ file_name = join(folder, file_name)
+ while exists(file_name):
+ file_count += 1
+ if "." in file_name:
+ file_split = file_name.split(".")
+ temp_name = "%s-%i.%s" % (".".join(file_split[:-1]), file_count, file_split[-1])
+ else:
+ temp_name = "%s-%i" % (file_name, file_count)
+ if not exists(temp_name):
+ file_name = temp_name
+ return file_name
+
+ def __del__(self):
+ self.clean()
+ if hasattr(self, "pycurl"):
+ del self.pycurl
+ if hasattr(self, "cookieJar"):
+ del self.cookieJar
+
+ def clean(self):
+ """ clean Reqest, its unusable after this """
+ try:
+ self.pycurl.close()
+ if hasattr(self, "cookieJar"):
+ del self.cookieJar
+ except:
+ pass
+
+ def __myquote(self, url):
+ return urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
+
+
+def getURL(url, get={}, post={}):
+ """
+ currently used for update check
+ """
+ req = Request()
+ c = req.load(url, get, post)
+ req.pycurl.close()
+ return c
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod()
diff --git a/module/network/Request.py b/module/network/Request.py
index 5314f97ab..9ac7d54aa 100755
--- a/module/network/Request.py
+++ b/module/network/Request.py
@@ -19,19 +19,21 @@
@author: mkaay
"""
-import base64
import time
from os.path import exists, join
from shutil import move
import urllib
-from cStringIO import StringIO
-import pycurl
from module.plugins.Plugin import Abort
+from module.network.Browser import Browser
+from module.network.helper import waitFor
+
class Request:
def __init__(self, interface=None):
-
+ self.browser = Browser(interface=interface)
+ self.d = None
+
self.dl_time = 0
self.dl_finished = 0
self.dl_size = 0
@@ -40,27 +42,13 @@ class Request:
self.abort = False
- self.lastEffectiveURL = None
- self.lastURL = None
+ self.lastEffectiveURL = self.lastURL = property(lambda: self.browser.lastUrl)
self.auth = False
- self.timeout = 5
-
- bufferBase = 1024
- bufferMulti = 4
- self.bufferSize = bufferBase*bufferMulti
self.canContinue = False
- self.offset = 0
self.dl_speed = 0.0
- self.averageSpeed = 0.0
- self.averageSpeeds = []
- self.averageSpeedTime = 0.0
- self.averageSpeedCount = 0.0
- self.speedLimitActive = False
- self.maxSpeed = 0
- self.isSlow = False
self.cookieJar = None
self.interface = interface
self.progressNotify = None
@@ -68,300 +56,85 @@ class Request:
# change this for connection information
self.debug = False
- self.init_curl()
-
def set_timeout(self, timeout):
self.timeout = int(timeout)
-
- def init_curl(self):
- self.rep = StringIO()
- self.header = ""
-
- self.pycurl = pycurl.Curl()
- self.pycurl.setopt(pycurl.FOLLOWLOCATION, 1)
- self.pycurl.setopt(pycurl.MAXREDIRS, 5)
- self.pycurl.setopt(pycurl.TIMEOUT, (self.timeout*3600))
- self.pycurl.setopt(pycurl.CONNECTTIMEOUT, 30)
- self.pycurl.setopt(pycurl.NOSIGNAL, 1)
- self.pycurl.setopt(pycurl.NOPROGRESS, 0)
- self.pycurl.setopt(pycurl.PROGRESSFUNCTION, self.progress)
- if hasattr(pycurl, "AUTOREFERER"):
- self.pycurl.setopt(pycurl.AUTOREFERER, 1)
- self.pycurl.setopt(pycurl.HEADERFUNCTION, self.write_header)
- #self.pycurl.setopt(pycurl.BUFFERSIZE, self.bufferSize)
- self.pycurl.setopt(pycurl.SSL_VERIFYPEER, 0)
- self.pycurl.setopt(pycurl.LOW_SPEED_TIME, 30)
- self.pycurl.setopt(pycurl.LOW_SPEED_LIMIT, 20)
-
- if self.debug:
- self.pycurl.setopt(pycurl.VERBOSE, 1)
- if self.interface and self.interface.lower() != "none":
- self.pycurl.setopt(pycurl.INTERFACE, self.interface)
-
-
- self.pycurl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.2.10) Gecko/20100916 Firefox/3.6.10")
- if pycurl.version_info()[7]:
- self.pycurl.setopt(pycurl.ENCODING, "gzip, deflate")
- self.pycurl.setopt(pycurl.HTTPHEADER, ["Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
- "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7",
- "Connection: keep-alive",
- "Keep-Alive: 300"])
def setCookieJar(self, j):
self.cookieJar = j
def addCookies(self):
- if self.cookieJar:
- self.cookieJar.addCookies(self.pycurl.getinfo(pycurl.INFO_COOKIELIST))
- return
+ #@TODO
+ pass
def getCookies(self):
- if self.cookieJar:
- for c in self.cookieJar.getCookies():
- self.pycurl.setopt(pycurl.COOKIELIST, c)
- return
+ #@TODO
+ pass
def getCookie(self, name):
- if self.cookieJar:
- return self.cookieJar.getCookie(name)
- return None
+ #@TODO
+ pass
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, no_post_encode=False, raw_cookies={}):
-
- self.pycurl.setopt(pycurl.NOPROGRESS, 1)
-
url = self.__myquote(str(url))
-
- if post:
- if not no_post_encode:
- post = urllib.urlencode(post)
- else:
- post = None
-
- if get:
- get = urllib.urlencode(get)
- url = "%s?%s" % (url, get)
- else:
- get = ""
-
- self.pycurl.setopt(pycurl.URL, url)
- self.pycurl.setopt(pycurl.WRITEFUNCTION, self.write_rep)
-
- if cookies:
- self.curl_enable_cookies()
- self.getCookies()
- if raw_cookies:
- self.pycurl.setopt(pycurl.COOKIELIST, "Set-Cookie: %s;"% "; ".join(["%s=%s"%(x,y) for x,y in raw_cookies.items()]))
-
- if post:
- self.pycurl.setopt(pycurl.POSTFIELDS, post)
-
- if ref and self.lastURL is not None:
- self.pycurl.setopt(pycurl.REFERER, self.lastURL)
-
- if just_header:
- self.pycurl.setopt(pycurl.NOBODY, 1)
- self.pycurl.perform()
- self.lastEffectiveURL = self.pycurl.getinfo(pycurl.EFFECTIVE_URL)
- self.pycurl.setopt(pycurl.NOPROGRESS, 0)
- self.pycurl.setopt(pycurl.NOBODY, 0)
- return self.header
-
- self.pycurl.perform()
- self.lastEffectiveURL = self.pycurl.getinfo(pycurl.EFFECTIVE_URL)
- self.addCookies()
-
- #reset progress
-
- self.dl_time = 0
- self.dl_finished = 0
- self.dl_size = 0
- self.dl_arrived = 0
+ #@TODO: cookies
+ #@TODO: auth
- self.lastURL = url
- header = self.get_header()
+ if not ref:
+ self.browser.clearReferer()
- return self.get_rep()
-
- def curl_enable_cookies(self):
- self.pycurl.setopt(pycurl.COOKIEFILE, "")
- self.pycurl.setopt(pycurl.COOKIEJAR, "")
+ return self.browser.getPage(url, get=get, post=post, cookies=cookies)
def add_auth(self, user, pw):
-
- self.auth = True
- self.user = user
- self.pw = pw
-
- upwstr = str("%s:%s" % (user,pw))
- self.pycurl.setopt(pycurl.HTTPHEADER, ['Authorization: Basic ' + base64.encodestring(upwstr)[:-1]])
- self.pycurl.setopt(pycurl.USERPWD, upwstr)
- self.pycurl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_ANY)
+ #@TODO
+ pass
def clearCookies(self):
- self.pycurl.setopt(pycurl.COOKIELIST, "")
+ #@TODO
+ pass
def add_proxy(self, protocol, adress):
- # @TODO: pycurl proxy protocol selection
- self.pycurl.setopt(pycurl.PROXY, adress.split(":")[0])
- self.pycurl.setopt(pycurl.PROXYPORT, adress.split(":")[1])
+ #@TODO
+ pass
def download(self, url, file_name, folder, get={}, post={}, ref=True, cookies=True, no_post_encode=False):
-
url = self.__myquote(str(url))
- self.pycurl.setopt(pycurl.NOPROGRESS, 0)
-
- if post:
- if not no_post_encode:
- post = urllib.urlencode(post)
- else:
- post = None
-
- if get:
- get = urllib.urlencode(get)
- url = "%s?%s" % (url, get)
- else:
- get = ""
-
- file_temp = self.get_free_name(folder,file_name) + ".part"
+ file_temp = self.get_free_name(folder,file_name)
- self.fp = open(file_temp, 'wb' if not self.canContinue else 'ab')
-
- partSize = self.fp.tell()
-
- self.init_curl()
-
- self.pycurl.setopt(pycurl.URL, url)
-
- if self.canContinue:
- self.offset = partSize
- self.pycurl.setopt(pycurl.RESUME_FROM, self.offset)
-
- self.dl_arrived = self.offset
-
- if cookies:
- self.curl_enable_cookies()
- self.getCookies()
-
- if post:
- self.pycurl.setopt(pycurl.POSTFIELDS, post)
+ #@TODO: cookies
+ #@TODO: auth
- if self.auth:
- self.add_auth(self.user, self.pw)
-
- if ref and self.lastURL is not None:
- self.pycurl.setopt(pycurl.REFERER, self.lastURL)
-
- self.dl_time = time.time()
- self.dl = True
+ if not ref:
+ self.browser.clearReferer()
- self.chunkSize = 0
- self.chunkRead = 0
- self.subStartTime = 0
- self.maxChunkSize = 0
+ self.d = self.browser.httpDownload(url, file_temp, get=get, post=post, cookies=cookies, chunks=1, resume=self.canContinue)
+ self.dl_time = property(lambda: self.d.startTime)
+ self.dl_finished = property(lambda: self.d.endTime)
+ self.dl_speed = property(lambda: self.d.speed)
+ self.dl_size = property(lambda: self.d.size)
+ self.dl = property(lambda: True if self.d.startTime and not self.d.endTime else False)
+ self.abort = property(self.d.getAbort, self.d.setAbort)
- def restLimit():
- subTime = time.time() - self.subStartTime
- if subTime <= 1:
- if self.speedLimitActive:
- return self.maxChunkSize
- else:
- return -1
- else:
- self.updateCurrentSpeed(float(self.chunkRead/1024) / subTime)
-
- self.subStartTime += subTime # time.time()
- self.chunkRead = 0
- if self.maxSpeed > 0:
- self.maxChunkSize = self.maxSpeed
- else:
- self.maxChunkSize = 0
- return 0
-
- def writefunc(buf):
- if self.abort:
- return False
- chunkSize = len(buf)
- while chunkSize > restLimit() > -1:
- time.sleep(0.05)
- self.maxChunkSize -= chunkSize
- self.fp.write(buf)
- self.chunkRead += chunkSize
- self.dl_arrived += chunkSize
-
- self.pycurl.setopt(pycurl.WRITEFUNCTION, writefunc)
- #self.pycurl.setopt(pycurl.WRITEDATA, self.fp)
-
- try:
- self.pycurl.perform()
- except Exception, e:
- code, msg = e
- if not code == 23:
- raise Exception, e
- finally:
- self.dl = False
- self.dl_finished = time.time()
-
- self.addCookies()
- self.fp.close()
+ waitFor(self.d)
if self.abort: raise Abort
free_name = self.get_free_name(folder, file_name)
move(file_temp, free_name)
- #@TODO content disposition
+ self.dl_time = 0
+ self.dl_finished = 0
+ self.dl_size = 0
+ self.dl_arrived = 0
+ self.dl = False
+ self.dl_speed = 0.0
return free_name
-
- def updateCurrentSpeed(self, speed):
- self.dl_speed = speed
- if self.averageSpeedTime + 10 < time.time():
- self.averageSpeeds = []
- self.averageSpeeds.append(self.averageSpeed)
- self.averageSpeeds.append(speed)
- self.averageSpeed = (speed + self.averageSpeed)/2
- self.averageSpeedTime = time.time()
- self.averageSpeedCount = 2
- else:
- self.averageSpeeds.append(speed)
- self.averageSpeedCount += 1
- allspeed = 0.0
- for s in self.averageSpeeds:
- allspeed += s
- self.averageSpeed = allspeed / self.averageSpeedCount
-
- def write_header(self, string):
- self.header += string
-
- def write_rep(self, buf):
- if self.rep.tell() > 400000 or self.abort:
- rep = self.get_rep()
- if self.abort: raise Abort
- f = open("response.dump", "wb")
- f.write(rep)
- f.close()
- raise Exception("Loaded Url exceeded limit")
-
- self.rep.write(buf)
-
- def get_rep(self):
- value = self.rep.getvalue()
- self.rep.close()
- self.rep = StringIO()
- return value
-
- def get_header(self):
- h = self.header
- self.header = ""
- return h
def get_speed(self):
try:
return self.dl_speed
- #return (self.dl_arrived / (time.time()- self.dl_time)) / 1024
except:
return 0
@@ -374,12 +147,7 @@ class Request:
def bytes_left(self):
return (self.dl_size - self.dl_arrived)
- def progress(self, dl_t, dl_d, up_t, up_d):
- if self.abort:
- return False
- self.dl_arrived = int(dl_d)+self.offset
- self.dl_size = int(dl_t)+self.offset
-
+ def progress(self):
if self.progressNotify:
try:
progress = int(float(self.dl_arrived)/self.dl_size*100)
@@ -400,22 +168,6 @@ class Request:
if not exists(temp_name):
file_name = temp_name
return file_name
-
- def __del__(self):
- self.clean()
- if hasattr(self, "pycurl"):
- del self.pycurl
- if hasattr(self, "cookieJar"):
- del self.cookieJar
-
- def clean(self):
- """ clean Reqest, its unusable after this """
- try:
- self.pycurl.close()
- if hasattr(self, "cookieJar"):
- del self.cookieJar
- except:
- pass
def __myquote(self, url):
return urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
@@ -425,10 +177,8 @@ def getURL(url, get={}, post={}):
"""
currently used for update check
"""
- req = Request()
- c = req.load(url, get, post)
- req.pycurl.close()
- return c
+ b = Browser()
+ return b.getPage(url, get=get, post=post)
if __name__ == "__main__":
import doctest
--
cgit v1.2.3