changed request files

4 files changed, 487 insertions, 484 deletions

diff --git a/module/network/Browser.py b/module/network/Browser.py
index c0742c4ff..d70cb3aba 100644
--- a/module/network/Browser.py
+++ b/module/network/Browser.py
@@ -78,13 +78,16 @@ class Browser():
         return d
 
 if __name__ == "__main__":
-    browser = Browser(proxies={"socks5": "localhost:5000"})
+    browser = Browser()#proxies={"socks5": "localhost:5000"})
     ip = "http://www.whatismyip.com/automation/n09230945.asp"
     #browser.getPage("http://google.com/search?q=bar")
     #browser.getPage("https://encrypted.google.com/")
-    print browser.getPage(ip)
+    #print browser.getPage(ip)
     #print browser.getRedirectLocation("http://google.com/")
     #browser.getPage("https://encrypted.google.com/")
     #browser.getPage("http://google.com/search?q=bar")
     
-    #browser.downloadFile("http://speedtest.netcologne.de/test_100mb.bin", "test_100mb.bin")
+    browser.httpDownload("http://speedtest.netcologne.de/test_100mb.bin", "test_100mb.bin")
+    from time import sleep
+    while True:
+        sleep(1)
\ No newline at end of file
diff --git a/module/network/NewRequest.py b/module/network/NewRequest.py
deleted file mode 100755
index 9ac7d54aa..000000000
--- a/module/network/NewRequest.py
+++ /dev/null
@@ -1,185 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 3 of the License,
-    or (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-    See the GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, see <http://www.gnu.org/licenses/>.
-    
-    @author: spoob
-    @author: RaNaN
-    @author: mkaay
-"""
-
-import time
-from os.path import exists, join
-from shutil import move
-import urllib
-
-from module.plugins.Plugin import Abort
-
-from module.network.Browser import Browser
-from module.network.helper import waitFor
-
-class Request:
-    def __init__(self, interface=None):
-        self.browser = Browser(interface=interface)
-        self.d = None
-        
-        self.dl_time = 0
-        self.dl_finished = 0
-        self.dl_size = 0
-        self.dl_arrived = 0
-        self.dl = False
-
-        self.abort = False
-
-        self.lastEffectiveURL = self.lastURL = property(lambda: self.browser.lastUrl)
-        self.auth = False
-        
-        self.canContinue = False
-        
-        self.dl_speed = 0.0
-        
-        self.cookieJar = None
-        self.interface = interface
-        self.progressNotify = None
-        
-        # change this for connection information
-        self.debug = False
-
-    def set_timeout(self, timeout):
-        self.timeout = int(timeout)
-    
-    def setCookieJar(self, j):
-        self.cookieJar = j
-    
-    def addCookies(self):
-        #@TODO
-        pass
-    
-    def getCookies(self):
-        #@TODO
-        pass
-    
-    def getCookie(self, name):
-        #@TODO
-        pass
-    
-    def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, no_post_encode=False, raw_cookies={}):
-        url = self.__myquote(str(url))
-        
-        #@TODO: cookies
-        #@TODO: auth
-        
-        if not ref:
-            self.browser.clearReferer()
-
-        return self.browser.getPage(url, get=get, post=post, cookies=cookies)
-
-    def add_auth(self, user, pw):
-        #@TODO
-        pass
-
-    def clearCookies(self):
-        #@TODO
-        pass
-
-    def add_proxy(self, protocol, adress):
-        #@TODO
-        pass
-
-    def download(self, url, file_name, folder, get={}, post={}, ref=True, cookies=True, no_post_encode=False):
-        url = self.__myquote(str(url))
-        
-        file_temp = self.get_free_name(folder,file_name)
-      
-        #@TODO: cookies
-        #@TODO: auth
-        
-        if not ref:
-            self.browser.clearReferer()
-        
-        self.d = self.browser.httpDownload(url, file_temp, get=get, post=post, cookies=cookies, chunks=1, resume=self.canContinue)
-        self.dl_time = property(lambda: self.d.startTime)
-        self.dl_finished = property(lambda: self.d.endTime)
-        self.dl_speed = property(lambda: self.d.speed)
-        self.dl_size = property(lambda: self.d.size)
-        self.dl = property(lambda: True if self.d.startTime and not self.d.endTime else False)
-        self.abort = property(self.d.getAbort, self.d.setAbort)
-        
-        waitFor(self.d)
-        
-        if self.abort: raise Abort
-
-        free_name = self.get_free_name(folder, file_name)
-        move(file_temp, free_name)
-        
-        self.dl_time = 0
-        self.dl_finished = 0
-        self.dl_size = 0
-        self.dl_arrived = 0
-        self.dl = False
-        self.dl_speed = 0.0
-        
-        return free_name
-
-    def get_speed(self):
-        try:
-            return self.dl_speed
-        except:
-            return 0
-
-    def get_ETA(self):
-        try:
-            return (self.dl_size - self.dl_arrived) / (self.dl_arrived / (time.time() - self.dl_time))
-        except:
-            return 0
-
-    def bytes_left(self):
-        return (self.dl_size - self.dl_arrived)
-    
-    def progress(self):
-        if self.progressNotify:
-            try:
-                progress = int(float(self.dl_arrived)/self.dl_size*100)
-                self.progressNotify(progress)
-            except:
-                pass
-        
-    def get_free_name(self, folder, file_name):
-        file_count = 0
-        file_name = join(folder, file_name)
-        while exists(file_name):
-            file_count += 1
-            if "." in file_name:
-                file_split = file_name.split(".")
-                temp_name = "%s-%i.%s" % (".".join(file_split[:-1]), file_count, file_split[-1])
-            else:
-                temp_name = "%s-%i" % (file_name, file_count)
-            if not exists(temp_name):
-                file_name = temp_name
-        return file_name
-            
-    def __myquote(self, url):
-        return urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
-
-
-def getURL(url, get={}, post={}):
-    """
-        currently used for update check
-    """
-    b = Browser()
-    return b.getPage(url, get=get, post=post)
-
-if __name__ == "__main__":
-    import doctest
-    doctest.testmod()
diff --git a/module/network/OldRequest.py b/module/network/OldRequest.py
new file mode 100755
index 000000000..5314f97ab
--- /dev/null
+++ b/module/network/OldRequest.py
@@ -0,0 +1,435 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+    
+    @author: spoob
+    @author: RaNaN
+    @author: mkaay
+"""
+
+import base64
+import time
+from os.path import exists, join
+from shutil import move
+import urllib
+from cStringIO import StringIO
+import pycurl
+
+from module.plugins.Plugin import Abort
+
+class Request:
+    def __init__(self, interface=None):
+
+        self.dl_time = 0
+        self.dl_finished = 0
+        self.dl_size = 0
+        self.dl_arrived = 0
+        self.dl = False
+
+        self.abort = False
+
+        self.lastEffectiveURL = None
+        self.lastURL = None
+        self.auth = False
+        
+        self.timeout = 5
+        
+        bufferBase = 1024
+        bufferMulti = 4
+        self.bufferSize = bufferBase*bufferMulti
+        self.canContinue = False
+        self.offset = 0
+        
+        self.dl_speed = 0.0
+        self.averageSpeed = 0.0
+        self.averageSpeeds = []
+        self.averageSpeedTime = 0.0
+        self.averageSpeedCount = 0.0
+        
+        self.speedLimitActive = False
+        self.maxSpeed = 0
+        self.isSlow = False
+        self.cookieJar = None
+        self.interface = interface
+        self.progressNotify = None
+        
+        # change this for connection information
+        self.debug = False
+
+        self.init_curl()
+
+    def set_timeout(self, timeout):
+        self.timeout = int(timeout)
+
+    def init_curl(self):
+        self.rep = StringIO()
+        self.header = ""
+        
+        self.pycurl = pycurl.Curl()
+        self.pycurl.setopt(pycurl.FOLLOWLOCATION, 1)
+        self.pycurl.setopt(pycurl.MAXREDIRS, 5)
+        self.pycurl.setopt(pycurl.TIMEOUT, (self.timeout*3600))
+        self.pycurl.setopt(pycurl.CONNECTTIMEOUT, 30)
+        self.pycurl.setopt(pycurl.NOSIGNAL, 1)
+        self.pycurl.setopt(pycurl.NOPROGRESS, 0)
+        self.pycurl.setopt(pycurl.PROGRESSFUNCTION, self.progress)
+        if hasattr(pycurl, "AUTOREFERER"):
+            self.pycurl.setopt(pycurl.AUTOREFERER, 1)
+        self.pycurl.setopt(pycurl.HEADERFUNCTION, self.write_header)
+        #self.pycurl.setopt(pycurl.BUFFERSIZE, self.bufferSize)
+        self.pycurl.setopt(pycurl.SSL_VERIFYPEER, 0)
+        self.pycurl.setopt(pycurl.LOW_SPEED_TIME, 30)
+        self.pycurl.setopt(pycurl.LOW_SPEED_LIMIT, 20)
+        
+        if self.debug:
+            self.pycurl.setopt(pycurl.VERBOSE, 1)
+        if self.interface and self.interface.lower() != "none":
+            self.pycurl.setopt(pycurl.INTERFACE, self.interface)
+
+
+        self.pycurl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.2.10) Gecko/20100916 Firefox/3.6.10")
+        if pycurl.version_info()[7]:
+            self.pycurl.setopt(pycurl.ENCODING, "gzip, deflate")
+        self.pycurl.setopt(pycurl.HTTPHEADER, ["Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                           "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7",
+                           "Connection: keep-alive",
+                           "Keep-Alive: 300"])
+    
+    def setCookieJar(self, j):
+        self.cookieJar = j
+    
+    def addCookies(self):
+        if self.cookieJar:
+            self.cookieJar.addCookies(self.pycurl.getinfo(pycurl.INFO_COOKIELIST))
+        return
+    
+    def getCookies(self):
+        if self.cookieJar:
+            for c in self.cookieJar.getCookies():
+                self.pycurl.setopt(pycurl.COOKIELIST, c)
+        return
+    
+    def getCookie(self, name):
+        if self.cookieJar:
+            return self.cookieJar.getCookie(name)
+        return None
+    
+    def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, no_post_encode=False, raw_cookies={}):
+
+        self.pycurl.setopt(pycurl.NOPROGRESS, 1)
+        
+        url = self.__myquote(str(url))
+
+        if post:
+            if not no_post_encode:
+                post = urllib.urlencode(post)
+        else:
+            post = None
+
+        if get:
+            get = urllib.urlencode(get)
+            url = "%s?%s" % (url, get)
+        else:
+            get = ""
+
+        self.pycurl.setopt(pycurl.URL, url)
+        self.pycurl.setopt(pycurl.WRITEFUNCTION, self.write_rep)
+        
+        if cookies:
+            self.curl_enable_cookies()
+            self.getCookies()
+            if raw_cookies:
+                self.pycurl.setopt(pycurl.COOKIELIST, "Set-Cookie: %s;"% "; ".join(["%s=%s"%(x,y) for x,y in raw_cookies.items()]))
+
+        if post:
+            self.pycurl.setopt(pycurl.POSTFIELDS, post)
+
+        if ref and self.lastURL is not None:
+            self.pycurl.setopt(pycurl.REFERER, self.lastURL)
+
+        if just_header:
+            self.pycurl.setopt(pycurl.NOBODY, 1)
+            self.pycurl.perform()
+            self.lastEffectiveURL = self.pycurl.getinfo(pycurl.EFFECTIVE_URL)
+            self.pycurl.setopt(pycurl.NOPROGRESS, 0)
+            self.pycurl.setopt(pycurl.NOBODY, 0)
+            return self.header
+
+        self.pycurl.perform()
+        
+        self.lastEffectiveURL = self.pycurl.getinfo(pycurl.EFFECTIVE_URL)
+        self.addCookies()
+        
+        #reset progress
+        
+        self.dl_time = 0
+        self.dl_finished = 0
+        self.dl_size = 0
+        self.dl_arrived = 0
+        
+        self.lastURL = url
+        header = self.get_header()
+
+        return self.get_rep()
+    
+    def curl_enable_cookies(self):
+        self.pycurl.setopt(pycurl.COOKIEFILE, "")
+        self.pycurl.setopt(pycurl.COOKIEJAR, "")
+
+    def add_auth(self, user, pw):
+        
+        self.auth = True
+        self.user = user
+        self.pw = pw
+        
+        upwstr = str("%s:%s" % (user,pw))
+        self.pycurl.setopt(pycurl.HTTPHEADER, ['Authorization: Basic ' + base64.encodestring(upwstr)[:-1]])
+        self.pycurl.setopt(pycurl.USERPWD, upwstr)
+        self.pycurl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_ANY)
+
+    def clearCookies(self):
+        self.pycurl.setopt(pycurl.COOKIELIST, "")
+
+    def add_proxy(self, protocol, adress):
+        # @TODO: pycurl proxy protocol selection
+        self.pycurl.setopt(pycurl.PROXY, adress.split(":")[0])
+        self.pycurl.setopt(pycurl.PROXYPORT, adress.split(":")[1])
+
+    def download(self, url, file_name, folder, get={}, post={}, ref=True, cookies=True, no_post_encode=False):
+
+        url = self.__myquote(str(url))
+        
+        self.pycurl.setopt(pycurl.NOPROGRESS, 0)
+        
+        if post:
+            if not no_post_encode:
+                post = urllib.urlencode(post)
+        else:
+            post = None
+
+        if get:
+            get = urllib.urlencode(get)
+            url = "%s?%s" % (url, get)
+        else:
+            get = ""
+
+        file_temp = self.get_free_name(folder,file_name) + ".part"
+      
+        self.fp = open(file_temp, 'wb' if not self.canContinue else 'ab')
+
+        partSize = self.fp.tell()
+
+        self.init_curl()
+
+        self.pycurl.setopt(pycurl.URL, url)
+        
+        if self.canContinue:
+            self.offset = partSize
+            self.pycurl.setopt(pycurl.RESUME_FROM, self.offset)
+            
+        self.dl_arrived = self.offset
+        
+        if cookies:
+            self.curl_enable_cookies()
+            self.getCookies()
+
+        if post:
+            self.pycurl.setopt(pycurl.POSTFIELDS, post)
+        
+        if self.auth:
+            self.add_auth(self.user, self.pw)
+
+        if ref and self.lastURL is not None:
+            self.pycurl.setopt(pycurl.REFERER, self.lastURL)
+
+        self.dl_time = time.time()
+        self.dl = True
+        
+        self.chunkSize = 0
+        self.chunkRead = 0
+        self.subStartTime = 0
+        self.maxChunkSize = 0
+        
+        def restLimit():
+            subTime = time.time() - self.subStartTime
+            if subTime <= 1:
+                if self.speedLimitActive:
+                    return self.maxChunkSize
+                else:
+                    return -1
+            else:
+                self.updateCurrentSpeed(float(self.chunkRead/1024) / subTime)
+                
+                self.subStartTime += subTime # time.time()
+                self.chunkRead = 0
+                if self.maxSpeed > 0:
+                    self.maxChunkSize = self.maxSpeed
+                else:
+                    self.maxChunkSize = 0
+                return 0
+
+        def writefunc(buf):
+            if self.abort:
+                return False
+            chunkSize = len(buf)
+            while chunkSize > restLimit() > -1:
+                time.sleep(0.05)
+            self.maxChunkSize -= chunkSize
+            self.fp.write(buf)
+            self.chunkRead += chunkSize
+            self.dl_arrived += chunkSize
+            
+        self.pycurl.setopt(pycurl.WRITEFUNCTION, writefunc)
+        #self.pycurl.setopt(pycurl.WRITEDATA, self.fp)
+
+        try:
+            self.pycurl.perform()
+        except Exception, e:
+            code, msg = e
+            if not code == 23:
+                raise Exception, e
+        finally:
+            self.dl = False
+            self.dl_finished = time.time()
+                
+        self.addCookies()
+        self.fp.close()
+        
+        if self.abort: raise Abort
+
+        free_name = self.get_free_name(folder, file_name)
+        move(file_temp, free_name)
+        
+        #@TODO content disposition
+        
+        return free_name
+    
+    def updateCurrentSpeed(self, speed):
+        self.dl_speed = speed
+        if self.averageSpeedTime + 10 < time.time():
+            self.averageSpeeds = []
+            self.averageSpeeds.append(self.averageSpeed)
+            self.averageSpeeds.append(speed)
+            self.averageSpeed = (speed + self.averageSpeed)/2
+            self.averageSpeedTime = time.time()
+            self.averageSpeedCount = 2
+        else:
+            self.averageSpeeds.append(speed)
+            self.averageSpeedCount += 1
+            allspeed = 0.0
+            for s in self.averageSpeeds:
+                allspeed += s
+            self.averageSpeed = allspeed / self.averageSpeedCount
+    
+    def write_header(self, string):
+        self.header += string
+
+    def write_rep(self, buf):
+        if self.rep.tell() > 400000 or self.abort:
+            rep = self.get_rep()
+            if self.abort: raise Abort
+            f = open("response.dump", "wb")
+            f.write(rep)
+            f.close()
+            raise Exception("Loaded Url exceeded limit")
+
+        self.rep.write(buf)
+
+    def get_rep(self):  
+        value = self.rep.getvalue()
+        self.rep.close()
+        self.rep = StringIO()
+        return value
+    
+    def get_header(self):
+        h = self.header
+        self.header = ""
+        return h
+
+    def get_speed(self):
+        try:
+            return self.dl_speed
+            #return (self.dl_arrived / (time.time()- self.dl_time)) / 1024
+        except:
+            return 0
+
+    def get_ETA(self):
+        try:
+            return (self.dl_size - self.dl_arrived) / (self.dl_arrived / (time.time() - self.dl_time))
+        except:
+            return 0
+
+    def bytes_left(self):
+        return (self.dl_size - self.dl_arrived)
+    
+    def progress(self, dl_t, dl_d, up_t, up_d):
+        if self.abort:
+            return False
+        self.dl_arrived = int(dl_d)+self.offset
+        self.dl_size = int(dl_t)+self.offset
+        
+        if self.progressNotify:
+            try:
+                progress = int(float(self.dl_arrived)/self.dl_size*100)
+                self.progressNotify(progress)
+            except:
+                pass
+        
+    def get_free_name(self, folder, file_name):
+        file_count = 0
+        file_name = join(folder, file_name)
+        while exists(file_name):
+            file_count += 1
+            if "." in file_name:
+                file_split = file_name.split(".")
+                temp_name = "%s-%i.%s" % (".".join(file_split[:-1]), file_count, file_split[-1])
+            else:
+                temp_name = "%s-%i" % (file_name, file_count)
+            if not exists(temp_name):
+                file_name = temp_name
+        return file_name
+    
+    def __del__(self):
+        self.clean()
+        if hasattr(self, "pycurl"):
+            del self.pycurl
+        if hasattr(self, "cookieJar"):
+            del self.cookieJar
+    
+    def clean(self):
+        """ clean Reqest, its unusable after this """
+        try:
+            self.pycurl.close()
+            if hasattr(self, "cookieJar"):
+                del self.cookieJar
+        except:
+            pass
+            
+    def __myquote(self, url):
+        return urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
+
+
+def getURL(url, get={}, post={}):
+    """
+        currently used for update check
+    """
+    req = Request()
+    c = req.load(url, get, post)
+    req.pycurl.close()
+    return c
+
+if __name__ == "__main__":
+    import doctest
+    doctest.testmod()
diff --git a/module/network/Request.py b/module/network/Request.py
index 5314f97ab..9ac7d54aa 100755
--- a/module/network/Request.py
+++ b/module/network/Request.py
@@ -19,19 +19,21 @@
     @author: mkaay
 """
 
-import base64
 import time
 from os.path import exists, join
 from shutil import move
 import urllib
-from cStringIO import StringIO
-import pycurl
 
 from module.plugins.Plugin import Abort
 
+from module.network.Browser import Browser
+from module.network.helper import waitFor
+
 class Request:
     def __init__(self, interface=None):
-
+        self.browser = Browser(interface=interface)
+        self.d = None
+        
         self.dl_time = 0
         self.dl_finished = 0
         self.dl_size = 0
@@ -40,27 +42,13 @@ class Request:
 
         self.abort = False
 
-        self.lastEffectiveURL = None
-        self.lastURL = None
+        self.lastEffectiveURL = self.lastURL = property(lambda: self.browser.lastUrl)
         self.auth = False
         
-        self.timeout = 5
-        
-        bufferBase = 1024
-        bufferMulti = 4
-        self.bufferSize = bufferBase*bufferMulti
         self.canContinue = False
-        self.offset = 0
         
         self.dl_speed = 0.0
-        self.averageSpeed = 0.0
-        self.averageSpeeds = []
-        self.averageSpeedTime = 0.0
-        self.averageSpeedCount = 0.0
         
-        self.speedLimitActive = False
-        self.maxSpeed = 0
-        self.isSlow = False
         self.cookieJar = None
         self.interface = interface
         self.progressNotify = None
@@ -68,300 +56,85 @@ class Request:
         # change this for connection information
         self.debug = False
 
-        self.init_curl()
-
     def set_timeout(self, timeout):
         self.timeout = int(timeout)
-
-    def init_curl(self):
-        self.rep = StringIO()
-        self.header = ""
-        
-        self.pycurl = pycurl.Curl()
-        self.pycurl.setopt(pycurl.FOLLOWLOCATION, 1)
-        self.pycurl.setopt(pycurl.MAXREDIRS, 5)
-        self.pycurl.setopt(pycurl.TIMEOUT, (self.timeout*3600))
-        self.pycurl.setopt(pycurl.CONNECTTIMEOUT, 30)
-        self.pycurl.setopt(pycurl.NOSIGNAL, 1)
-        self.pycurl.setopt(pycurl.NOPROGRESS, 0)
-        self.pycurl.setopt(pycurl.PROGRESSFUNCTION, self.progress)
-        if hasattr(pycurl, "AUTOREFERER"):
-            self.pycurl.setopt(pycurl.AUTOREFERER, 1)
-        self.pycurl.setopt(pycurl.HEADERFUNCTION, self.write_header)
-        #self.pycurl.setopt(pycurl.BUFFERSIZE, self.bufferSize)
-        self.pycurl.setopt(pycurl.SSL_VERIFYPEER, 0)
-        self.pycurl.setopt(pycurl.LOW_SPEED_TIME, 30)
-        self.pycurl.setopt(pycurl.LOW_SPEED_LIMIT, 20)
-        
-        if self.debug:
-            self.pycurl.setopt(pycurl.VERBOSE, 1)
-        if self.interface and self.interface.lower() != "none":
-            self.pycurl.setopt(pycurl.INTERFACE, self.interface)
-
-
-        self.pycurl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.2.10) Gecko/20100916 Firefox/3.6.10")
-        if pycurl.version_info()[7]:
-            self.pycurl.setopt(pycurl.ENCODING, "gzip, deflate")
-        self.pycurl.setopt(pycurl.HTTPHEADER, ["Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-                           "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7",
-                           "Connection: keep-alive",
-                           "Keep-Alive: 300"])
     
     def setCookieJar(self, j):
         self.cookieJar = j
     
     def addCookies(self):
-        if self.cookieJar:
-            self.cookieJar.addCookies(self.pycurl.getinfo(pycurl.INFO_COOKIELIST))
-        return
+        #@TODO
+        pass
     
     def getCookies(self):
-        if self.cookieJar:
-            for c in self.cookieJar.getCookies():
-                self.pycurl.setopt(pycurl.COOKIELIST, c)
-        return
+        #@TODO
+        pass
     
     def getCookie(self, name):
-        if self.cookieJar:
-            return self.cookieJar.getCookie(name)
-        return None
+        #@TODO
+        pass
     
     def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, no_post_encode=False, raw_cookies={}):
-
-        self.pycurl.setopt(pycurl.NOPROGRESS, 1)
-        
         url = self.__myquote(str(url))
-
-        if post:
-            if not no_post_encode:
-                post = urllib.urlencode(post)
-        else:
-            post = None
-
-        if get:
-            get = urllib.urlencode(get)
-            url = "%s?%s" % (url, get)
-        else:
-            get = ""
-
-        self.pycurl.setopt(pycurl.URL, url)
-        self.pycurl.setopt(pycurl.WRITEFUNCTION, self.write_rep)
-        
-        if cookies:
-            self.curl_enable_cookies()
-            self.getCookies()
-            if raw_cookies:
-                self.pycurl.setopt(pycurl.COOKIELIST, "Set-Cookie: %s;"% "; ".join(["%s=%s"%(x,y) for x,y in raw_cookies.items()]))
-
-        if post:
-            self.pycurl.setopt(pycurl.POSTFIELDS, post)
-
-        if ref and self.lastURL is not None:
-            self.pycurl.setopt(pycurl.REFERER, self.lastURL)
-
-        if just_header:
-            self.pycurl.setopt(pycurl.NOBODY, 1)
-            self.pycurl.perform()
-            self.lastEffectiveURL = self.pycurl.getinfo(pycurl.EFFECTIVE_URL)
-            self.pycurl.setopt(pycurl.NOPROGRESS, 0)
-            self.pycurl.setopt(pycurl.NOBODY, 0)
-            return self.header
-
-        self.pycurl.perform()
         
-        self.lastEffectiveURL = self.pycurl.getinfo(pycurl.EFFECTIVE_URL)
-        self.addCookies()
-        
-        #reset progress
-        
-        self.dl_time = 0
-        self.dl_finished = 0
-        self.dl_size = 0
-        self.dl_arrived = 0
+        #@TODO: cookies
+        #@TODO: auth
         
-        self.lastURL = url
-        header = self.get_header()
+        if not ref:
+            self.browser.clearReferer()
 
-        return self.get_rep()
-    
-    def curl_enable_cookies(self):
-        self.pycurl.setopt(pycurl.COOKIEFILE, "")
-        self.pycurl.setopt(pycurl.COOKIEJAR, "")
+        return self.browser.getPage(url, get=get, post=post, cookies=cookies)
 
     def add_auth(self, user, pw):
-        
-        self.auth = True
-        self.user = user
-        self.pw = pw
-        
-        upwstr = str("%s:%s" % (user,pw))
-        self.pycurl.setopt(pycurl.HTTPHEADER, ['Authorization: Basic ' + base64.encodestring(upwstr)[:-1]])
-        self.pycurl.setopt(pycurl.USERPWD, upwstr)
-        self.pycurl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_ANY)
+        #@TODO
+        pass
 
     def clearCookies(self):
-        self.pycurl.setopt(pycurl.COOKIELIST, "")
+        #@TODO
+        pass
 
     def add_proxy(self, protocol, adress):
-        # @TODO: pycurl proxy protocol selection
-        self.pycurl.setopt(pycurl.PROXY, adress.split(":")[0])
-        self.pycurl.setopt(pycurl.PROXYPORT, adress.split(":")[1])
+        #@TODO
+        pass
 
     def download(self, url, file_name, folder, get={}, post={}, ref=True, cookies=True, no_post_encode=False):
-
         url = self.__myquote(str(url))
         
-        self.pycurl.setopt(pycurl.NOPROGRESS, 0)
-        
-        if post:
-            if not no_post_encode:
-                post = urllib.urlencode(post)
-        else:
-            post = None
-
-        if get:
-            get = urllib.urlencode(get)
-            url = "%s?%s" % (url, get)
-        else:
-            get = ""
-
-        file_temp = self.get_free_name(folder,file_name) + ".part"
+        file_temp = self.get_free_name(folder,file_name)
       
-        self.fp = open(file_temp, 'wb' if not self.canContinue else 'ab')
-
-        partSize = self.fp.tell()
-
-        self.init_curl()
-
-        self.pycurl.setopt(pycurl.URL, url)
-        
-        if self.canContinue:
-            self.offset = partSize
-            self.pycurl.setopt(pycurl.RESUME_FROM, self.offset)
-            
-        self.dl_arrived = self.offset
-        
-        if cookies:
-            self.curl_enable_cookies()
-            self.getCookies()
-
-        if post:
-            self.pycurl.setopt(pycurl.POSTFIELDS, post)
+        #@TODO: cookies
+        #@TODO: auth
         
-        if self.auth:
-            self.add_auth(self.user, self.pw)
-
-        if ref and self.lastURL is not None:
-            self.pycurl.setopt(pycurl.REFERER, self.lastURL)
-
-        self.dl_time = time.time()
-        self.dl = True
+        if not ref:
+            self.browser.clearReferer()
         
-        self.chunkSize = 0
-        self.chunkRead = 0
-        self.subStartTime = 0
-        self.maxChunkSize = 0
+        self.d = self.browser.httpDownload(url, file_temp, get=get, post=post, cookies=cookies, chunks=1, resume=self.canContinue)
+        self.dl_time = property(lambda: self.d.startTime)
+        self.dl_finished = property(lambda: self.d.endTime)
+        self.dl_speed = property(lambda: self.d.speed)
+        self.dl_size = property(lambda: self.d.size)
+        self.dl = property(lambda: True if self.d.startTime and not self.d.endTime else False)
+        self.abort = property(self.d.getAbort, self.d.setAbort)
         
-        def restLimit():
-            subTime = time.time() - self.subStartTime
-            if subTime <= 1:
-                if self.speedLimitActive:
-                    return self.maxChunkSize
-                else:
-                    return -1
-            else:
-                self.updateCurrentSpeed(float(self.chunkRead/1024) / subTime)
-                
-                self.subStartTime += subTime # time.time()
-                self.chunkRead = 0
-                if self.maxSpeed > 0:
-                    self.maxChunkSize = self.maxSpeed
-                else:
-                    self.maxChunkSize = 0
-                return 0
-
-        def writefunc(buf):
-            if self.abort:
-                return False
-            chunkSize = len(buf)
-            while chunkSize > restLimit() > -1:
-                time.sleep(0.05)
-            self.maxChunkSize -= chunkSize
-            self.fp.write(buf)
-            self.chunkRead += chunkSize
-            self.dl_arrived += chunkSize
-            
-        self.pycurl.setopt(pycurl.WRITEFUNCTION, writefunc)
-        #self.pycurl.setopt(pycurl.WRITEDATA, self.fp)
-
-        try:
-            self.pycurl.perform()
-        except Exception, e:
-            code, msg = e
-            if not code == 23:
-                raise Exception, e
-        finally:
-            self.dl = False
-            self.dl_finished = time.time()
-                
-        self.addCookies()
-        self.fp.close()
+        waitFor(self.d)
         
         if self.abort: raise Abort
 
         free_name = self.get_free_name(folder, file_name)
         move(file_temp, free_name)
         
-        #@TODO content disposition
+        self.dl_time = 0
+        self.dl_finished = 0
+        self.dl_size = 0
+        self.dl_arrived = 0
+        self.dl = False
+        self.dl_speed = 0.0
         
         return free_name
-    
-    def updateCurrentSpeed(self, speed):
-        self.dl_speed = speed
-        if self.averageSpeedTime + 10 < time.time():
-            self.averageSpeeds = []
-            self.averageSpeeds.append(self.averageSpeed)
-            self.averageSpeeds.append(speed)
-            self.averageSpeed = (speed + self.averageSpeed)/2
-            self.averageSpeedTime = time.time()
-            self.averageSpeedCount = 2
-        else:
-            self.averageSpeeds.append(speed)
-            self.averageSpeedCount += 1
-            allspeed = 0.0
-            for s in self.averageSpeeds:
-                allspeed += s
-            self.averageSpeed = allspeed / self.averageSpeedCount
-    
-    def write_header(self, string):
-        self.header += string
-
-    def write_rep(self, buf):
-        if self.rep.tell() > 400000 or self.abort:
-            rep = self.get_rep()
-            if self.abort: raise Abort
-            f = open("response.dump", "wb")
-            f.write(rep)
-            f.close()
-            raise Exception("Loaded Url exceeded limit")
-
-        self.rep.write(buf)
-
-    def get_rep(self):  
-        value = self.rep.getvalue()
-        self.rep.close()
-        self.rep = StringIO()
-        return value
-    
-    def get_header(self):
-        h = self.header
-        self.header = ""
-        return h
 
     def get_speed(self):
         try:
             return self.dl_speed
-            #return (self.dl_arrived / (time.time()- self.dl_time)) / 1024
         except:
             return 0
 
@@ -374,12 +147,7 @@ class Request:
     def bytes_left(self):
         return (self.dl_size - self.dl_arrived)
     
-    def progress(self, dl_t, dl_d, up_t, up_d):
-        if self.abort:
-            return False
-        self.dl_arrived = int(dl_d)+self.offset
-        self.dl_size = int(dl_t)+self.offset
-        
+    def progress(self):
         if self.progressNotify:
             try:
                 progress = int(float(self.dl_arrived)/self.dl_size*100)
@@ -400,22 +168,6 @@ class Request:
             if not exists(temp_name):
                 file_name = temp_name
         return file_name
-    
-    def __del__(self):
-        self.clean()
-        if hasattr(self, "pycurl"):
-            del self.pycurl
-        if hasattr(self, "cookieJar"):
-            del self.cookieJar
-    
-    def clean(self):
-        """ clean Reqest, its unusable after this """
-        try:
-            self.pycurl.close()
-            if hasattr(self, "cookieJar"):
-                del self.cookieJar
-        except:
-            pass
             
     def __myquote(self, url):
         return urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
@@ -425,10 +177,8 @@ def getURL(url, get={}, post={}):
     """
         currently used for update check
     """
-    req = Request()
-    c = req.load(url, get, post)
-    req.pycurl.close()
-    return c
+    b = Browser()
+    return b.getPage(url, get=get, post=post)
 
 if __name__ == "__main__":
     import doctest