download speed limit

6 files changed, 135 insertions, 15 deletions

diff --git a/.hgignore b/.hgignore
index 9de870c0d..1af47aa3b 100644
--- a/.hgignore
+++ b/.hgignore
@@ -13,6 +13,7 @@ Logs/*
 Plugins/DLC.py
 failed_links.txt
 module/config/gui.xml
+module/config/core.xml
 links.txt
 module/links.pkl
 module/cookies.txt
diff --git a/module/XMLConfigParser.py b/module/XMLConfigParser.py
index b46f9c959..51741fdfd 100644
--- a/module/XMLConfigParser.py
+++ b/module/XMLConfigParser.py
@@ -17,18 +17,24 @@
 """
 from __future__ import with_statement
 
+from os.path import exists
+
 from xml.dom.minidom import parse
 
 class XMLConfigParser():
-    def __init__(self, data):
+    def __init__(self, data, default_data=None):
         self.xml = None
         self.file = data
+        self.file_default = default_data
         self.config = {}
         self.loadData()
         self.root = None
     
     def loadData(self):
-        with open(self.file, 'r') as fh:
+        file = self.file
+        if not exists(self.file):
+            file = self.file_default
+        with open(file, 'r') as fh:
             self.xml = parse(fh)
         self._read_config()
     
diff --git a/module/config/core.xml b/module/config/core_default.xml
index 6da85cf32..56ac2825c 100644
--- a/module/config/core.xml
+++ b/module/config/core_default.xml
@@ -37,6 +37,7 @@
         <reconnect_method>reconnect_method</reconnect_method>
         <debug_mode>False</debug_mode>
         <max_download_time>5</max_download_time>
+        <download_speed_limit>0</download_speed_limit>
     </general>
     <updates>
         <search_updates>True</search_updates>
diff --git a/module/network/Request.py b/module/network/Request.py
index 4aca935dc..752c16f04 100755
--- a/module/network/Request.py
+++ b/module/network/Request.py
@@ -51,14 +51,19 @@ class Request:
         self.timeout = 5
         
         bufferBase = 1024
-        bufferMulti = 2
+        bufferMulti = 4
         self.bufferSize = bufferBase*bufferMulti
         self.canContinue = False
         
         self.dl_speed = 0.0
+        self.averageSpeed = 0.0
+        self.averageSpeeds = []
+        self.averageSpeedTime = 0.0
+        self.averageSpeedCount = 0.0
         
         self.speedLimitActive = False
-        self.maxSpeed = 100 * 1024
+        self.maxSpeed = 0
+        self.isSlow = False
         
         try:
             if pycurl: self.curl = True
@@ -171,7 +176,7 @@ class Request:
                 self.add_cookies(req)
                 #add cookies
 
-            rep = self.opener.open(req, timeout=2)
+            rep = self.opener.open(req)
 
             for cookie in self.cj.make_cookies(rep, req):
                 self.cookies.append(cookie)
@@ -287,7 +292,7 @@ class Request:
                     else:
                         return -1
                 else:
-                    self.dl_speed = float(self.chunkRead/1024) / subTime
+                    self.updateCurrentSpeed(float(self.chunkRead/1024) / subTime)
                     
                     self.subStartTime = time.time()
                     self.chunkRead = 0
@@ -390,7 +395,7 @@ class Request:
                         else:
                             time.sleep(0.05)
                     subTime = time.time() - subStartTime
-                    self.dl_speed = float(chunkRead/1024) / subTime
+                    self.updateCurrentSpeed(float(chunkRead/1024) / subTime)
 
                 file.close()
                 if self.abort:
@@ -399,7 +404,24 @@ class Request:
                 self.dl_finished = time.time()
                 rename(file_temp, self.get_free_name(file_name))
                 return True
-
+    
+    def updateCurrentSpeed(self, speed):
+        self.dl_speed = speed
+        if self.averageSpeedTime + 10 < time.time():
+            self.averageSpeeds = []
+            self.averageSpeeds.append(self.averageSpeed)
+            self.averageSpeeds.append(speed)
+            self.averageSpeed = (speed + self.averageSpeed)/2
+            self.averageSpeedTime = time.time()
+            self.averageSpeedCount = 2
+        else:
+            self.averageSpeeds.append(speed)
+            self.averageSpeedCount += 1
+            allspeed = 0.0
+            for s in self.averageSpeeds:
+                allspeed += s
+            self.averageSpeed = allspeed / self.averageSpeedCount
+    
     def write_header(self, string):
         self.header += string
 
diff --git a/module/thread_list.py b/module/thread_list.py
index d78a9b95c..9cdb5fc8f 100644
--- a/module/thread_list.py
+++ b/module/thread_list.py
@@ -21,7 +21,7 @@ from __future__ import with_statement
 from os.path import exists
 import re
 import subprocess
-from threading import RLock
+from threading import RLock, Thread
 import time
 import urllib2
 
@@ -40,6 +40,7 @@ class Thread_List(object):
         self.reconnecting = False
 
         self.select_thread()
+        self.speedManager = self.SpeedManager(self)
 
     def create_thread(self):
         """ creates thread for Py_Load_File and append thread to self.threads
@@ -207,10 +208,94 @@ class Thread_List(object):
             out.wait()
 
     def scripts_download_finished(self, pluginname, url, filename, location):
-    	map(lambda script: subprocess.Popen([script, pluginname, url, filename, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished'])
+        map(lambda script: subprocess.Popen([script, pluginname, url, filename, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished'])
 
     def scripts_package_finished(self, name, location): #@TODO Implement!
-    	map(lambda script: subprocess.Popen([script, name, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished'])
+        map(lambda script: subprocess.Popen([script, name, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished'])
 
     def scripts_reconnected(self, ip):
-    	map(lambda script: subprocess.Popen([script, ip], stdout=subprocess.PIPE), self.parent.scripts['download_finished'])
+        map(lambda script: subprocess.Popen([script, ip], stdout=subprocess.PIPE), self.parent.scripts['download_finished'])
+    
+    class SpeedManager(Thread):
+        def __init__(self, parent):
+            Thread.__init__(self)
+            self.parent = parent
+            self.running = True
+            self.lastSlowCheck = 0.0
+            
+            stat = {}
+            stat["slow_downloads"] = None
+            stat["each_speed"] = None
+            stat["each_speed_optimized"] = None
+            self.stat = stat
+            
+            self.slowCheckInterval = 60
+            self.slowCheckTestTime = 25
+            
+            self.logger = self.parent.parent.logger
+            self.start()
+        
+        def run(self):
+            while self.running:
+                time.sleep(1)
+                self.manageSpeed()
+        
+        def getMaxSpeed(self):
+            return self.parent.parent.getMaxSpeed()
+        
+        def manageSpeed(self):
+            maxSpeed = self.getMaxSpeed()
+            if maxSpeed <= 0:
+                for thread in self.parent.py_downloading:
+                    thread.plugin.req.speedLimitActive = False
+                return
+            threads = self.parent.py_downloading
+            threadCount = len(threads)
+            if threadCount <= 0:
+                return
+            eachSpeed = maxSpeed/threadCount
+            
+            currentOverallSpeed = 0
+            restSpeed = maxSpeed - currentOverallSpeed
+            speeds = []
+            for thread in threads:
+                currentOverallSpeed += thread.plugin.req.dl_speed
+                speeds.append((thread.plugin.req.dl_speed, thread.plugin.req.averageSpeed, thread))
+                thread.plugin.req.speedLimitActive = True
+            
+            if currentOverallSpeed+50 < maxSpeed:
+                for thread in self.parent.py_downloading:
+                    thread.plugin.req.speedLimitActive = False
+                return
+                
+            print "-----"
+            
+            slowCount = 0
+            slowSpeed = 0
+            if self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime < time.time():
+                self.lastSlowCheck = time.time()
+            if self.lastSlowCheck + self.slowCheckInterval < time.time() < self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime:
+                for speed in speeds:
+                    speed[2].plugin.req.isSlow = False
+            else:
+                for speed in speeds:
+                    if speed[0] <= eachSpeed-7:
+                        if speed[1] < eachSpeed-15:
+                            if speed[2].plugin.req.dl_time > 0 and speed[2].plugin.req.dl_time+30 < time.time():
+                                speed[2].plugin.req.isSlow = True
+                                if not speed[1]-5 < speed[2].plugin.req.maxSpeed/1024 < speed[1]+5:
+                                    speed[2].plugin.req.maxSpeed = (speed[1]+10)*1024
+                    if speed[2].plugin.req.isSlow:
+                        slowCount += 1
+                        slowSpeed += speed[2].plugin.req.maxSpeed/1024
+            
+            stat["slow_downloads"] = slowCount
+            stat["each_speed"] = eachSpeed
+            eachSpeed = (maxSpeed - slowSpeed) / (threadCount - slowCount)
+            stat["each_speed_optimized"] = eachSpeed
+            for speed in speeds:
+                if speed[2].plugin.req.isSlow:
+                    continue
+                speed[2].plugin.req.maxSpeed = eachSpeed*1024
+                print "max", speed[2].plugin.req.maxSpeed, "current", speed[2].plugin.req.dl_speed
+
diff --git a/pyLoadCore.py b/pyLoadCore.py
index 232adc253..85ca5a5fe 100644
--- a/pyLoadCore.py
+++ b/pyLoadCore.py
@@ -90,7 +90,7 @@ class Core(object):
 

         self.plugin_folder = join("module", "plugins")

         

-        self.xmlconfig = XMLConfigParser(join(self.path,"module","config","core.xml"))

+        self.xmlconfig = XMLConfigParser(join(self.path,"module","config","core.xml"), join(self.path,"module","config","core_default.xml"))

         self.config = self.xmlconfig.getConfig()

         

         self.do_kill = False

@@ -116,7 +116,9 @@ class Core(object):
         if self.config['ssl']['activated']:

             self.check_install("OpenSSL", "OpenSSL for secure connection", True)

             self.check_file(self.config['ssl']['cert'], _("ssl certificate"), False, True)

-            self.check_file(self.config['ssl']['key'], _("ssl key"), False, True)

+            self.check_file(self.config['ssl']['key'], _("ssl key"), False, True)
+        
+        self.downloadSpeedLimit = int(self.xmlconfig.get("general", "download_speed_limit", 0))

 

         if self.config['general']['debug_mode']:

             self.init_logger(logging.DEBUG) # logging level

@@ -307,7 +309,10 @@ class Core(object):
         if start < now and end > now: return True

         elif start > end and (now > start or now < end): return True

         elif start < now and end < now and start > end: return True

-        else: return False

+        else: return False
+    
+    def getMaxSpeed(self):
+        return self.downloadSpeedLimit

         

     ####################################

     ########## XMLRPC Methods ##########