working speedlimit + proxy support, closed #197

11 files changed, 80 insertions, 179 deletions

diff --git a/module/SpeedManager.py b/module/SpeedManager.py
deleted file mode 100644
index 44d59dad4..000000000
--- a/module/SpeedManager.py
+++ /dev/null
@@ -1,111 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 3 of the License,
-    or (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-    See the GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, see <http://www.gnu.org/licenses/>.
-    
-    @author: mkaay
-    @version: v0.3.2
-"""
-
-from threading import Thread
-from time import sleep, time
-
-class SpeedManager(Thread):
-    def __init__(self, core, hook):
-        Thread.__init__(self)
-        self.setDaemon(True)
-
-        self.hook = hook
-        self.core = core
-        self.running = True
-        self.lastSlowCheck = 0.0
-
-        stat = {}
-        stat["slow_downloads"] = None
-        stat["each_speed"] = None
-        stat["each_speed_optimized"] = None
-        self.stat = stat
-
-        self.slowCheckInterval = 60
-        self.slowCheckTestTime = 25
-
-        self.log = self.core.log
-        self.start()
-
-    def run(self):
-        while self.running:
-            sleep(1)
-            self.manageSpeed()
-
-    def getMaxSpeed(self):
-        return int(self.hook.getConfig("speed"))
-
-    def manageSpeed(self):
-        maxSpeed = self.getMaxSpeed()
-        if maxSpeed <= 0 or not self.core.compare_time(self.hook.getConfig("start").split(":"),
-                                                   self.hook.getConfig("end").split(":")):
-            for pyfile in [x.active for x in self.core.threadManager.threads if x.active and x.active != "quit"]:
-                pyfile.plugin.req.speedLimitActive = False
-            return
-
-        threads = [x.active for x in self.core.threadManager.threads if x.active and x.active != "quit"]
-        threadCount = len(threads)
-        if not threads:
-            return
-        eachSpeed = maxSpeed / threadCount
-
-        currentOverallSpeed = 0
-        restSpeed = maxSpeed - currentOverallSpeed
-        speeds = []
-        for thread in threads:
-            currentOverallSpeed += thread.plugin.req.dl_speed
-            speeds.append((thread.plugin.req.dl_speed, thread.plugin.req.averageSpeed, thread))
-            thread.plugin.req.speedLimitActive = True
-
-        if currentOverallSpeed + 50 < maxSpeed:
-            for thread in threads:
-                thread.plugin.req.speedLimitActive = False
-            return
-
-        slowCount = 0
-        slowSpeed = 0
-        if self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime < time():
-            self.lastSlowCheck = time()
-        if self.lastSlowCheck + self.slowCheckInterval < time(
-        ) < self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime:
-            for speed in speeds:
-                speed[2].plugin.req.isSlow = False
-        else:
-            for speed in speeds:
-                if speed[0] <= eachSpeed - 7:
-                    if speed[1] < eachSpeed - 15:
-                        if speed[2].plugin.req.dl_time > 0 and speed[2].plugin.req.dl_time + 30 < time():
-                            speed[2].plugin.req.isSlow = True
-                            if not speed[1] - 5 < speed[2].plugin.req.maxSpeed / 1024 < speed[1] + 5:
-                                speed[2].plugin.req.maxSpeed = (speed[1] + 10) * 1024
-                if speed[2].plugin.req.isSlow:
-                    slowCount += 1
-                    slowSpeed += speed[2].plugin.req.maxSpeed / 1024
-        stat = {}
-        stat["slow_downloads"] = slowCount
-        stat["each_speed"] = eachSpeed
-        eachSpeed = (maxSpeed - slowSpeed) / (threadCount - slowCount)
-        stat["each_speed_optimized"] = eachSpeed
-        self.stat = stat
-
-        for speed in speeds:
-            if speed[2].plugin.req.isSlow:
-                continue
-            speed[2].plugin.req.maxSpeed = eachSpeed * 1024
diff --git a/module/ThreadManager.py b/module/ThreadManager.py
index 284997d4a..6a1802b9e 100644
--- a/module/ThreadManager.py
+++ b/module/ThreadManager.py
@@ -44,8 +44,6 @@ class ThreadManager:
         self.threads = []  # thread list
         self.localThreads = []  #hook+decrypter threads
 
-        #self.infoThread = PluginThread.InfoThread(self)
-
         self.pause = True
 
         self.reconnecting = Event()
@@ -54,7 +52,7 @@ class ThreadManager:
 
         pycurl.global_init(pycurl.GLOBAL_DEFAULT)
 
-        for i in range(0, self.core.config.get("general", "max_downloads")):
+        for i in range(0, self.core.config.get("download", "max_downloads")):
             self.createThread()
 
 
@@ -175,9 +173,9 @@ class ThreadManager:
     def checkThreadCount(self):
         """checks if there are need for increasing or reducing thread count"""
 
-        if len(self.threads) == self.core.config.get("general", "max_downloads"):
+        if len(self.threads) == self.core.config.get("download", "max_downloads"):
             return True
-        elif len(self.threads) < self.core.config.get("general", "max_downloads"):
+        elif len(self.threads) < self.core.config.get("download", "max_downloads"):
             self.createThread()
         else:
             free = [x for x in self.threads if not x.active]
diff --git a/module/config/default.conf b/module/config/default.conf
index 884891375..143e2520f 100644
--- a/module/config/default.conf
+++ b/module/config/default.conf
@@ -23,16 +23,19 @@ log - "Log":
 general - "General":

 	en;de;it;pl;es language : "Language" = en

 	folder download_folder : "Download Folder" = Downloads

-	int max_downloads : "Max Parallel Downloads" = 3

 	bool debug_mode : "Debug Mode" = False

-	int max_download_time : "Max Download Time" = 5

 	bool checksum : "Use Checksum" = False

 	int min_free_space : "Min Free Space (MB)" = 200

 	bool folder_per_package : "Create folder for each package" = True

 	bool skip_existing : "Skip already existing files" = False

-	ip download_interface : "Outgoing IP address for downloads" = None

 	int renice : "CPU Priority" = 0

-	int chunks : "Max connections for one download" = 4

+download - "Download":

+    int chunks : "Max connections for one download" = 4

+    int max_downloads : "Max Parallel Downloads" = 3

+    int max_speed : "Max Download Speed kb/s" = -1

+    bool limit_speed : "Limit Download Speed" = False

+    bool proxy : "Use Proxy" = False

+    str interface : "Download interface to bind (ip or Name)" = None

 permission - "Permissions":

     bool change_user : "Change user of running process" = False

     str user : "Username" = user

@@ -51,6 +54,8 @@ downloadTime - "Download Time":
 	time start : "Start" = 0:00

 	time end : "End" = 0:00

 proxy - "Proxy":

-	bool activated : "Activated" = False

-	str address : "Address" = http://localhost:8080

-	str protocol : "Protocol" = http

+	str address : "Address" = "localhost"

+	int port : "Port" = 7070

+	str type : "Protocol (http|socks4|socks5)" = http

+	str username : "Username" = None

+	str password : "Password" = None

diff --git a/module/network/Browser.py b/module/network/Browser.py
index be8dee27d..0a45c1ef4 100644
--- a/module/network/Browser.py
+++ b/module/network/Browser.py
@@ -9,17 +9,17 @@ from HTTPDownload import HTTPDownload
 
 
 class Browser(object):
-    def __init__(self, interface=None, cj=None, bucket=None, proxies={}):
+    def __init__(self, interface=None, bucket=None, proxies={}):
         self.log = getLogger("log")
 
         self.interface = interface
-        self.cj = cj
         self.bucket = bucket
         self.proxies = proxies
 
+        self.cj = None # needs to be setted later
         self._size = 0
 
-        self.http = HTTPRequest(cj, interface, proxies)
+        self.http = HTTPRequest(self.cj, interface, proxies)
         self.dl = None
 
     lastEffectiveURL = property(lambda self: self.http.lastEffectiveURL)
diff --git a/module/network/Bucket.py b/module/network/Bucket.py
index dc1280ede..60d8a757a 100644
--- a/module/network/Bucket.py
+++ b/module/network/Bucket.py
@@ -29,11 +29,12 @@ class Bucket:
     
     def setRate(self, rate):
         self.lock.acquire()
-        self.rate = rate
+        self.rate = int(rate)
         self.lock.release()
 
     def consumed(self, amount):
         """ return time the process have to sleep, after consumed specified amount """
+        if self.rate < 0: return 0
         self.lock.acquire()
 
         self.calc_tokens()
diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py
index 3a240b081..805305f80 100644
--- a/module/network/HTTPRequest.py
+++ b/module/network/HTTPRequest.py
@@ -51,7 +51,6 @@ class HTTPRequest():
         self.c.setopt(pycurl.NOPROGRESS, 1)
         if hasattr(pycurl, "AUTOREFERER"):
             self.c.setopt(pycurl.AUTOREFERER, 1)
-        self.c.setopt(pycurl.BUFFERSIZE, 32 * 1024)
         self.c.setopt(pycurl.SSL_VERIFYPEER, 0)
         self.c.setopt(pycurl.LOW_SPEED_TIME, 30)
         self.c.setopt(pycurl.LOW_SPEED_LIMIT, 20)
@@ -67,11 +66,22 @@ class HTTPRequest():
                            "Connection: keep-alive",
                            "Keep-Alive: 300"])
 
-    def setInterface(self, interface, proxies):
+    def setInterface(self, interface, proxy):
         if interface and interface.lower() != "none":
             self.c.setopt(pycurl.INTERFACE, interface)
 
-        #@TODO set proxies
+        if proxy:
+            if proxy["type"] == "socks4":
+                self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS4)
+            elif proxy["type"] == "socks5":
+                self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5)
+            else:
+                self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_HTTP)
+
+            self.c.setopt(pycurl.PROXY, "%s:%s" % (proxy["address"], proxy["port"]))
+
+            if proxy["username"]:
+                self.c.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (proxy["username"], proxy["password"]))
 
     def addCookies(self):
         if self.cj:
diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py
index 6ad64589a..6bc7e3fe7 100644
--- a/module/network/RequestFactory.py
+++ b/module/network/RequestFactory.py
@@ -20,6 +20,7 @@
 from threading import Lock
 
 from Browser import Browser
+from Bucket import Bucket
 from HTTPRequest import HTTPRequest
 from CookieJar import CookieJar
 
@@ -27,13 +28,14 @@ class RequestFactory():
     def __init__(self, core):
         self.lock = Lock()
         self.core = core
+        self.bucket = Bucket()
+        self.updateBucket()
         self.cookiejars = {}
 
     def getRequest(self, pluginName, account=None):
         self.lock.acquire()
 
-        req = Browser()
-        #@TODO proxy stuff, bucket
+        req = Browser(self.core.config["download"]["interface"], self.bucket, self.getProxies())
 
         if account:
             cj = self.getCookieJar(pluginName, account)
@@ -45,8 +47,7 @@ class RequestFactory():
         return req
 
     def getURL(self, url, get={}, post={}):
-        #a bit to much overhead for single url
-        h = HTTPRequest()
+        h = HTTPRequest(None, self.core.config["download"]["interface"], self.getProxies())
         rep = h.load(url, get, post)
         h.close()
         return rep
@@ -59,6 +60,39 @@ class RequestFactory():
         self.cookiejars[(pluginName, account)] = cj
         return cj
 
+    def getProxies(self):
+        """ returns a proxy list for the request classes """
+        if not self.core.config["download"]["proxy"]:
+            return {}
+        else:
+            type = "http"
+            setting = self.core.config["proxy"]["type"].lower()
+            if setting == "socks4": type = "socks4"
+            if setting == "socks5": type = "socks5"
+
+            username = None
+            if self.core.config["proxy"]["username"] and self.core.config["proxy"]["username"].lower() != "none":
+                username = self.core.config["proxy"]["username"]
+
+            pw = None
+            if self.core.config["proxy"]["password"] and self.core.config["proxy"]["password"].lower() != "none":
+                pw = self.core.config["proxy"]["password"]
+
+            return {
+                "type": type,
+                "address": self.core.config["proxy"]["address"],
+                "port": self.core.config["proxy"]["port"],
+                "username": username,
+                "password": pw,
+            }
+
+    def updateBucket(self):
+        """ set values in the bucket according to settings"""
+        if not self.core.config["download"]["limit_speed"]:
+            self.bucket.setRate(-1)
+        else:
+            self.bucket.setRate(self.core.config["download"]["max_speed"] * 1024)
+
 # needs pyreq in global namespace
 def getURL(url, get={}, post={}):
     return pyreq.getURL(url, get, post)
\ No newline at end of file
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index 6650562ea..3da6e5116 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -117,8 +117,8 @@ class Plugin(object):
     
     def getChunkCount(self):
         if self.chunkLimit <= 0:
-            return self.config["general"]["chunks"]
-        return min(self.config["general"]["chunks"], self.chunkLimit)
+            return self.config["download"]["chunks"]
+        return min(self.config["download"]["chunks"], self.chunkLimit)
     
     def __call__(self):
         return self.__name__
@@ -143,9 +143,6 @@ class Plugin(object):
         else:
             self.req.clearCookies()
 
-        if self.core.config["proxy"]["activated"]:
-            self.req.add_proxy(None, self.core.config["proxy"]["address"])
-
         self.pyfile.setStatus("starting")
 
         return self.process(self.pyfile)
diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py
index 3f11c067b..52712f160 100644
--- a/module/plugins/PluginManager.py
+++ b/module/plugins/PluginManager.py
@@ -37,7 +37,7 @@ except ImportError: # python 2.5
     from module.SafeEval import safe_eval as literal_eval
 
 
-IGNORE = ["FreakshareNet"]
+IGNORE = ["FreakshareNet", "SpeedManager"]
 #ignore this plugins in homefolder, add deleted plugins here
 
 class PluginManager():
diff --git a/module/plugins/hooks/SpeedManager.py b/module/plugins/hooks/SpeedManager.py
deleted file mode 100644
index ed90773e8..000000000
--- a/module/plugins/hooks/SpeedManager.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 3 of the License,
-    or (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-    See the GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, see <http://www.gnu.org/licenses/>.
-
-    @author: RaNaN
-    @interface-version: 0.1
-"""
-
-from module.plugins.Hook import Hook
-from module.SpeedManager import SpeedManager as SpeedHandler
-
-class SpeedManager(Hook):
-    __name__ = "SpeedManager"
-    __version__ = "0.1"
-    __description__ = """Limiting the speed if needed"""
-    __config__ = [("activated", "bool", "Activated", "False"),
-                  ("speed", "int", "Speedlimit in kb/s", "500"),
-                  ("start", "str", "Start-time of speed limiting", "0:00"),
-                  ("end", "str", "End-time of speed limiting", "0:00")]
-    __author_name__ = ("RaNaN")
-    __author_mail__ = ("ranan@pyload.org")
-
-    def coreReady(self):
-        self.speedManager = SpeedHandler(self.core, self)
-        self.core.log.debug("SpeedManager started")
-        #@TODO start speed manager when needed
\ No newline at end of file
diff --git a/pyLoadCore.py b/pyLoadCore.py
index dfbb2f51d..3349a7393 100755
--- a/pyLoadCore.py
+++ b/pyLoadCore.py
@@ -603,7 +603,12 @@ class ServerMethods():
     def set_conf_val(self, cat, opt, val, sec="core"):
         """ set config value """
         if sec == "core":
-            self.core.config[str(cat)][str(opt)] = val
+            opt = str(opt)
+            self.core.config[str(cat)][opt] = val
+
+            if opt in ("limit_speed", "max_speed"): #not so nice to update the limit
+                self.core.requestFactory.updateBucket()
+
         elif sec == "plugin":
             self.core.config.setPlugin(cat, opt, val)