summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--module/SpeedManager.py111
-rw-r--r--module/ThreadManager.py8
-rw-r--r--module/config/default.conf19
-rw-r--r--module/network/Browser.py6
-rw-r--r--module/network/Bucket.py3
-rw-r--r--module/network/HTTPRequest.py16
-rw-r--r--module/network/RequestFactory.py42
-rw-r--r--module/plugins/Plugin.py7
-rw-r--r--module/plugins/PluginManager.py2
-rw-r--r--module/plugins/hooks/SpeedManager.py38
-rwxr-xr-xpyLoadCore.py7
11 files changed, 80 insertions, 179 deletions
diff --git a/module/SpeedManager.py b/module/SpeedManager.py
deleted file mode 100644
index 44d59dad4..000000000
--- a/module/SpeedManager.py
+++ /dev/null
@@ -1,111 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, see <http://www.gnu.org/licenses/>.
-
- @author: mkaay
- @version: v0.3.2
-"""
-
-from threading import Thread
-from time import sleep, time
-
-class SpeedManager(Thread):
- def __init__(self, core, hook):
- Thread.__init__(self)
- self.setDaemon(True)
-
- self.hook = hook
- self.core = core
- self.running = True
- self.lastSlowCheck = 0.0
-
- stat = {}
- stat["slow_downloads"] = None
- stat["each_speed"] = None
- stat["each_speed_optimized"] = None
- self.stat = stat
-
- self.slowCheckInterval = 60
- self.slowCheckTestTime = 25
-
- self.log = self.core.log
- self.start()
-
- def run(self):
- while self.running:
- sleep(1)
- self.manageSpeed()
-
- def getMaxSpeed(self):
- return int(self.hook.getConfig("speed"))
-
- def manageSpeed(self):
- maxSpeed = self.getMaxSpeed()
- if maxSpeed <= 0 or not self.core.compare_time(self.hook.getConfig("start").split(":"),
- self.hook.getConfig("end").split(":")):
- for pyfile in [x.active for x in self.core.threadManager.threads if x.active and x.active != "quit"]:
- pyfile.plugin.req.speedLimitActive = False
- return
-
- threads = [x.active for x in self.core.threadManager.threads if x.active and x.active != "quit"]
- threadCount = len(threads)
- if not threads:
- return
- eachSpeed = maxSpeed / threadCount
-
- currentOverallSpeed = 0
- restSpeed = maxSpeed - currentOverallSpeed
- speeds = []
- for thread in threads:
- currentOverallSpeed += thread.plugin.req.dl_speed
- speeds.append((thread.plugin.req.dl_speed, thread.plugin.req.averageSpeed, thread))
- thread.plugin.req.speedLimitActive = True
-
- if currentOverallSpeed + 50 < maxSpeed:
- for thread in threads:
- thread.plugin.req.speedLimitActive = False
- return
-
- slowCount = 0
- slowSpeed = 0
- if self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime < time():
- self.lastSlowCheck = time()
- if self.lastSlowCheck + self.slowCheckInterval < time(
- ) < self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime:
- for speed in speeds:
- speed[2].plugin.req.isSlow = False
- else:
- for speed in speeds:
- if speed[0] <= eachSpeed - 7:
- if speed[1] < eachSpeed - 15:
- if speed[2].plugin.req.dl_time > 0 and speed[2].plugin.req.dl_time + 30 < time():
- speed[2].plugin.req.isSlow = True
- if not speed[1] - 5 < speed[2].plugin.req.maxSpeed / 1024 < speed[1] + 5:
- speed[2].plugin.req.maxSpeed = (speed[1] + 10) * 1024
- if speed[2].plugin.req.isSlow:
- slowCount += 1
- slowSpeed += speed[2].plugin.req.maxSpeed / 1024
- stat = {}
- stat["slow_downloads"] = slowCount
- stat["each_speed"] = eachSpeed
- eachSpeed = (maxSpeed - slowSpeed) / (threadCount - slowCount)
- stat["each_speed_optimized"] = eachSpeed
- self.stat = stat
-
- for speed in speeds:
- if speed[2].plugin.req.isSlow:
- continue
- speed[2].plugin.req.maxSpeed = eachSpeed * 1024
diff --git a/module/ThreadManager.py b/module/ThreadManager.py
index 284997d4a..6a1802b9e 100644
--- a/module/ThreadManager.py
+++ b/module/ThreadManager.py
@@ -44,8 +44,6 @@ class ThreadManager:
self.threads = [] # thread list
self.localThreads = [] #hook+decrypter threads
- #self.infoThread = PluginThread.InfoThread(self)
-
self.pause = True
self.reconnecting = Event()
@@ -54,7 +52,7 @@ class ThreadManager:
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
- for i in range(0, self.core.config.get("general", "max_downloads")):
+ for i in range(0, self.core.config.get("download", "max_downloads")):
self.createThread()
@@ -175,9 +173,9 @@ class ThreadManager:
def checkThreadCount(self):
"""checks if there are need for increasing or reducing thread count"""
- if len(self.threads) == self.core.config.get("general", "max_downloads"):
+ if len(self.threads) == self.core.config.get("download", "max_downloads"):
return True
- elif len(self.threads) < self.core.config.get("general", "max_downloads"):
+ elif len(self.threads) < self.core.config.get("download", "max_downloads"):
self.createThread()
else:
free = [x for x in self.threads if not x.active]
diff --git a/module/config/default.conf b/module/config/default.conf
index 884891375..143e2520f 100644
--- a/module/config/default.conf
+++ b/module/config/default.conf
@@ -23,16 +23,19 @@ log - "Log":
general - "General":
en;de;it;pl;es language : "Language" = en
folder download_folder : "Download Folder" = Downloads
- int max_downloads : "Max Parallel Downloads" = 3
bool debug_mode : "Debug Mode" = False
- int max_download_time : "Max Download Time" = 5
bool checksum : "Use Checksum" = False
int min_free_space : "Min Free Space (MB)" = 200
bool folder_per_package : "Create folder for each package" = True
bool skip_existing : "Skip already existing files" = False
- ip download_interface : "Outgoing IP address for downloads" = None
int renice : "CPU Priority" = 0
- int chunks : "Max connections for one download" = 4
+download - "Download":
+ int chunks : "Max connections for one download" = 4
+ int max_downloads : "Max Parallel Downloads" = 3
+ int max_speed : "Max Download Speed kb/s" = -1
+ bool limit_speed : "Limit Download Speed" = False
+ bool proxy : "Use Proxy" = False
+ str interface : "Download interface to bind (ip or Name)" = None
permission - "Permissions":
bool change_user : "Change user of running process" = False
str user : "Username" = user
@@ -51,6 +54,8 @@ downloadTime - "Download Time":
time start : "Start" = 0:00
time end : "End" = 0:00
proxy - "Proxy":
- bool activated : "Activated" = False
- str address : "Address" = http://localhost:8080
- str protocol : "Protocol" = http
+ str address : "Address" = "localhost"
+ int port : "Port" = 7070
+ str type : "Protocol (http|socks4|socks5)" = http
+ str username : "Username" = None
+ str password : "Password" = None
diff --git a/module/network/Browser.py b/module/network/Browser.py
index be8dee27d..0a45c1ef4 100644
--- a/module/network/Browser.py
+++ b/module/network/Browser.py
@@ -9,17 +9,17 @@ from HTTPDownload import HTTPDownload
class Browser(object):
- def __init__(self, interface=None, cj=None, bucket=None, proxies={}):
+ def __init__(self, interface=None, bucket=None, proxies={}):
self.log = getLogger("log")
self.interface = interface
- self.cj = cj
self.bucket = bucket
self.proxies = proxies
+ self.cj = None # needs to be setted later
self._size = 0
- self.http = HTTPRequest(cj, interface, proxies)
+ self.http = HTTPRequest(self.cj, interface, proxies)
self.dl = None
lastEffectiveURL = property(lambda self: self.http.lastEffectiveURL)
diff --git a/module/network/Bucket.py b/module/network/Bucket.py
index dc1280ede..60d8a757a 100644
--- a/module/network/Bucket.py
+++ b/module/network/Bucket.py
@@ -29,11 +29,12 @@ class Bucket:
def setRate(self, rate):
self.lock.acquire()
- self.rate = rate
+ self.rate = int(rate)
self.lock.release()
def consumed(self, amount):
""" return time the process have to sleep, after consumed specified amount """
+ if self.rate < 0: return 0
self.lock.acquire()
self.calc_tokens()
diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py
index 3a240b081..805305f80 100644
--- a/module/network/HTTPRequest.py
+++ b/module/network/HTTPRequest.py
@@ -51,7 +51,6 @@ class HTTPRequest():
self.c.setopt(pycurl.NOPROGRESS, 1)
if hasattr(pycurl, "AUTOREFERER"):
self.c.setopt(pycurl.AUTOREFERER, 1)
- self.c.setopt(pycurl.BUFFERSIZE, 32 * 1024)
self.c.setopt(pycurl.SSL_VERIFYPEER, 0)
self.c.setopt(pycurl.LOW_SPEED_TIME, 30)
self.c.setopt(pycurl.LOW_SPEED_LIMIT, 20)
@@ -67,11 +66,22 @@ class HTTPRequest():
"Connection: keep-alive",
"Keep-Alive: 300"])
- def setInterface(self, interface, proxies):
+ def setInterface(self, interface, proxy):
if interface and interface.lower() != "none":
self.c.setopt(pycurl.INTERFACE, interface)
- #@TODO set proxies
+ if proxy:
+ if proxy["type"] == "socks4":
+ self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS4)
+ elif proxy["type"] == "socks5":
+ self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5)
+ else:
+ self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_HTTP)
+
+ self.c.setopt(pycurl.PROXY, "%s:%s" % (proxy["address"], proxy["port"]))
+
+ if proxy["username"]:
+ self.c.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (proxy["username"], proxy["password"]))
def addCookies(self):
if self.cj:
diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py
index 6ad64589a..6bc7e3fe7 100644
--- a/module/network/RequestFactory.py
+++ b/module/network/RequestFactory.py
@@ -20,6 +20,7 @@
from threading import Lock
from Browser import Browser
+from Bucket import Bucket
from HTTPRequest import HTTPRequest
from CookieJar import CookieJar
@@ -27,13 +28,14 @@ class RequestFactory():
def __init__(self, core):
self.lock = Lock()
self.core = core
+ self.bucket = Bucket()
+ self.updateBucket()
self.cookiejars = {}
def getRequest(self, pluginName, account=None):
self.lock.acquire()
- req = Browser()
- #@TODO proxy stuff, bucket
+ req = Browser(self.core.config["download"]["interface"], self.bucket, self.getProxies())
if account:
cj = self.getCookieJar(pluginName, account)
@@ -45,8 +47,7 @@ class RequestFactory():
return req
def getURL(self, url, get={}, post={}):
- #a bit to much overhead for single url
- h = HTTPRequest()
+ h = HTTPRequest(None, self.core.config["download"]["interface"], self.getProxies())
rep = h.load(url, get, post)
h.close()
return rep
@@ -59,6 +60,39 @@ class RequestFactory():
self.cookiejars[(pluginName, account)] = cj
return cj
+ def getProxies(self):
+ """ returns a proxy list for the request classes """
+ if not self.core.config["download"]["proxy"]:
+ return {}
+ else:
+ type = "http"
+ setting = self.core.config["proxy"]["type"].lower()
+ if setting == "socks4": type = "socks4"
+ if setting == "socks5": type = "socks5"
+
+ username = None
+ if self.core.config["proxy"]["username"] and self.core.config["proxy"]["username"].lower() != "none":
+ username = self.core.config["proxy"]["username"]
+
+ pw = None
+ if self.core.config["proxy"]["password"] and self.core.config["proxy"]["password"].lower() != "none":
+ pw = self.core.config["proxy"]["password"]
+
+ return {
+ "type": type,
+ "address": self.core.config["proxy"]["address"],
+ "port": self.core.config["proxy"]["port"],
+ "username": username,
+ "password": pw,
+ }
+
+ def updateBucket(self):
+ """ set values in the bucket according to settings"""
+ if not self.core.config["download"]["limit_speed"]:
+ self.bucket.setRate(-1)
+ else:
+ self.bucket.setRate(self.core.config["download"]["max_speed"] * 1024)
+
# needs pyreq in global namespace
def getURL(url, get={}, post={}):
return pyreq.getURL(url, get, post) \ No newline at end of file
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index 6650562ea..3da6e5116 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -117,8 +117,8 @@ class Plugin(object):
def getChunkCount(self):
if self.chunkLimit <= 0:
- return self.config["general"]["chunks"]
- return min(self.config["general"]["chunks"], self.chunkLimit)
+ return self.config["download"]["chunks"]
+ return min(self.config["download"]["chunks"], self.chunkLimit)
def __call__(self):
return self.__name__
@@ -143,9 +143,6 @@ class Plugin(object):
else:
self.req.clearCookies()
- if self.core.config["proxy"]["activated"]:
- self.req.add_proxy(None, self.core.config["proxy"]["address"])
-
self.pyfile.setStatus("starting")
return self.process(self.pyfile)
diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py
index 3f11c067b..52712f160 100644
--- a/module/plugins/PluginManager.py
+++ b/module/plugins/PluginManager.py
@@ -37,7 +37,7 @@ except ImportError: # python 2.5
from module.SafeEval import safe_eval as literal_eval
-IGNORE = ["FreakshareNet"]
+IGNORE = ["FreakshareNet", "SpeedManager"]
#ignore this plugins in homefolder, add deleted plugins here
class PluginManager():
diff --git a/module/plugins/hooks/SpeedManager.py b/module/plugins/hooks/SpeedManager.py
deleted file mode 100644
index ed90773e8..000000000
--- a/module/plugins/hooks/SpeedManager.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, see <http://www.gnu.org/licenses/>.
-
- @author: RaNaN
- @interface-version: 0.1
-"""
-
-from module.plugins.Hook import Hook
-from module.SpeedManager import SpeedManager as SpeedHandler
-
-class SpeedManager(Hook):
- __name__ = "SpeedManager"
- __version__ = "0.1"
- __description__ = """Limiting the speed if needed"""
- __config__ = [("activated", "bool", "Activated", "False"),
- ("speed", "int", "Speedlimit in kb/s", "500"),
- ("start", "str", "Start-time of speed limiting", "0:00"),
- ("end", "str", "End-time of speed limiting", "0:00")]
- __author_name__ = ("RaNaN")
- __author_mail__ = ("ranan@pyload.org")
-
- def coreReady(self):
- self.speedManager = SpeedHandler(self.core, self)
- self.core.log.debug("SpeedManager started")
- #@TODO start speed manager when needed \ No newline at end of file
diff --git a/pyLoadCore.py b/pyLoadCore.py
index dfbb2f51d..3349a7393 100755
--- a/pyLoadCore.py
+++ b/pyLoadCore.py
@@ -603,7 +603,12 @@ class ServerMethods():
def set_conf_val(self, cat, opt, val, sec="core"):
""" set config value """
if sec == "core":
- self.core.config[str(cat)][str(opt)] = val
+ opt = str(opt)
+ self.core.config[str(cat)][opt] = val
+
+ if opt in ("limit_speed", "max_speed"): #not so nice to update the limit
+ self.core.requestFactory.updateBucket()
+
elif sec == "plugin":
self.core.config.setPlugin(cat, opt, val)
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-23 04:53:44 +0100