summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGravatar RaNaN <Mast3rRaNaN@hotmail.de> 2014-01-18 18:45:13 +0100
committerGravatar RaNaN <Mast3rRaNaN@hotmail.de> 2014-01-18 18:45:13 +0100
commit453c1e55c71a96c9529ecdca1d55278cc41088d6 (patch)
tree7a516a84e5590ce5f1f3def71c24bcb14f209023
parentsmall fixes and improvements for download engine (diff)
downloadpyload-453c1e55c71a96c9529ecdca1d55278cc41088d6.tar.xz
rewritten download scheduling, improved account manager, db version increased all data will be overwritten
-rw-r--r--pyload/AccountManager.py51
-rw-r--r--pyload/Core.py23
-rw-r--r--pyload/DownloadManager.py246
-rw-r--r--pyload/FileManager.py72
-rw-r--r--pyload/InitHomeDir.py9
-rw-r--r--pyload/PluginManager.py8
-rw-r--r--pyload/api/AccountApi.py27
-rw-r--r--pyload/api/CoreApi.py29
-rw-r--r--pyload/api/DownloadApi.py5
-rw-r--r--pyload/api/StatisticsApi.py28
-rw-r--r--pyload/api/__init__.py2
-rw-r--r--pyload/config/default.py14
-rw-r--r--pyload/database/AccountDatabase.py22
-rw-r--r--pyload/database/DatabaseBackend.py17
-rw-r--r--pyload/database/FileDatabase.py30
-rw-r--r--pyload/database/StatisticDatabase.py5
-rw-r--r--pyload/database/UserDatabase.py8
-rw-r--r--pyload/datatypes/PyFile.py13
-rw-r--r--pyload/datatypes/User.py3
-rw-r--r--pyload/network/__init__.py29
-rw-r--r--pyload/plugins/Account.py14
-rw-r--r--pyload/plugins/addons/MultiHoster.py2
-rw-r--r--pyload/remote/apitypes.py69
-rw-r--r--pyload/remote/apitypes_debug.py8
-rw-r--r--pyload/remote/pyload.thrift47
-rw-r--r--pyload/remote/wsbackend/AsyncHandler.py8
-rw-r--r--pyload/threads/DecrypterThread.py1
-rw-r--r--pyload/threads/DownloadThread.py19
-rw-r--r--pyload/threads/ThreadManager.py220
-rw-r--r--pyload/utils/PluginLoader.py16
-rw-r--r--pyload/web/app/scripts/models/Account.js8
-rw-r--r--pyload/web/app/scripts/models/StatusInfo.js (renamed from pyload/web/app/scripts/models/ServerStatus.js)5
-rw-r--r--pyload/web/app/scripts/utils/apitypes.js4
-rw-r--r--pyload/web/app/scripts/views/accounts/accountModal.js2
-rw-r--r--pyload/web/app/scripts/views/headerView.js8
-rw-r--r--tests/config/pyload.conf.org76
-rw-r--r--tests/config/pyload.db.orgbin27648 -> 27648 bytes
-rw-r--r--tests/helper/Stubs.py14
-rw-r--r--tests/manager/test_accountManager.py22
-rw-r--r--tests/manager/test_downloadManager.py28
-rw-r--r--tests/other/test_statsDB.py13
41 files changed, 684 insertions, 541 deletions
diff --git a/pyload/AccountManager.py b/pyload/AccountManager.py
index b9f1536d9..e2a1b28a6 100644
--- a/pyload/AccountManager.py
+++ b/pyload/AccountManager.py
@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
###############################################################################
-# Copyright(c) 2008-2013 pyLoad Team
+# Copyright(c) 2008-2014 pyLoad Team
# http://www.pyload.org
#
# This file is part of pyLoad.
@@ -13,9 +13,10 @@
#
# Subjected to the terms and conditions in LICENSE
#
-# @author: RaNaN, mkaay
+# @author: RaNaN
###############################################################################
+
from threading import Lock
from random import choice
@@ -81,45 +82,55 @@ class AccountManager:
data = []
for plugin, accounts in self.accounts.iteritems():
data.extend(
- [(plugin, acc.loginname, acc.owner, 1 if acc.activated else 0, 1 if acc.shared else 0, acc.password,
- json.dumps(acc.options)) for acc in
+ [(acc.loginname, 1 if acc.activated else 0, 1 if acc.shared else 0, acc.password,
+ json.dumps(acc.options), acc.aid) for acc in
accounts])
self.core.db.saveAccounts(data)
- def getAccount(self, plugin, loginname, user=None):
+ def getAccount(self, aid, plugin, user=None):
""" Find a account by specific user (if given) """
if plugin in self.accounts:
for acc in self.accounts[plugin]:
- if acc.loginname == loginname and (not user or acc.owner == user.true_primary):
+ if acc.aid == aid and (not user or acc.owner == user.true_primary):
return acc
@lock
- def updateAccount(self, plugin, loginname, password, user):
+ def createAccount(self, plugin, loginname, password, uid):
+ """ Creates a new account """
+
+ aid = self.core.db.createAccount(plugin, loginname, password, uid)
+ info = AccountInfo(aid, plugin, loginname, uid, activated=True)
+ account = self._createAccount(info, password, {})
+ account.scheduleRefresh()
+ self.saveAccounts()
+
+ self.core.eventManager.dispatchEvent("account:created", account.toInfoData())
+ return account
+
+ @lock
+ def updateAccount(self, aid, plugin, loginname, password, user):
"""add or update account"""
- account = self.getAccount(plugin, loginname, user)
- if account:
- if account.setPassword(password):
- self.saveAccounts()
- account.scheduleRefresh(force=True)
- else:
- info = AccountInfo(plugin, loginname, user.true_primary, activated=True)
- account = self._createAccount(info, password, {})
- account.scheduleRefresh()
+ account = self.getAccount(aid, plugin, user)
+ if not account:
+ return
+
+ if account.setLogin(loginname, password):
self.saveAccounts()
+ account.scheduleRefresh(force=True)
self.core.eventManager.dispatchEvent("account:updated", account.toInfoData())
return account
@lock
- def removeAccount(self, plugin, loginname, uid):
+ def removeAccount(self, aid, plugin, uid):
"""remove account"""
if plugin in self.accounts:
for acc in self.accounts[plugin]:
# admins may delete accounts
- if acc.loginname == loginname and (not uid or acc.owner == uid):
+ if acc.aid == aid and (not uid or acc.owner == uid):
self.accounts[plugin].remove(acc)
- self.core.db.removeAccount(plugin, loginname)
- self.core.evm.dispatchEvent("account:deleted", plugin, loginname)
+ self.core.db.removeAccount(aid)
+ self.core.evm.dispatchEvent("account:deleted", aid, user=uid)
break
@lock
diff --git a/pyload/Core.py b/pyload/Core.py
index 6f3893481..04a77f45f 100644
--- a/pyload/Core.py
+++ b/pyload/Core.py
@@ -388,7 +388,7 @@ class Core(object):
self.interactionManager = self.im = InteractionManager(self)
self.accountManager = AccountManager(self)
self.threadManager = ThreadManager(self)
- self.downloadManager = DownloadManager(self)
+ self.downloadManager = self.dlm = DownloadManager(self)
self.addonManager = AddonManager(self)
self.remoteManager = RemoteManager(self)
@@ -438,7 +438,8 @@ class Core(object):
self.log.info(_("Restarting failed downloads..."))
self.api.restartFailed()
- self.threadManager.pause = False
+ # start downloads
+ self.dlm.paused = False
self.running = True
self.addonManager.activateAddons()
@@ -474,11 +475,15 @@ class Core(object):
self.removeLogger()
_exit(0)
# TODO check exits codes, clean exit is still blocked
+ try:
+ self.downloadManager.work()
+ self.threadManager.work()
+ self.interactionManager.work()
+ self.scheduler.work()
+ except Exception, e:
+ self.log.critical(_("Critical error: ") + str(e))
+ self.print_exc()
- self.downloadManager.work()
- self.threadManager.work()
- self.interactionManager.work()
- self.scheduler.work()
def setupDB(self):
from database import DatabaseBackend
@@ -586,10 +591,8 @@ class Core(object):
pass # TODO: quit webserver?
# self.webserver.quit()
- for thread in self.threadManager.threads:
- thread.put("quit")
-
- self.api.stopAllDownloads()
+ self.dlm.abort()
+ self.dlm.shutdown()
self.addonManager.deactivateAddons()
except:
diff --git a/pyload/DownloadManager.py b/pyload/DownloadManager.py
index 706f9afeb..04c9f66df 100644
--- a/pyload/DownloadManager.py
+++ b/pyload/DownloadManager.py
@@ -16,18 +16,33 @@
# @author: RaNaN
###############################################################################
+from collections import defaultdict
from threading import Event
+from time import sleep
+from random import sample
+from subprocess import call
+
from ReadWriteLock import ReadWriteLock
-from utils import lock, read_lock, primary_uid
+from Api import DownloadStatus as DS
+
+from datatypes.PyFile import PyFile
+
+from utils import lock, read_lock
+from utils.fs import exists, join, free_space
+
+from network import get_ip
+
from threads.DownloadThread import DownloadThread
from threads.DecrypterThread import DecrypterThread
+
class DownloadManager:
""" Schedules and manages download and decrypter jobs. """
def __init__(self, core):
self.core = core
+ self.log = core.log
#: won't start download when true
self.paused = True
@@ -36,8 +51,10 @@ class DownloadManager:
self.free = []
#: a thread that in working must have a pyfile as active attribute
self.working = []
+ #: holds the decrypter threads
+ self.decrypter = []
- #: indicates when reconnect has occured
+ #: indicates when reconnect has occurred
self.reconnecting = Event()
self.reconnecting.clear()
@@ -47,24 +64,233 @@ class DownloadManager:
def done(self, thread):
""" Switch thread from working to free state """
self.working.remove(thread)
- self.free.append(thread)
+ # only download threads will be re-used
+ if isinstance(thread, DownloadThread):
+ self.free.append(thread)
+ thread.isWorking.clear()
+
+ @lock
+ def stop(self, thread):
+ """ Removes a thread from all lists """
+
+ if thread in self.free:
+ self.free.remove(thread)
+
+ if thread in self.working:
+ self.working.remove(thread)
+
+ @lock
+ def startDownloadThread(self, info):
+ """ Use a free dl thread or create a new one """
+ if self.free:
+ thread = self.free[0]
+ del self.free[0]
+ else:
+ thread = DownloadThread(self)
+
+ thread.put(PyFile.fromInfoData(self.core.files, info))
+
+ # wait until it picked up the task
+ thread.isWorking.wait()
+ self.working.append(thread)
+
+ @lock
+ def startDecrypterThread(self, info):
+ """ Start decrypting of entered data, all links in one package are accumulated to one thread."""
+ self.decrypter.append(DecrypterThread(self, [(info.plugin, info.url)], info.pid))
@read_lock
- def activeDownloads(self, user):
+ def activeDownloads(self, uid=None):
""" retrieve pyfiles of running downloads """
- uid = primary_uid(user)
- return [x.active for x in self.working if uid is None or x.active.owner == uid]
+ return [x.active for x in self.working
+ if uid is None or x.active.owner == uid]
- def getProgressList(self, user):
+ @read_lock
+ def getProgressList(self, uid):
""" Progress of all running downloads """
- return [p.getProgressInfo() for p in self.activeDownloads(user)]
+ # decrypter progress could be none
+ return filter(lambda x: x is not None,
+ [p.getProgress() for p in self.working + self.decrypter
+ if uid is None or p.owner == uid])
- def canDownload(self, user):
- """ check if a user is eligible to start a new download """
+ def processingIds(self):
+ """get a id list of all pyfiles processed"""
+ return [x.id for x in self.activeDownloads(None)]
+ @read_lock
def abort(self):
""" Cancels all downloads """
+ # TODO: may dead lock
+ for t in self.working:
+ t.active.abortDownload()
+
+ @read_lock
+ def shutdown(self):
+ """ End all threads """
+ for thread in self.working + self.free:
+ thread.put("quit")
def work(self):
""" main routine that does the periodical work """
+ self.tryReconnect()
+
+ if free_space(self.core.config["general"]["download_folder"]) / 1024 / 1024 < \
+ self.core.config["general"]["min_free_space"]:
+ self.log.warning(_("Not enough space left on device"))
+ self.paused = True
+
+ if self.paused or not self.core.api.isTimeDownload():
+ return False
+
+ # at least one thread want reconnect and we are supposed to wait
+ if self.core.config['reconnect']['wait'] and self.wantReconnect() > 1:
+ return False
+
+ self.assignJobs()
+
+ def assignJobs(self):
+ """ Load jobs from db and try to assign them """
+
+ limit = self.core.config['download']['max_downloads'] - len(self.activeDownloads())
+ slots = self.getRemainingPluginSlots()
+ occ = tuple([plugin for plugin, v in slots.iteritems() if v == 0])
+ jobs = self.core.files.getJobs(occ)
+
+ # map plugin to list of jobs
+ plugins = defaultdict(list)
+
+ for uid, info in jobs.items():
+ # check the quota of each user and filter
+ quota = self.core.api.calcQuota(uid)
+ if -1 < quota < info.size:
+ del jobs[uid]
+
+ plugins[info.download.plugin].append(info)
+
+ for plugin, jobs in plugins.iteritems():
+ # we know exactly the number of remaining jobs
+ # or only can start one job if limit is not known
+ to_schedule = slots[plugin] if plugin in slots else 1
+ # start all chosen jobs
+ for job in self.chooseJobs(jobs, to_schedule):
+ # if the job was started the limit will be reduced
+ if self.startJob(job, limit):
+ limit -= 1
+
+ def chooseJobs(self, jobs, k):
+ """ make a fair choice of which k jobs to start """
+ # TODO: prefer admins, make a fairer choice?
+ if k >= len(jobs):
+ return jobs
+
+ return sample(jobs, k)
+
+ def startJob(self, info, limit):
+ """ start a download or decrypter thread with given file info """
+
+ plugin = self.core.pluginManager.findPlugin(info.download.plugin)
+ # this plugin does not exits
+ if plugin is None:
+ self.log.error(_("Plugin '%s' does not exists") % info.download.plugin)
+ self.core.files.setDownloadStatus(info.fid, DS.Failed)
+ return False
+
+ if plugin == "hoster":
+ # this job can't be started
+ if limit == 0:
+ return False
+
+ self.startDownloadThread(info)
+ return True
+
+ elif plugin == "crypter":
+ self.startDecrypterThread(info)
+ else:
+ self.log.error(_("Plugin type '%s' is can be used for downloading") % plugin)
+
+ return False
+
+ @read_lock
+ def tryReconnect(self):
+ """checks if reconnect needed"""
+
+ if not self.core.config["reconnect"]["activated"] or not self.core.api.isTimeReconnect():
+ return False
+
+ # only reconnect when all threads are ready
+ if not (0 < self.wantReconnect() == len(self.working)):
+ return False
+
+ if not exists(self.core.config['reconnect']['method']):
+ if exists(join(pypath, self.core.config['reconnect']['method'])):
+ self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method'])
+ else:
+ self.core.config["reconnect"]["activated"] = False
+ self.log.warning(_("Reconnect script not found!"))
+ return
+
+ self.reconnecting.set()
+
+ self.log.info(_("Starting reconnect"))
+
+ # wait until all thread got the event
+ while [x.active.plugin.waiting for x in self.working].count(True) != 0:
+ sleep(0.25)
+
+ old_ip = get_ip()
+
+ self.core.evm.dispatchEvent("reconnect:before", old_ip)
+ self.log.debug("Old IP: %s" % old_ip)
+
+ try:
+ call(self.core.config['reconnect']['method'], shell=True)
+ except:
+ self.log.warning(_("Failed executing reconnect script!"))
+ self.core.config["reconnect"]["activated"] = False
+ self.reconnecting.clear()
+ self.core.print_exc()
+ return
+
+ sleep(1)
+ ip = get_ip()
+ self.core.evm.dispatchEvent("reconnect:after", ip)
+
+ if not old_ip or old_ip == ip:
+ self.log.warning(_("Reconnect not successful"))
+ else:
+ self.log.info(_("Reconnected, new IP: %s") % ip)
+
+ self.reconnecting.clear()
+
+ @read_lock
+ def wantReconnect(self):
+ """ number of downloads that are waiting for reconnect """
+ active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.working]
+ return active.count(True)
+
+ @read_lock
+ def getRemainingPluginSlots(self):
+ """ dict of plugin names mapped to remaining dls """
+ occ = defaultdict(lambda: -1)
+ # decrypter are treated as occupied
+ for p in self.decrypter:
+ progress = p.getProgressInfo()
+ if progress:
+ occ[progress.plugin] = 0
+
+ # get all default dl limits
+ for t in self.working:
+ if not t.active.hasPlugin(): continue
+ limit = t.active.plugin.getDownloadLimit()
+ if limit < 0: continue
+ occ[t.active.pluginname] = limit
+
+ # subtract with running downloads
+ for t in self.working:
+ if not t.active.hasPlugin(): continue
+ plugin = t.active.pluginname
+ if plugin in occ:
+ occ[plugin] -= 1
+
+ return occ \ No newline at end of file
diff --git a/pyload/FileManager.py b/pyload/FileManager.py
index 2edf81bfc..9702307a0 100644
--- a/pyload/FileManager.py
+++ b/pyload/FileManager.py
@@ -35,6 +35,7 @@ def invalidate(func):
return new
+
class FileManager:
"""Handles all request made to obtain information,
modify status or other request for links or packages"""
@@ -49,9 +50,10 @@ class FileManager:
# translations
self.statusMsg = [_("none"), _("offline"), _("online"), _("queued"), _("paused"),
- _("finished"), _("skipped"), _("failed"), _("starting"),_("waiting"),
+ _("finished"), _("skipped"), _("failed"), _("starting"), _("waiting"),
_("downloading"), _("temp. offline"), _("aborted"), _("not possible"), _("missing"),
- _("file mismatch"), _("decrypting"), _("processing"), _("custom"), _("unknown")]
+ _("file mismatch"), _("occupied"), _("decrypting"), _("processing"), _("custom"),
+ _("unknown")]
self.files = {} # holds instances for files
self.packages = {} # same for packages
@@ -93,7 +95,7 @@ class FileManager:
@invalidate
def addLinks(self, data, pid, owner):
- """Add links, data = (plugin, url) tuple. Internal method should use API."""
+ """Add links, data = (url, plugin) tuple. Internal method should use API."""
self.db.addLinks(data, pid, owner)
self.evm.dispatchEvent("package:updated", pid)
@@ -102,7 +104,7 @@ class FileManager:
def addPackage(self, name, folder, root, password, site, comment, paused, owner):
"""Adds a package to database"""
pid = self.db.addPackage(name, folder, root, password, site, comment,
- PackageStatus.Paused if paused else PackageStatus.Ok, owner)
+ PackageStatus.Paused if paused else PackageStatus.Ok, owner)
p = self.db.getPackageInfo(pid)
self.evm.dispatchEvent("package:inserted", pid, p.root, p.packageorder)
@@ -250,46 +252,14 @@ class FileManager:
@lock
- def getJob(self, occ):
- """get suitable job"""
-
- #TODO only accessed by one thread, should not need a lock
- #TODO needs to be approved for new database
- #TODO clean mess
- #TODO improve selection of valid jobs
-
- if occ in self.jobCache:
- if self.jobCache[occ]:
- id = self.jobCache[occ].pop()
- if id == "empty":
- pyfile = None
- self.jobCache[occ].append("empty")
- else:
- pyfile = self.getFile(id)
- else:
- jobs = self.db.getJob(occ)
- jobs.reverse()
- if not jobs:
- self.jobCache[occ].append("empty")
- pyfile = None
- else:
- self.jobCache[occ].extend(jobs)
- pyfile = self.getFile(self.jobCache[occ].pop())
-
- else:
- self.jobCache = {} #better not caching to much
- jobs = self.db.getJob(occ)
- jobs.reverse()
- self.jobCache[occ] = jobs
+ def getJobs(self, occ):
- if not jobs:
- self.jobCache[occ].append("empty")
- pyfile = None
- else:
- pyfile = self.getFile(self.jobCache[occ].pop())
+ # load jobs with file info
+ if occ not in self.jobCache:
+ self.jobCache[occ] = dict([(k, self.getFileInfo(fid)) for k, fid
+ in self.db.getJobs(occ).iteritems()])
-
- return pyfile
+ return self.jobCache[occ]
def getDownloadStats(self, user=None):
""" return number of downloads """
@@ -346,7 +316,6 @@ class FileManager:
if fid in self.core.threadManager.processingIds():
f.abortDownload()
-
self.db.deleteFile(fid, f.fileorder, f.packageid)
self.releaseFile(fid)
@@ -377,6 +346,17 @@ class FileManager:
self.evm.dispatchEvent("file:updated", pyfile)
@invalidate
+ @read_lock
+ def setDownloadStatus(self, fid, status):
+ """ sets a download status for a file """
+ if fid in self.files:
+ self.files[fid].setStatus(status)
+ else:
+ self.db.setDownloadStatus(fid, status)
+
+ self.evm.dispatchEvent("file:updated", fid)
+
+ @invalidate
def updatePackage(self, pypack):
"""updates a package"""
self.db.updatePackage(pypack)
@@ -488,7 +468,7 @@ class FileManager:
raise Exception("Tried to reorder non continuous block of files")
# minimum fileorder
- f = reduce(lambda x,y: x if x.fileorder < y.fileorder else y, files)
+ f = reduce(lambda x, y: x if x.fileorder < y.fileorder else y, files)
order = f.fileorder
self.db.orderFiles(pid, fids, order, position)
@@ -507,12 +487,12 @@ class FileManager:
elif f.fileorder < position:
for pyfile in self.files.itervalues():
if pyfile.packageid != f.package or pyfile.fileorder < 0: continue
- if position >= pyfile.fileorder >= f.fileorder+diff:
+ if position >= pyfile.fileorder >= f.fileorder + diff:
pyfile.fileorder -= diff
for i, fid in enumerate(fids):
if fid in self.files:
- self.files[fid].fileorder = position -diff + i + 1
+ self.files[fid].fileorder = position - diff + i + 1
self.db.commit()
diff --git a/pyload/InitHomeDir.py b/pyload/InitHomeDir.py
index a68e1a197..51dfc7686 100644
--- a/pyload/InitHomeDir.py
+++ b/pyload/InitHomeDir.py
@@ -59,6 +59,7 @@ else:
__builtin__.homedir = homedir
configdir = None
+final = False
args = " ".join(argv)
# dirty method to set configdir from commandline arguments
if "--configdir=" in args:
@@ -83,9 +84,15 @@ if not configdir:
configname = ".pyload" if platform in ("posix", "linux2", "darwin") else "pyload"
configdir = path.join(homedir, configname + dev)
-def init_dir(other_path=None):
+
+def init_dir(other_path=None, no_change=False):
# switch to pyload home directory, or path at other_path
global configdir
+ global final
+
+ if final: return
+
+ if no_change: final = True
if other_path is not None:
configdir = join(pypath, other_path)
diff --git a/pyload/PluginManager.py b/pyload/PluginManager.py
index f3d2b999d..389eb86a2 100644
--- a/pyload/PluginManager.py
+++ b/pyload/PluginManager.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
###############################################################################
-# Copyright(c) 2008-2013 pyLoad Team
+# Copyright(c) 2008-2014 pyLoad Team
# http://www.pyload.org
#
# This file is part of pyLoad.
@@ -12,7 +12,7 @@
#
# Subjected to the terms and conditions in LICENSE
#
-# @author: RaNaN, mkaay
+# @author: RaNaN
###############################################################################
import sys
@@ -132,6 +132,10 @@ class PluginManager:
return res["hoster"], res["crypter"]
+ def findPlugin(self, name):
+ """ Finds the type to a plugin name """
+ return self.loader.findPlugin(name)
+
def getPlugin(self, plugin, name):
""" Retrieves the plugin tuple for a single plugin or none """
return self.loader.getPlugin(plugin, name)
diff --git a/pyload/api/AccountApi.py b/pyload/api/AccountApi.py
index d4b39c12b..6b89a2aad 100644
--- a/pyload/api/AccountApi.py
+++ b/pyload/api/AccountApi.py
@@ -27,12 +27,12 @@ class AccountApi(ApiComponent):
return [acc.toInfoData() for acc in accounts]
@RequirePerm(Permission.Accounts)
- def getAccountInfo(self, plugin, loginname, refresh=False):
+ def getAccountInfo(self, aid, plugin, refresh=False):
""" Returns :class:`AccountInfo` for a specific account
:param refresh: reload account info
"""
- account = self.core.accountManager.getAccount(plugin, loginname)
+ account = self.core.accountManager.getAccount(aid, plugin)
# Admins can see and refresh accounts
if not account or (self.primaryUID and self.primaryUID != account.owner):
@@ -45,20 +45,27 @@ class AccountApi(ApiComponent):
return account.toInfoData()
@RequirePerm(Permission.Accounts)
- def updateAccount(self, plugin, loginname, password):
- """Creates an account if not existent or updates the password
+ def createAccount(self, plugin, loginname, password):
+ """ Creates a new account
- :return: newly created or updated account info
+ :return class:`AccountInfo`
"""
- # TODO: None pointer
- return self.core.accountManager.updateAccount(plugin, loginname, password, self.user).toInfoData()
+ return self.core.accountManager.createAccount(plugin, loginname, password, self.user.true_primary).toInfoData()
+
+ @RequirePerm(Permission.Accounts)
+ def updateAccount(self, aid, plugin, loginname, password):
+ """Updates loginname and password of an existent account
+
+ :return: updated account info
+ """
+ return self.core.accountManager.updateAccount(aid, plugin, loginname, password, self.user).toInfoData()
@RequirePerm(Permission.Accounts)
def updateAccountInfo(self, account):
""" Update account settings from :class:`AccountInfo` """
- inst = self.core.accountManager.getAccount(account.plugin, account.loginname, self.user)
- if not account:
+ inst = self.core.accountManager.getAccount(account.aid, account.plugin, self.user)
+ if not inst:
return
inst.activated = to_bool(account.activated)
@@ -72,7 +79,7 @@ class AccountApi(ApiComponent):
:param account: :class:`ÀccountInfo` instance
"""
- self.core.accountManager.removeAccount(account.plugin, account.loginname, self.primaryUID)
+ self.core.accountManager.removeAccount(account.aid, account.plugin, self.primaryUID)
if Api.extend(AccountApi):
diff --git a/pyload/api/CoreApi.py b/pyload/api/CoreApi.py
index 187286b48..b15272196 100644
--- a/pyload/api/CoreApi.py
+++ b/pyload/api/CoreApi.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-from pyload.Api import Api, RequirePerm, Permission, ServerStatus, Interaction
+from pyload.Api import Api, RequirePerm, Permission, StatusInfo, Interaction
from pyload.utils.fs import join, free_space, exists
from pyload.utils import compare_time
@@ -37,24 +37,24 @@ class CoreApi(ApiComponent):
return "%s://%%s:%d" % (ws, self.core.config['webUI']['wsPort'])
@RequirePerm(Permission.All)
- def getServerStatus(self):
+ def getStatusInfo(self):
"""Some general information about the current status of pyLoad.
- :return: `ServerStatus`
+ :return: `StatusInfo`
"""
queue = self.core.files.getQueueStats(self.primaryUID)
total = self.core.files.getDownloadStats(self.primaryUID)
- serverStatus = ServerStatus(0,
+ serverStatus = StatusInfo(0,
total[0], queue[0],
total[1], queue[1],
self.isInteractionWaiting(Interaction.All),
- not self.core.threadManager.pause and self.isTimeDownload(),
- self.core.threadManager.pause,
- self.core.config['reconnect']['activated'] and self.isTimeReconnect())
+ not self.core.dlm.paused and self.isTimeDownload(),
+ self.core.dlm.paused,
+ self.core.config['reconnect']['activated'] and self.isTimeReconnect(),
+ self.getQuota())
-
- for pyfile in self.core.threadManager.getActiveDownloads(self.primaryUID):
+ for pyfile in self.core.dlm.activeDownloads(self.primaryUID):
serverStatus.speed += pyfile.getSpeed() #bytes/s
return serverStatus
@@ -65,23 +65,24 @@ class CoreApi(ApiComponent):
:rtype: list of :class:`ProgressInfo`
"""
- return self.core.threadManager.getProgressList(self.primaryUID)
+ return self.core.dlm.getProgressList(self.primaryUID) +\
+ self.core.threadManager.getProgressList(self.primaryUID)
def pauseServer(self):
"""Pause server: It won't start any new downloads, but nothing gets aborted."""
- self.core.threadManager.pause = True
+ self.core.dlm.paused = True
def unpauseServer(self):
"""Unpause server: New Downloads will be started."""
- self.core.threadManager.pause = False
+ self.core.dlm.paused = False
def togglePause(self):
"""Toggle pause state.
:return: new pause state
"""
- self.core.threadManager.pause ^= True
- return self.core.threadManager.pause
+ self.core.dlm.paused ^= True
+ return self.core.dlm.paused
def toggleReconnect(self):
"""Toggle reconnect activation.
diff --git a/pyload/api/DownloadApi.py b/pyload/api/DownloadApi.py
index b29f9c06c..71d112e44 100644
--- a/pyload/api/DownloadApi.py
+++ b/pyload/api/DownloadApi.py
@@ -148,10 +148,7 @@ class DownloadApi(ApiComponent):
@RequirePerm(Permission.Modify)
def stopAllDownloads(self):
"""Aborts all running downloads."""
-
- pyfiles = self.core.files.cachedFiles()
- for pyfile in pyfiles:
- pyfile.abortDownload()
+ self.core.dlm.abort()
@RequirePerm(Permission.Modify)
def stopDownloads(self, fids):
diff --git a/pyload/api/StatisticsApi.py b/pyload/api/StatisticsApi.py
new file mode 100644
index 000000000..d313e4d0e
--- /dev/null
+++ b/pyload/api/StatisticsApi.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pyload.Api import Api, RequirePerm, Permission
+
+from ApiComponent import ApiComponent
+
+CACHE = {}
+QUOTA_UNLIMITED = -1
+
+
+class StatisticsApi(ApiComponent):
+ """ Retrieve download statistics and quota """
+
+ def recordDownload(self, pyfile):
+ """ Add download record to the statistics """
+ del CACHE[:]
+
+ def calcQuota(self, uid):
+ return QUOTA_UNLIMITED
+
+ def getQuota(self):
+ """ Number of bytes the user has left for download """
+ return self.calcQuota(self.user.true_primary)
+
+
+if Api.extend(StatisticsApi):
+ del StatisticsApi \ No newline at end of file
diff --git a/pyload/api/__init__.py b/pyload/api/__init__.py
index a2b292a27..e25c82a52 100644
--- a/pyload/api/__init__.py
+++ b/pyload/api/__init__.py
@@ -1,5 +1,5 @@
__all__ = ["CoreApi", "ConfigApi", "DownloadApi", "DownloadPreparingApi", "FileApi",
- "UserInteractionApi", "AccountApi", "AddonApi", "UserApi"]
+ "UserInteractionApi", "AccountApi", "AddonApi", "UserApi", "StatisticsApi"]
# Import all components
# from .import *
diff --git a/pyload/config/default.py b/pyload/config/default.py
index 1ad164c31..d4ac84c9b 100644
--- a/pyload/config/default.py
+++ b/pyload/config/default.py
@@ -38,10 +38,9 @@ def make_config(config):
[
("language", "en;de;fr;it;es;nl;sv;ru;pl;cs;sr;pt_BR", _("Language"), "en"),
("download_folder", "folder", _("Download Folder"), "Downloads"),
- ("checksum", "bool", _("Use Checksum"), False),
("folder_per_package", "bool", _("Create folder for each package"), True),
("debug_mode", "bool", _("Debug Mode"), False),
- ("min_free_space", "int", _("Min Free Space (MB)"), 200),
+ ("min_free_space", "int", _("Min Free Space (MB)"), 512),
("renice", "int", _("CPU Priority"), 0),
])
@@ -79,20 +78,23 @@ def make_config(config):
config.addConfigSection("reconnect", _("Reconnect"), _("Description"), _("Long description"),
[
- ("endTime", "time", _("End"), "0:00"),
("activated", "bool", _("Use Reconnect"), False),
("method", "str", _("Method"), "./reconnect.sh"),
+ ("wait", "str", _("Wait for reconnect"), False),
("startTime", "time", _("Start"), "0:00"),
+ ("endTime", "time", _("End"), "0:00"),
])
config.addConfigSection("download", _("Download"), _("Description"), _("Long description"),
[
- ("max_downloads", "int", _("Max Parallel Downloads"), 3),
- ("limit_speed", "bool", _("Limit Download Speed"), False),
+ ("max_downloads", "int", _("Max parallel downloads"), 3),
+ ("wait_downloads", "int", _("Start downloads while waiting"), 2),
+ ("limit_speed", "bool", _("Limit download speed"), False),
("interface", "str", _("Download interface to bind (ip or Name)"), ""),
("skip_existing", "bool", _("Skip already existing files"), False),
- ("max_speed", "int", _("Max Download Speed in kb/s"), -1),
+ ("max_speed", "int", _("Max download speed in kb/s"), -1),
("ipv6", "bool", _("Allow IPv6"), False),
+ ("ssl", "bool", _("Prefer SSL downloads"), True),
("chunks", "int", _("Max connections for one download"), 3),
("restart_failed", "bool", _("Restart failed downloads on startup"), False),
])
diff --git a/pyload/database/AccountDatabase.py b/pyload/database/AccountDatabase.py
index 3ca841fbc..e432af192 100644
--- a/pyload/database/AccountDatabase.py
+++ b/pyload/database/AccountDatabase.py
@@ -7,23 +7,33 @@ from pyload.database import DatabaseMethods, queue, async
class AccountMethods(DatabaseMethods):
@queue
def loadAccounts(self):
- self.c.execute('SELECT plugin, loginname, owner, activated, shared, password, options FROM accounts')
+ self.c.execute('SELECT aid, plugin, loginname, owner, activated, shared, password, options FROM accounts')
- return [(AccountInfo(r[0], r[1], r[2], activated=r[3] is 1, shared=r[4] is 1), r[5], r[6]) for r in self.c]
+ return [(AccountInfo(r[0], r[1], r[2], r[3], activated=r[4] is 1, shared=r[5] is 1), r[6], r[7]) for r in
+ self.c]
+
+ @queue
+ def createAccount(self, plugin, loginname, password, owner):
+ self.c.execute('INSERT INTO accounts(plugin, loginname, password, owner) VALUES(?,?,?,?)',
+ (plugin, loginname, password, owner))
+
+ return self.c.lastrowid
@async
def saveAccounts(self, data):
-
self.c.executemany(
- 'INSERT INTO accounts(plugin, loginname, owner, activated, shared, password, options) VALUES(?,?,?,?,?,?,?)',
+ 'UPDATE accounts SET '
+ 'loginname=?, activated=?, shared=?, password=?, options=? '
+ 'WHERE aid=?',
data)
@async
- def removeAccount(self, plugin, loginname):
- self.c.execute('DELETE FROM accounts WHERE plugin=? AND loginname=?', (plugin, loginname))
+ def removeAccount(self, aid):
+ self.c.execute('DELETE FROM accounts WHERE aid=?', (aid,))
@queue
def purgeAccounts(self):
self.c.execute('DELETE FROM accounts')
+
AccountMethods.register() \ No newline at end of file
diff --git a/pyload/database/DatabaseBackend.py b/pyload/database/DatabaseBackend.py
index df8c6e704..1bdcdc582 100644
--- a/pyload/database/DatabaseBackend.py
+++ b/pyload/database/DatabaseBackend.py
@@ -30,7 +30,7 @@ except:
import sqlite3
DB = None
-DB_VERSION = 6
+DB_VERSION = 7
def set_DB(db):
@@ -298,6 +298,7 @@ class DatabaseBackend(Thread):
)
self.c.execute('CREATE INDEX IF NOT EXISTS "file_index" ON files(package, owner)')
self.c.execute('CREATE INDEX IF NOT EXISTS "file_owner" ON files(owner)')
+ self.c.execute('CREATE INDEX IF NOT EXISTS "file_plugin" ON files(plugin)')
self.c.execute(
'CREATE TRIGGER IF NOT EXISTS "insert_file" AFTER INSERT ON "files"'
@@ -366,30 +367,32 @@ class DatabaseBackend(Thread):
self.c.execute(
'CREATE TABLE IF NOT EXISTS "accounts" ('
+ '"aid" INTEGER PRIMARY KEY AUTOINCREMENT, '
'"plugin" TEXT NOT NULL, '
'"loginname" TEXT NOT NULL, '
- '"owner" INTEGER NOT NULL DEFAULT -1, '
+ '"owner" INTEGER NOT NULL, '
'"activated" INTEGER NOT NULL DEFAULT 1, '
'"password" TEXT DEFAULT "", '
'"shared" INTEGER NOT NULL DEFAULT 0, '
'"options" TEXT DEFAULT "", '
- 'FOREIGN KEY(owner) REFERENCES users(uid), '
- 'PRIMARY KEY (plugin, loginname, owner) ON CONFLICT REPLACE'
+ 'FOREIGN KEY(owner) REFERENCES users(uid)'
')'
)
+ self.c.execute('CREATE INDEX IF NOT EXISTS "accounts_login" ON accounts(plugin, loginname)')
+
self.c.execute(
'CREATE TABLE IF NOT EXISTS "stats" ('
+ '"id" INTEGER PRIMARY KEY AUTOINCREMENT, '
'"user" INTEGER NOT NULL, '
'"plugin" TEXT NOT NULL, '
'"time" INTEGER NOT NULL, '
'"premium" INTEGER DEFAULT 0 NOT NULL, '
'"amount" INTEGER DEFAULT 0 NOT NULL, '
- 'FOREIGN KEY(user) REFERENCES users(uid), '
- 'PRIMARY KEY(user, plugin, time)'
+ 'FOREIGN KEY(user) REFERENCES users(uid)'
')'
)
- self.c.execute('CREATE INDEX IF NOT EXISTS "stats_time" ON stats(time)')
+ self.c.execute('CREATE INDEX IF NOT EXISTS "stats_time" ON stats(user, time)')
#try to lower ids
self.c.execute('SELECT max(fid) FROM files')
diff --git a/pyload/database/FileDatabase.py b/pyload/database/FileDatabase.py
index 219b16663..e6e051a92 100644
--- a/pyload/database/FileDatabase.py
+++ b/pyload/database/FileDatabase.py
@@ -314,6 +314,10 @@ class FileMethods(DatabaseMethods):
f.hash, f.status, f.error, f.fid))
@async
+ def setDownloadStatus(self, fid, status):
+ self.c.execute('UPDATE files SET dlstatus=? WHERE fid=?', (status, fid))
+
+ @async
def updatePackage(self, p):
self.c.execute(
'UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, tags=?, status=?, shared=? WHERE pid=?',
@@ -383,22 +387,28 @@ class FileMethods(DatabaseMethods):
# status -> queued
self.c.execute('UPDATE files SET status=3 WHERE package=?', (pid,))
-
- # TODO: multi user approach
@queue
- def getJob(self, occ):
+ def getJobs(self, occ):
"""return pyfile ids, which are suitable for download and don't use a occupied plugin"""
cmd = "(%s)" % ", ".join(["'%s'" % x for x in occ])
- #TODO
- # dlstatus in online, queued | package status = ok
- cmd = ("SELECT f.fid FROM files as f INNER JOIN packages as p ON f.package=p.pid "
- "WHERE f.plugin NOT IN %s AND f.dlstatus IN (2,3) AND p.status=0 "
- "ORDER BY p.packageorder ASC, f.fileorder ASC LIMIT 5") % cmd
+ # dlstatus in online, queued, occupied | package status = ok
+ cmd = ("SELECT f.owner, f.fid FROM files as f INNER JOIN packages as p ON f.package=p.pid "
+ "WHERE f.owner=? AND f.plugin NOT IN %s AND f.dlstatus IN (2,3,16) AND p.status=0 "
+ "ORDER BY p.packageorder ASC, f.fileorder ASC LIMIT 1") % cmd
+
- self.c.execute(cmd)
+ self.c.execute("SELECT uid FROM users")
+ uids = self.c.fetchall()
+ jobs = {}
+ # get jobs for all uids
+ for uid in uids:
+ self.c.execute(cmd, uid)
+ r = self.c.fetchone()
+ if r:
+ jobs[r[0]] = r[1]
- return [x[0] for x in self.c]
+ return jobs
@queue
def getUnfinished(self, pid):
diff --git a/pyload/database/StatisticDatabase.py b/pyload/database/StatisticDatabase.py
index d5f9658f2..5dd5ec7ed 100644
--- a/pyload/database/StatisticDatabase.py
+++ b/pyload/database/StatisticDatabase.py
@@ -3,11 +3,10 @@
from pyload.database import DatabaseMethods, queue, async, inner
-# TODO
class StatisticMethods(DatabaseMethods):
- pass
-
+ def addEntry(self, user, plugin, premium, amount):
+ pass
StatisticMethods.register() \ No newline at end of file
diff --git a/pyload/database/UserDatabase.py b/pyload/database/UserDatabase.py
index 8d8381a40..14b4ae40c 100644
--- a/pyload/database/UserDatabase.py
+++ b/pyload/database/UserDatabase.py
@@ -44,6 +44,14 @@ class UserMethods(DatabaseMethods):
self.c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password))
@queue
+ def addDebugUser(self, uid):
+ # just add a user with uid to db
+ try:
+ self.c.execute('INSERT INTO users (uid, name, password) VALUES (?, ?, ?)', (uid, "debugUser", random_salt()))
+ except:
+ pass
+
+ @queue
def getUserData(self, name=None, uid=None):
qry = ('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, '
'hddquota, user, template FROM "users" WHERE ')
diff --git a/pyload/datatypes/PyFile.py b/pyload/datatypes/PyFile.py
index 18ac06c50..b83a057aa 100644
--- a/pyload/datatypes/PyFile.py
+++ b/pyload/datatypes/PyFile.py
@@ -41,10 +41,11 @@ statusMap = {
"not possible": 13,
"missing": 14,
"file mismatch": 15,
- "decrypting": 16,
- "processing": 17,
- "custom": 18,
- "unknown": 19,
+ "occupied": 16,
+ "decrypting": 17,
+ "processing": 18,
+ "custom": 19,
+ "unknown": 20,
}
@@ -205,7 +206,7 @@ class PyFile(object):
def abortDownload(self):
"""abort pyfile if possible"""
# TODO: abort timeout, currently dead locks
- while self.id in self.m.core.threadManager.processingIds():
+ while self.id in self.m.core.dlm.processingIds():
self.abort = True
if self.plugin and self.plugin.req:
self.plugin.req.abort()
@@ -225,7 +226,7 @@ class PyFile(object):
def finishIfDone(self):
"""set status to finish and release file if every thread is finished with it"""
- if self.id in self.m.core.threadManager.processingIds():
+ if self.id in self.m.core.dlm.processingIds():
return False
self.setStatus("finished")
diff --git a/pyload/datatypes/User.py b/pyload/datatypes/User.py
index 645fd0983..fbfb24378 100644
--- a/pyload/datatypes/User.py
+++ b/pyload/datatypes/User.py
@@ -54,6 +54,9 @@ class User(UserData):
def isAdmin(self):
return self.hasRole(Role.Admin)
+ def isOwner(self, obj):
+ return self.primary is None or obj.owner == self.true_primary
+
@property
def primary(self):
""" Primary user id, Internal user handle used for most operations
diff --git a/pyload/network/__init__.py b/pyload/network/__init__.py
index 8b1378917..1abafc327 100644
--- a/pyload/network/__init__.py
+++ b/pyload/network/__init__.py
@@ -1 +1,30 @@
+# -*- coding: utf-8 -*-
+import re
+from random import choice
+from time import sleep
+
+from RequestFactory import getURL
+
+
+def get_ip(n=10):
+ """retrieve current ip. try n times for n seconds"""
+ services = [
+ ("http://checkip.dyndns.org", r".*Current IP Address: (\S+)</body>.*"),
+ ("http://myexternalip.com/raw", r"(\S+)"),
+ ("http://icanhazip.com", r"(\S+)"),
+ ("http://ifconfig.me/ip", r"(\S+)")
+ ]
+
+ ip = ""
+ for i in range(n):
+ try:
+ sv = choice(services)
+ ip = getURL(sv[0])
+ ip = re.match(sv[1], ip).group(1)
+ break
+ except:
+ ip = ""
+ sleep(1)
+
+ return ip
diff --git a/pyload/plugins/Account.py b/pyload/plugins/Account.py
index cbf545611..e81609971 100644
--- a/pyload/plugins/Account.py
+++ b/pyload/plugins/Account.py
@@ -42,14 +42,15 @@ class Account(Base):
@classmethod
def fromInfoData(cls, m, info, password, options):
- return cls(m, info.loginname, info.owner,
+ return cls(m, info.aid, info.loginname, info.owner,
True if info.activated else False, True if info.shared else False, password, options)
__type__ = "account"
- def __init__(self, manager, loginname, owner, activated, shared, password, options):
+ def __init__(self, manager, aid, loginname, owner, activated, shared, password, options):
Base.__init__(self, manager.core, owner)
+ self.aid = aid
self.loginname = loginname
self.owner = owner
self.activated = activated
@@ -74,7 +75,7 @@ class Account(Base):
self.init()
def toInfoData(self):
- info = AccountInfo(self.__name__, self.loginname, self.owner, self.valid, self.validuntil, self.trafficleft,
+ info = AccountInfo(self.aid, self.__name__, self.loginname, self.owner, self.valid, self.validuntil, self.trafficleft,
self.maxtraffic, self.premium, self.activated, self.shared, self.options)
info.config = [ConfigItem(name, item.label, item.description, item.input,
@@ -158,13 +159,14 @@ class Account(Base):
self.maxtraffic = Account.maxtraffic
self.premium = Account.premium
- def setPassword(self, password):
- """ updates the password and returns true if anything changed """
+ def setLogin(self, loginname, password):
+ """ updates the loginname and password and returns true if anything changed """
- if password != self.password:
+ if password != self.password or loginname != self.loginname:
self.login_ts = 0
self.valid = True #set valid, so the login will be retried
+ self.loginname = loginname
self.password = password
return True
diff --git a/pyload/plugins/addons/MultiHoster.py b/pyload/plugins/addons/MultiHoster.py
index 871defb05..ff5da32ae 100644
--- a/pyload/plugins/addons/MultiHoster.py
+++ b/pyload/plugins/addons/MultiHoster.py
@@ -90,7 +90,7 @@ class MultiHoster(Addon, PluginMatcher):
@AddEventListener(["account:deleted", "account:updated"])
- def refreshAccounts(self, plugin=None, loginname=None):
+ def refreshAccounts(self, plugin=None, loginname=None, user=None):
self.logDebug("Re-checking accounts")
self.plugins = {}
diff --git a/pyload/remote/apitypes.py b/pyload/remote/apitypes.py
index ab68116c3..4c66800c6 100644
--- a/pyload/remote/apitypes.py
+++ b/pyload/remote/apitypes.py
@@ -36,10 +36,11 @@ class DownloadStatus:
NotPossible = 13
Missing = 14
FileMismatch = 15
- Decrypting = 16
- Processing = 17
- Custom = 18
- Unknown = 19
+ Occupied = 16
+ Decrypting = 17
+ Processing = 18
+ Custom = 19
+ Unknown = 20
class FileStatus:
Ok = 0
@@ -55,13 +56,15 @@ class InputType:
Textbox = 5
Password = 6
Time = 7
- Bool = 8
- Click = 9
- Select = 10
- Multiple = 11
- List = 12
- PluginList = 13
- Table = 14
+ TimeSpan = 8
+ ByteSize = 9
+ Bool = 10
+ Click = 11
+ Select = 12
+ Multiple = 13
+ List = 14
+ PluginList = 15
+ Table = 16
class Interaction:
All = 0
@@ -109,9 +112,10 @@ class Role:
User = 1
class AccountInfo(BaseObject):
- __slots__ = ['plugin', 'loginname', 'owner', 'valid', 'validuntil', 'trafficleft', 'maxtraffic', 'premium', 'activated', 'shared', 'config']
+ __slots__ = ['aid', 'plugin', 'loginname', 'owner', 'valid', 'validuntil', 'trafficleft', 'maxtraffic', 'premium', 'activated', 'shared', 'config']
- def __init__(self, plugin=None, loginname=None, owner=None, valid=None, validuntil=None, trafficleft=None, maxtraffic=None, premium=None, activated=None, shared=None, config=None):
+ def __init__(self, aid=None, plugin=None, loginname=None, owner=None, valid=None, validuntil=None, trafficleft=None, maxtraffic=None, premium=None, activated=None, shared=None, config=None):
+ self.aid = aid
self.plugin = plugin
self.loginname = loginname
self.owner = owner
@@ -322,20 +326,6 @@ class ProgressInfo(BaseObject):
self.type = type
self.download = download
-class ServerStatus(BaseObject):
- __slots__ = ['speed', 'linkstotal', 'linksqueue', 'sizetotal', 'sizequeue', 'notifications', 'paused', 'download', 'reconnect']
-
- def __init__(self, speed=None, linkstotal=None, linksqueue=None, sizetotal=None, sizequeue=None, notifications=None, paused=None, download=None, reconnect=None):
- self.speed = speed
- self.linkstotal = linkstotal
- self.linksqueue = linksqueue
- self.sizetotal = sizetotal
- self.sizequeue = sizequeue
- self.notifications = notifications
- self.paused = paused
- self.download = download
- self.reconnect = reconnect
-
class ServiceDoesNotExist(ExceptionObject):
__slots__ = ['plugin', 'func']
@@ -349,6 +339,21 @@ class ServiceException(ExceptionObject):
def __init__(self, msg=None):
self.msg = msg
+class StatusInfo(BaseObject):
+ __slots__ = ['speed', 'linkstotal', 'linksqueue', 'sizetotal', 'sizequeue', 'notifications', 'paused', 'download', 'reconnect', 'quota']
+
+ def __init__(self, speed=None, linkstotal=None, linksqueue=None, sizetotal=None, sizequeue=None, notifications=None, paused=None, download=None, reconnect=None, quota=None):
+ self.speed = speed
+ self.linkstotal = linkstotal
+ self.linksqueue = linksqueue
+ self.sizetotal = sizetotal
+ self.sizequeue = sizequeue
+ self.notifications = notifications
+ self.paused = paused
+ self.download = download
+ self.reconnect = reconnect
+ self.quota = quota
+
class TreeCollection(BaseObject):
__slots__ = ['root', 'files', 'packages']
@@ -402,6 +407,8 @@ class Iface(object):
pass
def checkLinks(self, links):
pass
+ def createAccount(self, plugin, loginname, password):
+ pass
def createPackage(self, name, folder, root, password, site, comment, paused):
pass
def deleteConfig(self, plugin):
@@ -420,7 +427,7 @@ class Iface(object):
pass
def generatePackages(self, links):
pass
- def getAccountInfo(self, plugin, loginname, refresh):
+ def getAccountInfo(self, aid, plugin, refresh):
pass
def getAccountTypes(self):
pass
@@ -464,10 +471,12 @@ class Iface(object):
pass
def getProgressInfo(self):
pass
- def getServerStatus(self):
+ def getQuota(self):
pass
def getServerVersion(self):
pass
+ def getStatusInfo(self):
+ pass
def getUserData(self):
pass
def getWSAddress(self):
@@ -538,7 +547,7 @@ class Iface(object):
pass
def unpauseServer(self):
pass
- def updateAccount(self, plugin, loginname, password):
+ def updateAccount(self, aid, plugin, loginname, password):
pass
def updateAccountInfo(self, account):
pass
diff --git a/pyload/remote/apitypes_debug.py b/pyload/remote/apitypes_debug.py
index d5e5f36a0..a30009bad 100644
--- a/pyload/remote/apitypes_debug.py
+++ b/pyload/remote/apitypes_debug.py
@@ -19,7 +19,7 @@ enums = [
]
classes = {
- 'AccountInfo' : [basestring, basestring, int, bool, int, int, int, bool, bool, bool, (list, ConfigItem)],
+ 'AccountInfo' : [int, basestring, basestring, int, bool, int, int, int, bool, bool, bool, (list, ConfigItem)],
'AddonInfo' : [basestring, basestring, basestring],
'AddonService' : [basestring, basestring, basestring, (list, basestring), bool, int],
'ConfigHolder' : [basestring, basestring, basestring, basestring, (list, ConfigItem), (None, (list, AddonInfo))],
@@ -39,9 +39,9 @@ classes = {
'PackageInfo' : [int, basestring, basestring, int, int, basestring, basestring, basestring, int, (list, basestring), int, bool, int, PackageStats, (list, int), (list, int)],
'PackageStats' : [int, int, int, int],
'ProgressInfo' : [basestring, basestring, basestring, int, int, int, int, int, (None, DownloadProgress)],
- 'ServerStatus' : [int, int, int, int, int, bool, bool, bool, bool],
'ServiceDoesNotExist' : [basestring, basestring],
'ServiceException' : [basestring],
+ 'StatusInfo' : [int, int, int, int, int, bool, bool, bool, bool, int],
'TreeCollection' : [PackageInfo, (dict, int, FileInfo), (dict, int, PackageInfo)],
'UserData' : [int, basestring, basestring, int, int, basestring, int, int, basestring, int, int, basestring],
'UserDoesNotExist' : [basestring],
@@ -57,6 +57,7 @@ methods = {
'checkContainer': OnlineCheck,
'checkHTML': OnlineCheck,
'checkLinks': OnlineCheck,
+ 'createAccount': AccountInfo,
'createPackage': int,
'deleteConfig': None,
'deleteFiles': bool,
@@ -88,8 +89,9 @@ methods = {
'getPackageInfo': PackageInfo,
'getPluginConfig': (list, ConfigInfo),
'getProgressInfo': (list, ProgressInfo),
- 'getServerStatus': ServerStatus,
+ 'getQuota': int,
'getServerVersion': basestring,
+ 'getStatusInfo': StatusInfo,
'getUserData': UserData,
'getWSAddress': basestring,
'invokeAddon': basestring,
diff --git a/pyload/remote/pyload.thrift b/pyload/remote/pyload.thrift
index a9431ea7c..9d400c4e2 100644
--- a/pyload/remote/pyload.thrift
+++ b/pyload/remote/pyload.thrift
@@ -5,6 +5,7 @@ typedef i32 PackageID
typedef i32 ResultID
typedef i32 InteractionID
typedef i32 UserID
+typedef i32 AccountID
typedef i64 UTCDate
typedef i64 ByteCount
typedef list<string> LinkList
@@ -28,6 +29,7 @@ enum DownloadStatus {
NotPossible,
Missing,
FileMismatch,
+ Occupied,
Decrypting,
Processing,
Custom,
@@ -71,7 +73,6 @@ enum PackageStatus {
// types for user interaction
// some may only be place holder currently not supported
// also all input - output combination are not reasonable, see InteractionManager for further info
-// Todo: how about: time, ip, s.o.
enum InputType {
NA,
Text,
@@ -81,6 +82,8 @@ enum InputType {
Textbox,
Password,
Time,
+ TimeSpan,
+ ByteSize, // size in bytes
Bool, // confirm like, yes or no dialog
Click, // for positional captchas
Select, // select from list
@@ -217,7 +220,7 @@ struct LinkStatus {
6: optional string hash
}
-struct ServerStatus {
+struct StatusInfo {
1: ByteCount speed,
2: i16 linkstotal,
3: i16 linksqueue,
@@ -227,6 +230,7 @@ struct ServerStatus {
7: bool paused,
8: bool download,
9: bool reconnect,
+ 10: ByteCount quota
}
struct InteractionTask {
@@ -300,17 +304,18 @@ struct UserData {
}
struct AccountInfo {
- 1: PluginName plugin,
- 2: string loginname,
- 3: UserID owner,
- 4: bool valid,
- 5: UTCDate validuntil,
- 6: ByteCount trafficleft,
- 7: ByteCount maxtraffic,
- 8: bool premium,
- 9: bool activated,
- 10: bool shared,
- 11: list <ConfigItem> config,
+ 1: AccountID aid,
+ 2: PluginName plugin,
+ 3: string loginname,
+ 4: UserID owner,
+ 5: bool valid,
+ 6: UTCDate validuntil,
+ 7: ByteCount trafficleft,
+ 8: ByteCount maxtraffic,
+ 9: bool premium,
+ 10: bool activated,
+ 11: bool shared,
+ 13: list <ConfigItem> config,
}
struct OnlineCheck {
@@ -363,7 +368,7 @@ service Pyload {
string getServerVersion(),
string getWSAddress(),
- ServerStatus getServerStatus(),
+ StatusInfo getStatusInfo(),
list<ProgressInfo> getProgressInfo(),
list<string> getLog(1: i32 offset),
@@ -500,9 +505,10 @@ service Pyload {
list<string> getAccountTypes(),
list<AccountInfo> getAccounts(),
- AccountInfo getAccountInfo(1: PluginName plugin, 2: string loginname, 3: bool refresh),
+ AccountInfo getAccountInfo(1: AccountID aid, 2: PluginName plugin, 3: bool refresh),
- AccountInfo updateAccount(1: PluginName plugin, 2: string loginname, 3: string password),
+ AccountInfo createAccount(1: PluginName plugin, 2: string loginname, 3: string password),
+ AccountInfo updateAccount(1:AccountID aid, 2: PluginName plugin, 3: string loginname, 4: string password),
void updateAccountInfo(1: AccountInfo account),
void removeAccount(1: AccountInfo account),
@@ -542,6 +548,15 @@ service Pyload {
JSONString invokeAddonHandler(1: PluginName plugin, 2: string func, 3: PackageID pid_or_fid)
throws (1: ServiceDoesNotExist e, 2: ServiceException ex),
+
+ ///////////////////////
+ // Statistics Api
+ ///////////////////////
+
+ ByteCount getQuota(),
+
+
+
///////////////////////
// Media finder
///////////////////////
diff --git a/pyload/remote/wsbackend/AsyncHandler.py b/pyload/remote/wsbackend/AsyncHandler.py
index c7a26cd6b..bf931d70d 100644
--- a/pyload/remote/wsbackend/AsyncHandler.py
+++ b/pyload/remote/wsbackend/AsyncHandler.py
@@ -171,12 +171,8 @@ class AsyncHandler(AbstractHandler):
pass
if req.t <= time():
- # TODO: server status is not enough
- # modify core api to include progress? think of other needed information to show
- # eta is quite wrong currently
- # notifications
- self.send(req, self.api.getServerStatus())
- self.send(req, self.api.getProgressInfo())
+ self.send(req, req.api.getStatusInfo())
+ self.send(req, req.api.getProgressInfo())
# update time for next update
req.t = time() + req.interval \ No newline at end of file
diff --git a/pyload/threads/DecrypterThread.py b/pyload/threads/DecrypterThread.py
index 9f796da22..419f153a2 100644
--- a/pyload/threads/DecrypterThread.py
+++ b/pyload/threads/DecrypterThread.py
@@ -23,7 +23,6 @@ class DecrypterThread(BaseThread):
# holds the progress, while running
self.progress = None
- self.m.addThread(self)
self.start()
def getProgress(self):
diff --git a/pyload/threads/DownloadThread.py b/pyload/threads/DownloadThread.py
index d1672531b..b8f7e4965 100644
--- a/pyload/threads/DownloadThread.py
+++ b/pyload/threads/DownloadThread.py
@@ -18,6 +18,7 @@
@author: RaNaN
"""
+from threading import Event
from Queue import Queue
from time import sleep, time
from traceback import print_exc
@@ -37,6 +38,9 @@ class DownloadThread(BaseThread):
"""Constructor"""
BaseThread.__init__(self, manager)
+ self.isWorking = Event()
+ self.isWorking.clear()
+
self.queue = Queue() # job queue
self.active = None
@@ -53,12 +57,19 @@ class DownloadThread(BaseThread):
if self.active == "quit":
self.active = None
- self.m.threads.remove(self)
+ self.m.stop(self)
return True
try:
- if not pyfile.hasPlugin(): continue
+ pyfile.initPlugin()
+
+ # after initialization the thread is fully ready
+ self.isWorking.set()
+
#this pyfile was deleted while queuing
+ # TODO: what will happen with new thread manager?
+ #if not pyfile.hasPlugin(): continue
+
pyfile.plugin.checkForSameFiles(starting=True)
self.log.info(_("Download starts: %s" % pyfile.name))
@@ -204,7 +215,9 @@ class DownloadThread(BaseThread):
self.core.files.save()
pyfile.checkIfProcessed()
exc_clear()
-
+ # manager could still be waiting for it
+ self.isWorking.set()
+ self.m.done(self)
#pyfile.plugin.req.clean()
diff --git a/pyload/threads/ThreadManager.py b/pyload/threads/ThreadManager.py
index 298b0402d..f6cb3daea 100644
--- a/pyload/threads/ThreadManager.py
+++ b/pyload/threads/ThreadManager.py
@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
###############################################################################
-# Copyright(c) 2008-2013 pyLoad Team
+# Copyright(c) 2008-2014 pyLoad Team
# http://www.pyload.org
#
# This file is part of pyLoad.
@@ -16,22 +16,11 @@
# @author: RaNaN
###############################################################################
-from os.path import exists, join
-import re
-from subprocess import Popen
-from threading import Event, RLock
-from time import sleep, time
-from traceback import print_exc
-from random import choice
+from threading import RLock
+from time import time
-from pyload.datatypes.PyFile import PyFile
from pyload.datatypes.OnlineCheck import OnlineCheck
-from pyload.network.RequestFactory import getURL
-from pyload.utils import lock, uniqify, to_list
-from pyload.utils.fs import free_space
-
-from DecrypterThread import DecrypterThread
-from DownloadThread import DownloadThread
+from pyload.utils import lock, to_list
from InfoThread import InfoThread
@@ -44,13 +33,6 @@ class ThreadManager:
self.log = core.log
self.threads = [] # thread list
- self.localThreads = [] #addon+decrypter threads
-
- self.pause = True
-
- self.reconnecting = Event()
- self.reconnecting.clear()
- self.downloaded = 0 #number of files downloaded since last cleanup
self.lock = RLock()
@@ -67,24 +49,15 @@ class ThreadManager:
# timeout for cache purge
self.timestamp = 0
- for i in range(self.core.config.get("download", "max_downloads")):
- self.createThread()
-
- def createThread(self):
- """create a download thread"""
-
- thread = DownloadThread(self)
- self.threads.append(thread)
-
@lock
def addThread(self, thread):
- self.localThreads.append(thread)
+ self.threads.append(thread)
@lock
def removeThread(self, thread):
""" Remove a thread from the local list """
- if thread in self.localThreads:
- self.localThreads.remove(thread)
+ if thread in self.threads:
+ self.threads.remove(thread)
@lock
def createInfoThread(self, data, pid):
@@ -108,11 +81,6 @@ class ThreadManager:
return rid
@lock
- def createDecryptThread(self, data, pid):
- """ Start decrypting of entered data, all links in one package are accumulated to one thread."""
- if data: DecrypterThread(self, data, pid)
-
- @lock
def getInfoResult(self, rid):
return self.infoResults.get(rid)
@@ -120,14 +88,10 @@ class ThreadManager:
self.core.evm.dispatchEvent("linkcheck:updated", oc.rid, result, owner=oc.owner)
oc.update(result)
- def getActiveDownloads(self, user=None):
- # TODO: user context
- return [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)]
-
def getProgressList(self, user=None):
info = []
- for thread in self.threads + self.localThreads:
+ for thread in self.threads:
# skip if not belong to current user
if user is not None and thread.owner != user: continue
@@ -136,38 +100,8 @@ class ThreadManager:
return info
- def getActiveFiles(self):
- active = self.getActiveDownloads()
-
- for t in self.localThreads:
- active.extend(t.getActiveFiles())
-
- return active
-
- def processingIds(self):
- """get a id list of all pyfiles processed"""
- return [x.id for x in self.getActiveFiles()]
-
def work(self):
"""run all task which have to be done (this is for repetitive call by core)"""
- try:
- self.tryReconnect()
- except Exception, e:
- self.log.error(_("Reconnect Failed: %s") % str(e))
- self.reconnecting.clear()
- self.core.print_exc()
-
- self.checkThreadCount()
-
- try:
- self.assignJob()
- except Exception, e:
- self.log.warning("Assign job error", e)
- self.core.print_exc()
-
- sleep(0.5)
- self.assignJob()
- #it may be failed non critical so we try it again
if self.infoCache and self.timestamp < time():
self.infoCache.clear()
@@ -176,141 +110,3 @@ class ThreadManager:
for rid in self.infoResults.keys():
if self.infoResults[rid].isStale():
del self.infoResults[rid]
-
- def tryReconnect(self):
- """checks if reconnect needed"""
-
- if not (self.core.config["reconnect"]["activated"] and self.core.api.isTimeReconnect()):
- return False
-
- active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active]
-
- if not (0 < active.count(True) == len(active)):
- return False
-
- if not exists(self.core.config['reconnect']['method']):
- if exists(join(pypath, self.core.config['reconnect']['method'])):
- self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method'])
- else:
- self.core.config["reconnect"]["activated"] = False
- self.log.warning(_("Reconnect script not found!"))
- return
-
- self.reconnecting.set()
-
- #Do reconnect
- self.log.info(_("Starting reconnect"))
-
- while [x.active.plugin.waiting for x in self.threads if x.active].count(True) != 0:
- sleep(0.25)
-
- ip = self.getIP()
-
- self.core.evm.dispatchEvent("reconnect:before", ip)
-
- self.log.debug("Old IP: %s" % ip)
-
- try:
- reconn = Popen(self.core.config['reconnect']['method'], bufsize=-1, shell=True)#, stdout=subprocess.PIPE)
- except:
- self.log.warning(_("Failed executing reconnect script!"))
- self.core.config["reconnect"]["activated"] = False
- self.reconnecting.clear()
- self.core.print_exc()
- return
-
- reconn.wait()
- sleep(1)
- ip = self.getIP()
- self.core.evm.dispatchEvent("reconnect:after", ip)
-
- self.log.info(_("Reconnected, new IP: %s") % ip)
-
- self.reconnecting.clear()
-
- def getIP(self):
- """retrieve current ip"""
- services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"),
- ("http://checkip.dyndns.org/", ".*Current IP Address: (\S+)</body>.*")]
-
- ip = ""
- for i in range(10):
- try:
- sv = choice(services)
- ip = getURL(sv[0])
- ip = re.match(sv[1], ip).group(1)
- break
- except:
- ip = ""
- sleep(1)
-
- return ip
-
- def checkThreadCount(self):
- """checks if there is a need for increasing or reducing thread count"""
-
- if len(self.threads) == self.core.config.get("download", "max_downloads"):
- return True
- elif len(self.threads) < self.core.config.get("download", "max_downloads"):
- self.createThread()
- else:
- free = [x for x in self.threads if not x.active]
- if free:
- free[0].put("quit")
-
-
- def cleanPycurl(self):
- """ make a global curl cleanup (currently unused) """
- if self.processingIds():
- return False
- import pycurl
-
- pycurl.global_cleanup()
- pycurl.global_init(pycurl.GLOBAL_DEFAULT)
- self.downloaded = 0
- self.log.debug("Cleaned up pycurl")
- return True
-
-
- def assignJob(self):
- """assign a job to a thread if possible"""
-
- if self.pause or not self.core.api.isTimeDownload(): return
-
- #if self.downloaded > 20:
- # if not self.cleanPyCurl(): return
-
- free = [x for x in self.threads if not x.active]
-
- inuse = [(x.active.pluginname, x.active.plugin.getDownloadLimit()) for x in self.threads if
- x.active and x.active.hasPlugin()]
- inuse = [(x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) for x in
- inuse]
- occ = tuple(sorted(uniqify([x[0] for x in inuse if 0 < x[1] <= x[2]])))
-
- job = self.core.files.getJob(occ)
- if job:
- try:
- job.initPlugin()
- except Exception, e:
- self.log.critical(str(e))
- print_exc()
- job.setStatus("failed")
- job.error = str(e)
- job.release()
- return
-
- spaceLeft = free_space(self.core.config["general"]["download_folder"]) / 1024 / 1024
- if spaceLeft < self.core.config["general"]["min_free_space"]:
- self.log.warning(_("Not enough space left on device"))
- self.pause = True
-
- if free and not self.pause:
- thread = free[0]
- #self.downloaded += 1
- thread.put(job)
- else:
- #put job back
- if occ not in self.core.files.jobCache:
- self.core.files.jobCache[occ] = []
- self.core.files.jobCache[occ].append(job.id)
diff --git a/pyload/utils/PluginLoader.py b/pyload/utils/PluginLoader.py
index 57a899e39..743f0e537 100644
--- a/pyload/utils/PluginLoader.py
+++ b/pyload/utils/PluginLoader.py
@@ -40,6 +40,7 @@ class BaseAttributes(defaultdict):
return getattr(Base, attr)
+
class LoaderFactory:
""" Container for multiple plugin loaders """
@@ -62,6 +63,15 @@ class LoaderFactory:
if l2 is not loader:
l2.removePlugin(plugin_type, plugin, info.version)
+ def findPlugin(self, name):
+ """ Finds a plugin type for given name """
+ for loader in self.loader:
+ for t in loader.TYPES:
+ if loader.hasPlugin(t, name):
+ return t
+
+ return None
+
def getPlugin(self, plugin, name):
""" retrieve a plugin from an available loader """
for loader in self.loader:
@@ -79,7 +89,7 @@ class PluginLoader:
SINGLE = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?<!")"(?!")|\').*(?:(?<!")"(?!")|\'))',
re.I)
# finds the beginning of a expression that could span multiple lines
- MULTI = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(\(|\{|\[|"{3})',re.I)
+ MULTI = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(\(|\{|\[|"{3})', re.I)
# closing symbols
MULTI_MATCH = {
@@ -185,14 +195,14 @@ class PluginLoader:
#TODO: strings must be parsed too, otherwise breaks very easily
for i in xrange(m.end(2), len(content) - size + 1):
- if content[i:i+size] == endchar:
+ if content[i:i + size] == endchar:
# closing char seen and match now complete
if stack == 0:
endpos = i
break
else:
stack -= 1
- elif content[i:i+size] == char:
+ elif content[i:i + size] == char:
stack += 1
# in case the end was not found match will be empty
diff --git a/pyload/web/app/scripts/models/Account.js b/pyload/web/app/scripts/models/Account.js
index 26241d8e3..b9fc40036 100644
--- a/pyload/web/app/scripts/models/Account.js
+++ b/pyload/web/app/scripts/models/Account.js
@@ -3,9 +3,10 @@ define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', './ConfigIt
return Backbone.Model.extend({
- idAttribute: 'loginname',
+ idAttribute: 'aid',
defaults: {
+ aid: null,
plugin: null,
loginname: null,
owner: -1,
@@ -53,14 +54,15 @@ define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', './ConfigIt
var refresh = _.has(options, 'refresh') && options.refresh;
options = App.apiRequest('getAccountInfo',
{plugin: this.get('plugin'),
- loginname: this.get('loginname'), refresh: refresh}, options);
+ aid: this.get('aid'), refresh: refresh}, options);
return Backbone.Model.prototype.fetch.call(this, options);
},
setPassword: function(password, options) {
options = App.apiRequest('updateAccount',
- {plugin: this.get('plugin'), loginname: this.get('loginname'), password: password}, options);
+ {aid: this.get('aid'),
+ plugin: this.get('plugin'), loginname: this.get('loginname'), password: password}, options);
return $.ajax(options);
},
diff --git a/pyload/web/app/scripts/models/ServerStatus.js b/pyload/web/app/scripts/models/StatusInfo.js
index 59739b41e..8712defa7 100644
--- a/pyload/web/app/scripts/models/ServerStatus.js
+++ b/pyload/web/app/scripts/models/StatusInfo.js
@@ -13,7 +13,8 @@ define(['jquery', 'backbone', 'underscore'],
notifications: -1,
paused: false,
download: false,
- reconnect: false
+ reconnect: false,
+ quota: -1
},
// Model Constructor
@@ -23,7 +24,7 @@ define(['jquery', 'backbone', 'underscore'],
fetch: function(options) {
options || (options = {});
- options.url = 'api/getServerStatus';
+ options.url = 'api/getStatusInfo';
return Backbone.Model.prototype.fetch.call(this, options);
},
diff --git a/pyload/web/app/scripts/utils/apitypes.js b/pyload/web/app/scripts/utils/apitypes.js
index fc92425de..88123f7ea 100644
--- a/pyload/web/app/scripts/utils/apitypes.js
+++ b/pyload/web/app/scripts/utils/apitypes.js
@@ -4,9 +4,9 @@ define([], function() {
'use strict';
return {
DownloadState: {'Failed': 3, 'All': 0, 'Unmanaged': 4, 'Finished': 1, 'Unfinished': 2},
- DownloadStatus: {'NotPossible': 13, 'FileMismatch': 15, 'Downloading': 10, 'Missing': 14, 'NA': 0, 'Processing': 17, 'Waiting': 9, 'Decrypting': 16, 'Paused': 4, 'Failed': 7, 'Finished': 5, 'Skipped': 6, 'Unknown': 19, 'Aborted': 12, 'Online': 2, 'TempOffline': 11, 'Offline': 1, 'Custom': 18, 'Starting': 8, 'Queued': 3},
+ DownloadStatus: {'NotPossible': 13, 'FileMismatch': 15, 'Downloading': 10, 'Missing': 14, 'NA': 0, 'Processing': 18, 'Waiting': 9, 'Decrypting': 17, 'Paused': 4, 'Failed': 7, 'Finished': 5, 'Skipped': 6, 'Unknown': 20, 'Aborted': 12, 'Online': 2, 'Starting': 8, 'TempOffline': 11, 'Offline': 1, 'Custom': 19, 'Occupied': 16, 'Queued': 3},
FileStatus: {'Remote': 2, 'Ok': 0, 'Missing': 1},
- InputType: {'PluginList': 13, 'Multiple': 11, 'Int': 2, 'NA': 0, 'Time': 7, 'List': 12, 'Bool': 8, 'File': 3, 'Text': 1, 'Table': 14, 'Folder': 4, 'Password': 6, 'Click': 9, 'Select': 10, 'Textbox': 5},
+ InputType: {'PluginList': 15, 'Multiple': 13, 'TimeSpan': 8, 'Int': 2, 'ByteSize': 9, 'Time': 7, 'List': 14, 'Textbox': 5, 'Bool': 10, 'File': 3, 'NA': 0, 'Table': 16, 'Folder': 4, 'Password': 6, 'Click': 11, 'Select': 12, 'Text': 1},
Interaction: {'Captcha': 2, 'All': 0, 'Query': 4, 'Notification': 1},
MediaType: {'All': 0, 'Audio': 2, 'Image': 4, 'Executable': 64, 'Other': 1, 'Video': 8, 'Document': 16, 'Archive': 32},
PackageStatus: {'Paused': 1, 'Remote': 3, 'Folder': 2, 'Ok': 0},
diff --git a/pyload/web/app/scripts/views/accounts/accountModal.js b/pyload/web/app/scripts/views/accounts/accountModal.js
index 31e05dff6..11eed1355 100644
--- a/pyload/web/app/scripts/views/accounts/accountModal.js
+++ b/pyload/web/app/scripts/views/accounts/accountModal.js
@@ -59,7 +59,7 @@ define(['jquery', 'underscore', 'app', 'views/abstract/modalView', 'hbs!tpl/dial
password = this.$('#password').val(),
self = this;
- $.ajax(App.apiRequest('updateAccount', {
+ $.ajax(App.apiRequest('createAccount', {
plugin: plugin, loginname: login, password: password
}, { success: function(data) {
App.vent.trigger('account:updated', data);
diff --git a/pyload/web/app/scripts/views/headerView.js b/pyload/web/app/scripts/views/headerView.js
index d4d07ac39..a12248a7a 100644
--- a/pyload/web/app/scripts/views/headerView.js
+++ b/pyload/web/app/scripts/views/headerView.js
@@ -1,8 +1,8 @@
-define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus',
+define(['jquery', 'underscore', 'backbone', 'app', 'models/StatusInfo',
'views/progressView', 'views/notificationView', 'helpers/formatSize', 'hbs!tpl/header/layout',
'hbs!tpl/header/status', 'hbs!tpl/header/progressbar', 'hbs!tpl/header/progressSup', 'hbs!tpl/header/progressSub' , 'flot'],
function(
- $, _, Backbone, App, ServerStatus, ProgressView, NotificationView, formatSize, template, templateStatus, templateProgress, templateSup, templateSub) {
+ $, _, Backbone, App, StatusInfo, ProgressView, NotificationView, formatSize, template, templateStatus, templateProgress, templateSup, templateSub) {
'use strict';
// Renders the header with all information
return Backbone.Marionette.ItemView.extend({
@@ -47,7 +47,7 @@ define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus',
this.model = App.user;
- this.status = new ServerStatus();
+ this.status = new StatusInfo();
this.listenTo(this.status, 'change', this.update);
this.listenTo(App.progressList, 'add', function(model) {
@@ -194,7 +194,7 @@ define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus',
var data = JSON.parse(evt.data);
if (data === null) return;
- if (data['@class'] === 'ServerStatus') {
+ if (data['@class'] === 'StatusInfo') {
this.status.set(data);
// There tasks at the server, but not in queue: so fetch them
diff --git a/tests/config/pyload.conf.org b/tests/config/pyload.conf.org
index 0a422f258..9d3611e4e 100644
--- a/tests/config/pyload.conf.org
+++ b/tests/config/pyload.conf.org
@@ -1,75 +1 @@
-version: 2
-
-[remote]
-nolocalauth = False
-activated = True
-port = 7558
-listenaddr = 127.0.0.1
-
-[log]
-log_size = 100
-log_folder = Logs
-file_log = False
-log_count = 5
-log_rotate = True
-
-[permission]
-group = users
-change_dl = False
-change_file = False
-user = user
-file = 0644
-change_group = False
-folder = 0755
-change_user = False
-
-[general]
-language = en
-download_folder = Downloads
-checksum = False
-folder_per_package = True
-debug_mode = True
-min_free_space = 200
-renice = 0
-
-[ssl]
-cert = ssl.crt
-activated = False
-key = ssl.key
-
-[webinterface]
-template = default
-activated = True
-prefix =
-server = builtin
-host = 127.0.0.1
-https = False
-port = 8921
-
-[proxy]
-username =
-proxy = False
-address = localhost
-password =
-type = http
-port = 7070
-
-[reconnect]
-endTime = 0:00
-activated = False
-method = ./reconnect.sh
-startTime = 0:00
-
-[download]
-max_downloads = 3
-limit_speed = False
-interface =
-skip_existing = False
-max_speed = -1
-ipv6 = False
-chunks = 3
-
-[downloadTime]
-start = 0:00
-end = 0:00
-
+version:2 \ No newline at end of file
diff --git a/tests/config/pyload.db.org b/tests/config/pyload.db.org
index 26af474c1..6fb8de6c1 100644
--- a/tests/config/pyload.db.org
+++ b/tests/config/pyload.db.org
Binary files differ
diff --git a/tests/helper/Stubs.py b/tests/helper/Stubs.py
index 3541e4ffb..4bb8ade9e 100644
--- a/tests/helper/Stubs.py
+++ b/tests/helper/Stubs.py
@@ -1,20 +1,22 @@
# -*- coding: utf-8 -*-
+import sys
from os.path import join
from time import strftime
from traceback import format_exc
import __builtin__
+from pyload.InitHomeDir import init_dir
+
+init_dir(join("tests", "config"), True)
+
from pyload.Api import Role
from pyload.Core import Core
-from pyload.InitHomeDir import init_dir
from pyload.datatypes.User import User
from pyload.threads.BaseThread import BaseThread
from pyload.config.ConfigParser import ConfigParser
-init_dir(join("tests", "config"))
-
from logging import log, DEBUG, INFO, WARN, ERROR
# Do nothing
@@ -94,3 +96,9 @@ adminUser = User(None, uid=0, role=Role.Admin)
normalUser = User(None, uid=1, role=Role.User)
otherUser = User(None, uid=2, role=Role.User)
+# fixes the module paths because we changed the directory
+for name, m in sys.modules.iteritems():
+ if not name.startswith("tests") or not m or not hasattr(m, "__path__"):
+ continue
+
+ m.__path__[0] = join("..", "..", m.__path__[0]) \ No newline at end of file
diff --git a/tests/manager/test_accountManager.py b/tests/manager/test_accountManager.py
index b7166b2bd..dccba9d7c 100644
--- a/tests/manager/test_accountManager.py
+++ b/tests/manager/test_accountManager.py
@@ -4,6 +4,8 @@ from unittest import TestCase
from tests.helper.Stubs import Core, adminUser, normalUser
+from pyload.AccountManager import AccountManager
+
class TestAccountManager(TestCase):
@classmethod
@@ -17,19 +19,20 @@ class TestAccountManager(TestCase):
def setUp(self):
self.db.purgeAccounts()
- self.manager = self.core.accountManager
+ self.manager = AccountManager(self.core)
def test_access(self):
- account = self.manager.updateAccount("Http", "User", "somepw", adminUser)
+ account = self.manager.createAccount("Http", "User", "somepw", adminUser.uid)
- assert account is self.manager.updateAccount("Http", "User", "newpw", adminUser)
+ assert account is self.manager.updateAccount(account.aid, "Http", "User", "newpw", adminUser)
self.assertEqual(account.password, "newpw")
- assert self.manager.getAccount("Http", "User") is account
- assert self.manager.getAccount("Http", "User", normalUser) is None
+ assert self.manager.getAccount(account.aid, "Http", adminUser) is account
+ assert self.manager.getAccount(account.aid, "Http", normalUser) is None
+
def test_config(self):
- account = self.manager.updateAccount("Http", "User", "somepw", adminUser)
+ account = self.manager.createAccount("Http", "User", "somepw", adminUser.uid)
info = account.toInfoData()
self.assertEqual(info.config[0].name, "domain")
@@ -48,7 +51,7 @@ class TestAccountManager(TestCase):
def test_shared(self):
- account = self.manager.updateAccount("Http", "User", "somepw", adminUser)
+ account = self.manager.createAccount("Http", "User", "somepw", adminUser.uid)
assert self.manager.selectAccount("Http", adminUser) is account
assert account.loginname == "User"
@@ -61,5 +64,10 @@ class TestAccountManager(TestCase):
assert self.manager.selectAccount("Http", normalUser) is account
assert self.manager.selectAccount("sdf", normalUser) is None
+ self.manager.removeAccount(account.aid, "Http", adminUser.uid)
+
+ assert self.manager.selectAccount("Http", adminUser) is None
+
+
diff --git a/tests/manager/test_downloadManager.py b/tests/manager/test_downloadManager.py
index 9906ca1a0..305451d7f 100644
--- a/tests/manager/test_downloadManager.py
+++ b/tests/manager/test_downloadManager.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-from tests.helper.Stubs import Core, normalUser, adminUser
+from tests.helper.Stubs import Core, normalUser, adminUser, otherUser
from tests.helper.BenchmarkTest import BenchmarkTest
from pyload.database import DatabaseBackend
@@ -9,13 +9,21 @@ DatabaseBackend.async = DatabaseBackend.queue
class TestDownloadManager(BenchmarkTest):
- bench = ["add_links", "db"]
+ bench = ["add_links", "simple", "empty"]
+
+ USER = 2
+ PACKAGES = 10
+ LINKS = 50
+ PLUGINS = 10
@classmethod
def setUpClass(cls):
cls.c = Core()
cls.db = cls.c.db
cls.db.purgeAll()
+ cls.db.addDebugUser(normalUser.uid)
+ cls.db.addDebugUser(adminUser.uid)
+ cls.db.addDebugUser(otherUser.uid)
cls.files = cls.c.files
cls.m = cls.c.downloadManager
@@ -30,12 +38,18 @@ class TestDownloadManager(BenchmarkTest):
def test_add_links(self):
# just generate some links and files
- for i in range(10):
- pid = self.files.addPackage("name %d", "folder", -1, "", "", "", False, normalUser.uid)
- self.files.addLinks([("plugin%d" % i, "url%d" %i) for i in range(50)], pid, normalUser.uid)
+ for user in (adminUser, normalUser):
+ for i in range(self.PACKAGES):
+ pid = self.files.addPackage("name %d", "folder", -1, "", "", "", False, user.uid)
+ self.files.addLinks([( "url%d" %i, "plugin%d" % (i % self.PLUGINS)) for i in range(self.LINKS)], pid, user.uid)
+
+ def test_simple(self):
+ jobs = self.db.getJobs([])
+ assert len(jobs) == 2
+
+ def test_empty(self):
+ assert not self.db.getJobs(["plugin%d" % i for i in range(self.PLUGINS)])
- def test_db(self):
- pass
if __name__ == "__main__":
diff --git a/tests/other/test_statsDB.py b/tests/other/test_statsDB.py
new file mode 100644
index 000000000..f311b181e
--- /dev/null
+++ b/tests/other/test_statsDB.py
@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+from tests.helper.Stubs import Core
+
+class TestStatDatabase():
+
+ @classmethod
+ def setUpClass(cls):
+ cls.core = Core()
+ cls.db = cls.core.db
+
+ def test_simple(self):
+ assert 1 == 0 \ No newline at end of file