diff options
Diffstat (limited to 'module')
-rw-r--r-- | module/FileManager.py | 35 | ||||
-rw-r--r-- | module/database/FileDatabase.py | 13 | ||||
-rw-r--r-- | module/datatypes/PyFile.py | 16 | ||||
-rw-r--r-- | module/lib/ReadWriteLock.py | 232 | ||||
-rw-r--r-- | module/network/CookieJar.py | 1 | ||||
-rw-r--r-- | module/network/HTTPRequest.py | 4 | ||||
-rw-r--r-- | module/network/RequestFactory.py | 1 | ||||
-rw-r--r-- | module/plugins/Crypter.py | 6 | ||||
-rw-r--r-- | module/plugins/internal/ReCaptcha.py (renamed from module/plugins/ReCaptcha.py) | 0 | ||||
-rw-r--r-- | module/utils/__init__.py | 14 | ||||
-rw-r--r-- | module/web/static/css/default/dashboard.less | 7 | ||||
-rw-r--r-- | module/web/static/css/default/style.less | 10 | ||||
-rw-r--r-- | module/web/static/js/helpers/fileHelper.js | 2 | ||||
-rw-r--r-- | module/web/static/js/models/File.js | 3 | ||||
-rw-r--r-- | module/web/static/js/models/Progress.js | 11 | ||||
-rw-r--r-- | module/web/static/js/views/headerView.js | 12 | ||||
-rw-r--r-- | module/web/templates/default/base.html | 11 |
17 files changed, 328 insertions, 50 deletions
diff --git a/module/FileManager.py b/module/FileManager.py index 0d8a35a64..52fdab703 100644 --- a/module/FileManager.py +++ b/module/FileManager.py @@ -17,9 +17,9 @@ ############################################################################### from time import time -from threading import RLock +from ReadWriteLock import ReadWriteLock -from module.utils import lock +from module.utils import lock, read_lock from Api import PackageStatus, DownloadStatus as DS, TreeCollection, PackageDoesNotExists from datatypes.PyFile import PyFile @@ -60,8 +60,8 @@ class FileManager: self.jobCache = {} - # locking the cache, db is already locked implicit - self.lock = RLock() + # locking the caches, db is already locked implicit + self.lock = ReadWriteLock() #self.lock._Verbose__verbose = True self.downloadstats = {} # cached dl stats @@ -73,7 +73,7 @@ class FileManager: """saves all data to backend""" self.db.commit() - @lock + @read_lock def syncSave(self): """saves all data to backend and waits until all data are written""" for pyfile in self.files.values(): @@ -129,7 +129,7 @@ class FileManager: return pack - @lock + @read_lock def getPackageInfo(self, pid): """returns dict with package information""" if pid == self.ROOT_PACKAGE: @@ -166,7 +166,7 @@ class FileManager: self.files[fid] = f return f - @lock + @read_lock def getFileInfo(self, fid): """returns dict with file information""" if fid in self.files: @@ -174,7 +174,7 @@ class FileManager: return self.db.getFileInfo(fid) - @lock + @read_lock def getTree(self, pid, full, state, search=None): """ return a TreeCollection and fill the info data of containing packages. optional filter only unfnished files @@ -255,6 +255,7 @@ class FileManager: def getJob(self, occ): """get suitable job""" + #TODO only accessed by one thread, should not need a lock #TODO needs to be approved for new database #TODO clean mess #TODO improve selection of valid jobs @@ -300,9 +301,9 @@ class FileManager: return self.downloadstats[user] def getQueueStats(self, user=None, force=False): - """number of files that have to be processed""" - if user not in self.queuestats or force: - self.queuestats[user] = self.db.queuestats() + """number of files that have to be processed, failed files will not be included""" + if user not in self.queuestats or force: + self.queuestats[user] = self.db.processstats() return self.queuestats[user] @@ -395,7 +396,7 @@ class FileManager: """checks if all files are finished and dispatch event""" # TODO: user context? - if not self.getQueueStats(None, True)[0]: + if not self.db.queuestats()[0]: self.core.addonManager.dispatchEvent("download:allFinished") self.core.log.debug("All downloads finished") return True @@ -429,7 +430,7 @@ class FileManager: def resetCount(self): self.queuecount = -1 - @lock + @read_lock @invalidate def restartPackage(self, pid): """restart package""" @@ -444,7 +445,7 @@ class FileManager: self.evm.dispatchEvent("package:updated", pid) - @lock + @read_lock @invalidate def restartFile(self, fid): """ restart file""" @@ -521,7 +522,7 @@ class FileManager: self.evm.dispatchEvent("file:reordered", pid) - @lock + @read_lock @invalidate def movePackage(self, pid, root): """ move pid - root """ @@ -542,9 +543,7 @@ class FileManager: return True - - - @lock + @read_lock @invalidate def moveFiles(self, fids, pid): """ move all fids to pid """ diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 632961c2a..67a15912a 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -59,6 +59,8 @@ class FileMethods(DatabaseMethods): r = self.c.fetchone() return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) + + # TODO: multi user? @queue def processcount(self, fid=-1, user=None): """ number of files which have to be processed """ @@ -66,7 +68,16 @@ class FileMethods(DatabaseMethods): self.c.execute("SELECT COUNT(*), SUM(size) FROM files WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (fid, )) return self.c.fetchone()[0] - # TODO: think about multiuser side effects on *count methods + @queue + def processstats(self, user=None): + if user is None: + self.c.execute("SELECT COUNT(*), SUM(size) FROM files WHERE dlstatus IN (2,3,8,9,10)") + else: + self.c.execute( + "SELECT COUNT(*), SUM(f.size) FROM files f, packages p WHERE f.package = p.pid AND dlstatus IN (2,3,8,9,10)", + user) + r = self.c.fetchone() + return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) @queue def addLink(self, url, name, plugin, package, owner): diff --git a/module/datatypes/PyFile.py b/module/datatypes/PyFile.py index 14baa68ab..bd335a05a 100644 --- a/module/datatypes/PyFile.py +++ b/module/datatypes/PyFile.py @@ -16,11 +16,11 @@ # @author: RaNaN ############################################################################### -from time import sleep, time -from threading import RLock +from time import sleep +from ReadWriteLock import ReadWriteLock from module.Api import ProgressInfo, DownloadProgress, FileInfo, DownloadInfo, DownloadStatus -from module.utils import format_size, format_time, lock +from module.utils import lock, read_lock statusMap = { "none": 0, @@ -83,10 +83,10 @@ class PyFile(object): self.status = status self.error = error self.ownerid = owner - self.packageid = package #should not be used, use package() instead + self.packageid = package # database information ends here - self.lock = RLock() + self.lock = ReadWriteLock() self.plugin = None @@ -137,7 +137,7 @@ class PyFile(object): self.pluginclass = self.m.core.pluginManager.getPlugin(self.pluginname) self.plugin = self.pluginclass(self) - @lock + @read_lock def hasPlugin(self): """Thread safe way to determine this file has initialized plugin attribute""" return hasattr(self, "plugin") and self.plugin @@ -191,8 +191,10 @@ class PyFile(object): def move(self, pid): pass + @read_lock def abortDownload(self): """abort pyfile if possible""" + # TODO: abort timeout, currently dead locks while self.id in self.m.core.threadManager.processingIds(): self.abort = True if self.plugin and self.plugin.req: @@ -200,7 +202,7 @@ class PyFile(object): sleep(0.1) self.abort = False - if self.hasPlugin() and self.plugin.req: + if self.plugin and self.plugin.req: self.plugin.req.abortDownloads() self.release() diff --git a/module/lib/ReadWriteLock.py b/module/lib/ReadWriteLock.py new file mode 100644 index 000000000..cc82f3d48 --- /dev/null +++ b/module/lib/ReadWriteLock.py @@ -0,0 +1,232 @@ +# -*- coding: iso-8859-15 -*- +"""locks.py - Read-Write lock thread lock implementation + +See the class documentation for more info. + +Copyright (C) 2007, Heiko Wundram. +Released under the BSD-license. + +http://code.activestate.com/recipes/502283-read-write-lock-class-rlock-like/ +""" + +# Imports +# ------- + +from threading import Condition, Lock, currentThread +from time import time + + +# Read write lock +# --------------- + +class ReadWriteLock(object): + """Read-Write lock class. A read-write lock differs from a standard + threading.RLock() by allowing multiple threads to simultaneously hold a + read lock, while allowing only a single thread to hold a write lock at the + same point of time. + + When a read lock is requested while a write lock is held, the reader + is blocked; when a write lock is requested while another write lock is + held or there are read locks, the writer is blocked. + + Writers are always preferred by this implementation: if there are blocked + threads waiting for a write lock, current readers may request more read + locks (which they eventually should free, as they starve the waiting + writers otherwise), but a new thread requesting a read lock will not + be granted one, and block. This might mean starvation for readers if + two writer threads interweave their calls to acquireWrite() without + leaving a window only for readers. + + In case a current reader requests a write lock, this can and will be + satisfied without giving up the read locks first, but, only one thread + may perform this kind of lock upgrade, as a deadlock would otherwise + occur. After the write lock has been granted, the thread will hold a + full write lock, and not be downgraded after the upgrading call to + acquireWrite() has been match by a corresponding release(). + """ + + def __init__(self): + """Initialize this read-write lock.""" + + # Condition variable, used to signal waiters of a change in object + # state. + self.__condition = Condition(Lock()) + + # Initialize with no writers. + self.__writer = None + self.__upgradewritercount = 0 + self.__pendingwriters = [] + + # Initialize with no readers. + self.__readers = {} + + def acquire(self, blocking=True, timeout=None, shared=False): + if shared: + self.acquireRead(timeout) + else: + self.acquireWrite(timeout) + + def acquireRead(self, timeout=None): + """Acquire a read lock for the current thread, waiting at most + timeout seconds or doing a non-blocking check in case timeout is <= 0. + + In case timeout is None, the call to acquireRead blocks until the + lock request can be serviced. + + In case the timeout expires before the lock could be serviced, a + RuntimeError is thrown.""" + + if timeout is not None: + endtime = time() + timeout + me = currentThread() + self.__condition.acquire() + try: + if self.__writer is me: + # If we are the writer, grant a new read lock, always. + self.__writercount += 1 + return + while True: + if self.__writer is None: + # Only test anything if there is no current writer. + if self.__upgradewritercount or self.__pendingwriters: + if me in self.__readers: + # Only grant a read lock if we already have one + # in case writers are waiting for their turn. + # This means that writers can't easily get starved + # (but see below, readers can). + self.__readers[me] += 1 + return + # No, we aren't a reader (yet), wait for our turn. + else: + # Grant a new read lock, always, in case there are + # no pending writers (and no writer). + self.__readers[me] = self.__readers.get(me, 0) + 1 + return + if timeout is not None: + remaining = endtime - time() + if remaining <= 0: + # Timeout has expired, signal caller of this. + raise RuntimeError("Acquiring read lock timed out") + self.__condition.wait(remaining) + else: + self.__condition.wait() + finally: + self.__condition.release() + + def acquireWrite(self, timeout=None): + """Acquire a write lock for the current thread, waiting at most + timeout seconds or doing a non-blocking check in case timeout is <= 0. + + In case the write lock cannot be serviced due to the deadlock + condition mentioned above, a ValueError is raised. + + In case timeout is None, the call to acquireWrite blocks until the + lock request can be serviced. + + In case the timeout expires before the lock could be serviced, a + RuntimeError is thrown.""" + + if timeout is not None: + endtime = time() + timeout + me, upgradewriter = currentThread(), False + self.__condition.acquire() + try: + if self.__writer is me: + # If we are the writer, grant a new write lock, always. + self.__writercount += 1 + return + elif me in self.__readers: + # If we are a reader, no need to add us to pendingwriters, + # we get the upgradewriter slot. + if self.__upgradewritercount: + # If we are a reader and want to upgrade, and someone + # else also wants to upgrade, there is no way we can do + # this except if one of us releases all his read locks. + # Signal this to user. + raise ValueError( + "Inevitable dead lock, denying write lock" + ) + upgradewriter = True + self.__upgradewritercount = self.__readers.pop(me) + else: + # We aren't a reader, so add us to the pending writers queue + # for synchronization with the readers. + self.__pendingwriters.append(me) + while True: + if not self.__readers and self.__writer is None: + # Only test anything if there are no readers and writers. + if self.__upgradewritercount: + if upgradewriter: + # There is a writer to upgrade, and it's us. Take + # the write lock. + self.__writer = me + self.__writercount = self.__upgradewritercount + 1 + self.__upgradewritercount = 0 + return + # There is a writer to upgrade, but it's not us. + # Always leave the upgrade writer the advance slot, + # because he presumes he'll get a write lock directly + # from a previously held read lock. + elif self.__pendingwriters[0] is me: + # If there are no readers and writers, it's always + # fine for us to take the writer slot, removing us + # from the pending writers queue. + # This might mean starvation for readers, though. + self.__writer = me + self.__writercount = 1 + self.__pendingwriters = self.__pendingwriters[1:] + return + if timeout is not None: + remaining = endtime - time() + if remaining <= 0: + # Timeout has expired, signal caller of this. + if upgradewriter: + # Put us back on the reader queue. No need to + # signal anyone of this change, because no other + # writer could've taken our spot before we got + # here (because of remaining readers), as the test + # for proper conditions is at the start of the + # loop, not at the end. + self.__readers[me] = self.__upgradewritercount + self.__upgradewritercount = 0 + else: + # We were a simple pending writer, just remove us + # from the FIFO list. + self.__pendingwriters.remove(me) + raise RuntimeError("Acquiring write lock timed out") + self.__condition.wait(remaining) + else: + self.__condition.wait() + finally: + self.__condition.release() + + def release(self): + """Release the currently held lock. + + In case the current thread holds no lock, a ValueError is thrown.""" + + me = currentThread() + self.__condition.acquire() + try: + if self.__writer is me: + # We are the writer, take one nesting depth away. + self.__writercount -= 1 + if not self.__writercount: + # No more write locks; take our writer position away and + # notify waiters of the new circumstances. + self.__writer = None + self.__condition.notifyAll() + elif me in self.__readers: + # We are a reader currently, take one nesting depth away. + self.__readers[me] -= 1 + if not self.__readers[me]: + # No more read locks, take our reader position away. + del self.__readers[me] + if not self.__readers: + # No more readers, notify waiters of the new + # circumstances. + self.__condition.notifyAll() + else: + raise ValueError("Trying to release unheld lock") + finally: + self.__condition.release() diff --git a/module/network/CookieJar.py b/module/network/CookieJar.py index a020d6f9e..ea2c43a9e 100644 --- a/module/network/CookieJar.py +++ b/module/network/CookieJar.py @@ -19,6 +19,7 @@ from time import time +# TODO: replace with simplecookie? class CookieJar(): def __init__(self, pluginname): self.cookies = {} diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 990148e0f..874da368b 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -193,13 +193,14 @@ class HTTPRequest(): self.setRequestContext(url, get, post, referer, cookies, multipart) + # TODO: use http/rfc message instead self.header = "" self.c.setopt(pycurl.HTTPHEADER, self.headers) if just_header: self.c.setopt(pycurl.FOLLOWLOCATION, 0) - self.c.setopt(pycurl.NOBODY, 1) + self.c.setopt(pycurl.NOBODY, 1) #TODO: nobody= no post? # overwrite HEAD request, we want a common request type if post: @@ -233,6 +234,7 @@ class HTTPRequest(): def verifyHeader(self): """ raise an exceptions on bad headers """ code = int(self.c.getinfo(pycurl.RESPONSE_CODE)) + # TODO: raise anyway to be consistent, also rename exception if code in bad_headers: #404 will NOT raise an exception raise BadHeader(code, self.getResponse()) diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 932184678..1581be9fc 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -36,6 +36,7 @@ class RequestFactory(): return self.core.config["download"]["interface"] def getRequest(self, pluginName, cj=None): + # TODO: mostly obsolete, browser could be removed completely req = Browser(self.bucket, self.getOptions()) if cj: diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index 920009f44..61370541f 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -10,9 +10,11 @@ from Base import Base, Retry class Package: """ Container that indicates that a new package should be created """ - def __init__(self, name, urls=None): + def __init__(self, name, urls=None, folder=None): self.name = name self.urls = urls if urls else [] + self.folder = folder + # nested packages self.packs = [] @@ -36,7 +38,7 @@ class Package: return u"<CrypterPackage name=%s, links=%s, packs=%s" % (self.name, self.urls, self.packs) def __hash__(self): - return hash(self.name) ^ hash(frozenset(self.urls)) + return hash(self.name) ^ hash(frozenset(self.urls)) ^ hash(self.name) class PyFileMockup: """ Legacy class needed by old crypter plugins """ diff --git a/module/plugins/ReCaptcha.py b/module/plugins/internal/ReCaptcha.py index 6f7ebe22c..6f7ebe22c 100644 --- a/module/plugins/ReCaptcha.py +++ b/module/plugins/internal/ReCaptcha.py diff --git a/module/utils/__init__.py b/module/utils/__init__.py index 4692c59cb..8f7ed6231 100644 --- a/module/utils/__init__.py +++ b/module/utils/__init__.py @@ -125,11 +125,21 @@ def parseFileSize(string, unit=None): #returns bytes def lock(func): - def new(*args): + def new(*args, **kwargs): #print "Handler: %s args: %s" % (func,args[1:]) args[0].lock.acquire() try: - return func(*args) + return func(*args, **kwargs) + finally: + args[0].lock.release() + + return new + +def read_lock(func): + def new(*args, **kwargs): + args[0].lock.acquire(shared=True) + try: + return func(*args, **kwargs) finally: args[0].lock.release() diff --git a/module/web/static/css/default/dashboard.less b/module/web/static/css/default/dashboard.less index 2a4adf0e7..865812e41 100644 --- a/module/web/static/css/default/dashboard.less +++ b/module/web/static/css/default/dashboard.less @@ -186,6 +186,11 @@ .gradient(top, @yellow, @yellowDark);
color: @dark;
border-color: @greenDark;
+
+ .file-row.downloading .bar {
+ .gradient(top, @green, @greenLight);
+ }
+
}
img { // plugin logo
@@ -255,7 +260,7 @@ }
.bar {
- .gradient(top, @green, @greenLight);
+ .gradient(top, @yellow, @yellowDark);
color: @light;
}
diff --git a/module/web/static/css/default/style.less b/module/web/static/css/default/style.less index 260f9fa52..baa8cc413 100644 --- a/module/web/static/css/default/style.less +++ b/module/web/static/css/default/style.less @@ -182,10 +182,7 @@ header .logo { float: right; // font-family: SansationRegular, sans-serif;
margin: 10px 8px 0;
line-height: 18px;
- font-size: small; // i {
-// margin-top: 0;
-// vertical-align: text-bottom;
-// }
+ font-size: small;
.btn {
margin-top: 6px;
@@ -234,12 +231,15 @@ header .logo { #progress-area {
position: relative;
- text-align: center;
float: right;
width: 26%;
margin-right: 15px;
line-height: 16px;
+ #progress-info {
+ padding-left: 5px;
+ }
+
.popover { // display: block;
max-width: none;
width: 120%;
diff --git a/module/web/static/js/helpers/fileHelper.js b/module/web/static/js/helpers/fileHelper.js index d7cf03f53..dde831bdd 100644 --- a/module/web/static/js/helpers/fileHelper.js +++ b/module/web/static/js/helpers/fileHelper.js @@ -37,7 +37,7 @@ define('helpers/fileHelper', ['handlebars', 'utils/apitypes'], } else if (file.finished) s = "<i class='iconf-ok'></i> " + msg; else if(file.downloading) - s= "<div class='progress'><div class='bar' style='width: 50%'></div></div>"; + s= "<div class='progress'><div class='bar' style='width: " + file.progress + "%'></div></div>"; else s = msg; diff --git a/module/web/static/js/models/File.js b/module/web/static/js/models/File.js index 2ac6c05f5..22ff231cc 100644 --- a/module/web/static/js/models/File.js +++ b/module/web/static/js/models/File.js @@ -22,7 +22,8 @@ define(['jquery', 'backbone', 'underscore', 'utils/apitypes'], function($, Backb // UI attributes selected: false, - visible: true + visible: true, + progress: 0 }, diff --git a/module/web/static/js/models/Progress.js b/module/web/static/js/models/Progress.js index c6a2fc4d1..d2d54bdb4 100644 --- a/module/web/static/js/models/Progress.js +++ b/module/web/static/js/models/Progress.js @@ -16,6 +16,12 @@ define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { download: null }, + getPercent: function() { + if (this.get('total') > 0) + return Math.round(this.get('done') * 100 / this.get('total')); + return 0; + }, + // Model Constructor initialize: function() { @@ -28,10 +34,7 @@ define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { toJSON: function(options) { var obj = Backbone.Model.prototype.toJSON.call(this, options); - if (obj.total > 0) - obj.percent = Math.round(obj.done * 100 / obj.total); - else - obj.percent = 0; + obj.percent = this.getPercent(); return obj; }, diff --git a/module/web/static/js/views/headerView.js b/module/web/static/js/views/headerView.js index dddae4705..d9c56b332 100644 --- a/module/web/static/js/views/headerView.js +++ b/module/web/static/js/views/headerView.js @@ -6,7 +6,7 @@ define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus', 'colle el: 'header', events: { - 'click i.iconf-list': 'toggle_taskList', + 'click .iconf-list': 'toggle_taskList', 'click .popover .close': 'hide_taskList', 'click .btn-grabber': 'open_grabber' }, @@ -169,6 +169,7 @@ define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus', 'colle }, onProgressUpdate: function(progress) { + // generate a unique id _.each(progress, function(prog) { if (prog.download) prog.pid = prog.download.fid; @@ -177,6 +178,15 @@ define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus', 'colle }); this.progressList.update(progress); + // update currently open files with progress + this.progressList.each(function(prog) { + if(prog.isDownload() && App.dashboard.files){ + var file = App.dashboard.files.get(prog.get('download').fid); + if (file) + file.set('progress', prog.getPercent()); + } + }); + // TODO: only render when changed this.render(); }, diff --git a/module/web/templates/default/base.html b/module/web/templates/default/base.html index f995f79b7..ebaac59e9 100644 --- a/module/web/templates/default/base.html +++ b/module/web/templates/default/base.html @@ -44,17 +44,16 @@ <% else %>
No running tasks
<%/if%>
- <i class="icon-white iconf-list pull-right"></i>
+ <i class="iconf-list pull-right"></i>
<div class="progress" id="globalprogress">
<div class="bar" style="width: 48%">48%</div>
</div>
</script>
<script type="text/template" id="template-header-status">
- <span class="pull-right eta"><% formatTime eta %></span><br>
+ <span class="pull-right"><% linksqueue %></span><br>
<span class="pull-right remeaning"><% formatSize sizequeue %></span><br>
- <span class="pull-right"><span
- style="font-weight:bold;color: #fff !important;"><% linksqueue %></span> of <% linkstotal %></span>
+ <span class="pull-right eta"><% formatTime eta %></span>
</script>
<script type="text/template" id="template-header-progress">
@@ -114,9 +113,9 @@ <div class="header_block right-border status-block">
</div>
<div class="header_block left-border">
- <i class="icon-time icon-white"></i> approx. ETA :<br>
- <i class=" icon-hdd icon-white"></i> Remaining:<br>
<i class="icon-download-alt icon-white"></i> Downloads:<br>
+ <i class=" icon-hdd icon-white"></i> Remaining:<br>
+ <i class="icon-time icon-white"></i> approx. ETA :<br>
</div>
<div id="progress-area" style="margin-top: 16px">
|