diff options
author | RaNaN <Mast3rRaNaN@hotmail.de> | 2010-08-11 21:04:48 +0200 |
---|---|---|
committer | RaNaN <Mast3rRaNaN@hotmail.de> | 2010-08-11 21:04:48 +0200 |
commit | ecadad99f03e33d4672e9ac2d5d1dca0d541b983 (patch) | |
tree | 65ac2f5a94cc50d8ef8044b6cb1616937105d2b9 | |
parent | fixes (diff) | |
download | pyload-ecadad99f03e33d4672e9ac2d5d1dca0d541b983.tar.xz |
jeixs irc interface, cleaned some code
-rw-r--r-- | module/ConfigParser.py | 1 | ||||
-rw-r--r-- | module/FileDatabase.py | 130 | ||||
-rw-r--r-- | module/HookManager.py | 11 | ||||
-rw-r--r-- | module/PluginThread.py | 374 | ||||
-rw-r--r-- | module/SpeedManager.py | 1 | ||||
-rw-r--r-- | module/ThreadManager.py | 26 | ||||
-rw-r--r-- | module/plugins/Plugin.py | 4 | ||||
-rw-r--r-- | module/plugins/accounts/NetloadIn.py | 2 | ||||
-rw-r--r-- | module/plugins/hooks/IRCInterface.py | 346 | ||||
-rw-r--r-- | module/plugins/hoster/MegauploadCom.py | 3 | ||||
-rw-r--r-- | module/web/templates/default/settings.html | 10 | ||||
-rwxr-xr-x | pyLoadCli.py | 5 | ||||
-rwxr-xr-x | pyLoadCore.py | 53 |
13 files changed, 659 insertions, 307 deletions
diff --git a/module/ConfigParser.py b/module/ConfigParser.py index 91396a83b..a7526fd17 100644 --- a/module/ConfigParser.py +++ b/module/ConfigParser.py @@ -5,7 +5,6 @@ from pprint import pprint from os.path import exists from os.path import join from shutil import copy -from os import remove CONF_VERSION = 1 diff --git a/module/FileDatabase.py b/module/FileDatabase.py index c89656407..37248b31a 100644 --- a/module/FileDatabase.py +++ b/module/FileDatabase.py @@ -17,17 +17,21 @@ @author: mkaay """ from Queue import Queue +from os import remove +from os.path import exists +from shutil import move import sqlite3 +from threading import Lock +from threading import RLock from threading import Thread -from threading import Lock, RLock from time import sleep from time import time import traceback -from os.path import exists -from os import remove -from shutil import move -from module.PullEvents import UpdateEvent, RemoveEvent, InsertEvent, ReloadAllEvent +from module.PullEvents import InsertEvent +from module.PullEvents import ReloadAllEvent +from module.PullEvents import RemoveEvent +from module.PullEvents import UpdateEvent DB_VERSION = 2 @@ -54,7 +58,7 @@ def formatSize(size): """formats size of bytes""" size = int(size) steps = 0 - sizes = ["B", "KB", "MB", "GB" , "TB"] + sizes = ["B", "KB", "MB", "GB", "TB"] while size > 1000: size /= 1024.0 @@ -75,7 +79,7 @@ class FileHandler: self.core = core # translations - self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("checking"), _("waiting"), _("reconnected"), _("starting"),_("failed"), _("aborted"), _("decrypting"), _("custom"),_("downloading"), _("processing"), _("unknown")] + self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("checking"), _("waiting"), _("reconnected"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), _("downloading"), _("processing"), _("unknown")] self.cache = {} #holds instances for files self.packageCache = {} # same for packages @@ -124,8 +128,8 @@ class FileHandler: data = self.db.getAllLinks(queue) packs = self.db.getAllPackages(queue) - data.update( [ (str(x.id), x.toDbDict()[x.id]) for x in self.cache.itervalues() ] ) - packs.update( [ (str(x.id), x.toDict()[x.id]) for x in self.packageCache.itervalues() if x.queue == queue] ) + data.update([(str(x.id), x.toDbDict()[x.id]) for x in self.cache.itervalues()]) + packs.update([(str(x.id), x.toDict()[x.id]) for x in self.packageCache.itervalues() if x.queue == queue]) for key, value in data.iteritems(): if packs.has_key(str(value["package"])): @@ -444,7 +448,7 @@ class FileHandler: for pyfile in data.itervalues(): if pyfile.status not in (0, 12, 13): - urls.append( ( pyfile["url"], pyfile["plugin"]) ) + urls.append((pyfile["url"], pyfile["plugin"])) self.core.threadManager.createInfoThread(urls, pid) @@ -517,19 +521,19 @@ class FileDatabaseBackend(Thread): def _checkVersion(self): """ check db version and delete it if needed""" if not exists("files.version"): - f = open("files.version" , "wb") + f = open("files.version", "wb") f.write(str(DB_VERSION)) f.close() return - f = open("files.version" , "rb") + f = open("files.version", "rb") v = int(f.read().strip()) f.close() if v < DB_VERSION: self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) remove("files.version") move("files.db", "files.backup.db") - f = open("files.version" , "wb") + f = open("files.version", "wb") f.write(str(DB_VERSION)) f.close() @@ -545,23 +549,23 @@ class FileDatabaseBackend(Thread): @queue def filecount(self, queue): """returns number of files in queue""" - self.c.execute("SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.id", (queue,)) + self.c.execute("SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.id", (queue, )) r = self.c.fetchall() return len(r) def _nextPackageOrder(self, queue=0): - self.c.execute('SELECT packageorder FROM packages WHERE queue=?', (queue, )) + self.c.execute('SELECT packageorder FROM packages WHERE queue=?', (queue,)) o = -1 for r in self.c: if r[0] > o: o = r[0] - return o+1 + return o + 1 def _nextFileOrder(self, package): - self.c.execute('SELECT linkorder FROM links WHERE package=?', (package, )) + self.c.execute('SELECT linkorder FROM links WHERE package=?', (package,)) o = -1 for r in self.c: if r[0] > o: o = r[0] - return o+1 + return o + 1 @queue def addLink(self, url, name, plugin, package): @@ -573,8 +577,8 @@ class FileDatabaseBackend(Thread): def addLinks(self, links, package): """ links is a list of tupels (url,plugin)""" order = self._nextFileOrder(package) - orders = [order+x for x in range(len(links))] - links = [(x[0],x[0],x[1],package,o) for x, o in zip(links, orders)] + orders = [order + x for x in range(len(links))] + links = [(x[0], x[0], x[1], package, o) for x, o in zip(links, orders)] self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) @queue @@ -586,15 +590,15 @@ class FileDatabaseBackend(Thread): @queue def deletePackage(self, p): - self.c.execute('DELETE FROM links WHERE package=?', (str(p.id), )) - self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id), )) - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', ( p.order, p.queue) ) + self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) + self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue)) @queue def deleteLink(self, f): - self.c.execute('DELETE FROM links WHERE id=?', (str(f.id), )) - self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', ( f.order, str(f.packageid)) ) + self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) + self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid))) @queue @@ -611,7 +615,7 @@ class FileDatabaseBackend(Thread): } """ - self.c.execute('SELECT l.id,l.url,l.name,l.size,l.status,l.error,l.plugin,l.package,l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', (q, )) + self.c.execute('SELECT l.id,l.url,l.name,l.size,l.status,l.error,l.plugin,l.package,l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', (q,)) data = {} for r in self.c: data[str(r[0])] = { @@ -663,28 +667,28 @@ class FileDatabaseBackend(Thread): def getLinkData(self, id): """get link information as dict""" - self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE id=?', (str(id),)) + self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE id=?', (str(id), )) data = {} r = self.c.fetchone() data[str(r[0])] = { - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': r[6], - 'package': r[7], - 'order': r[8] - } + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': r[6], + 'package': r[7], + 'order': r[8] + } return data @queue def getPackageData(self, id): """get package data""" - self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id),)) + self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id), )) data = {} for r in self.c: @@ -722,31 +726,31 @@ class FileDatabaseBackend(Thread): if position == -1: position = self._nextPackageOrder(p.queue) if not noMove: - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND packageorder > 0', ( p.order, p.queue) ) - self.c.execute('UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND queue=? AND packageorder > 0', ( position, p.queue) ) - self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', ( position, str(p.id) ) ) + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND packageorder > 0', (p.order, p.queue)) + self.c.execute('UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND queue=? AND packageorder > 0', (position, p.queue)) + self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) @queue def reorderLink(self, f, position): """ reorder link with f as dict for pyfile """ id = f.keys[0] - self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', ( f[str(id)]["order"], str(f[str(id)]["package"]) ) ) - self.c.execute('UPDATE links SET linkorder=linkorder+1 WHERE linkorder >= ? AND package=?', ( position, str(f[str(id)]["package"]) ) ) - self.c.execute('UPDATE links SET linkorder=? WHERE id=?', ( position, str(id) ) ) + self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f[str(id)]["order"], str(f[str(id)]["package"]))) + self.c.execute('UPDATE links SET linkorder=linkorder+1 WHERE linkorder >= ? AND package=?', (position, str(f[str(id)]["package"]))) + self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, str(id))) @queue def clearPackageOrder(self, p): - self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', ( -1, str(p.id) ) ) - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', ( p.order, p.queue, str(p.id)) ) + self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) @async def restartFile(self, id): - self.c.execute('UPDATE links SET status=3,error="" WHERE id=?', ( str(id), ) ) + self.c.execute('UPDATE links SET status=3,error="" WHERE id=?', (str(id),)) @async def restartPackage(self, id): - self.c.execute('UPDATE links SET status=3 WHERE package=?', ( str(id), ) ) + self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) @async def commit(self): @@ -759,19 +763,19 @@ class FileDatabaseBackend(Thread): @queue def getPackage(self, id): """return package instance from id""" - self.c.execute("SELECT name,folder,site,password,queue,packageorder,priority FROM packages WHERE id=?", (str(id),)) + self.c.execute("SELECT name,folder,site,password,queue,packageorder,priority FROM packages WHERE id=?", (str(id), )) r = self.c.fetchone() if not r: return None - return PyPackage(self.manager, id, *r) + return PyPackage(self.manager, id, * r) #---------------------------------------------------------------------- @queue def getFile(self, id): """return link instance from id""" - self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id),)) + self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id), )) r = self.c.fetchone() if not r: return None - return PyFile(self.manager, id, *r) + return PyFile(self.manager, id, * r) @queue @@ -790,13 +794,13 @@ class FileDatabaseBackend(Thread): self.c.execute(cmd) # very bad! - return [x[0] for x in self.c ] + return [x[0] for x in self.c] @queue def getUnfinished(self, pid): """return list of ids with pyfiles in package not finished or processed""" - self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 13)", (str(pid), ) ) + self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 13)", (str(pid),)) return [r[0] for r in self.c] @@ -879,7 +883,7 @@ class PyFile(): self.id: { 'url': self.url, 'name': self.name, - 'plugin' : self.pluginname, + 'plugin': self.pluginname, 'size': self.getSize(), 'format_size': self.formatSize(), 'status': self.status, @@ -914,7 +918,7 @@ class PyFile(): """ formats and return wait time in humanreadable format """ seconds = self.waitUntil - time() - if seconds < 0 : return "00:00:00" + if seconds < 0: return "00:00:00" hours, seconds = divmod(seconds, 3600) minutes, seconds = divmod(seconds, 60) @@ -928,7 +932,7 @@ class PyFile(): """ formats eta to readable format """ seconds = self.getETA() - if seconds < 0 : return "00:00:00" + if seconds < 0: return "00:00:00" hours, seconds = divmod(seconds, 3600) minutes, seconds = divmod(seconds, 60) @@ -1043,7 +1047,7 @@ class PyPackage(): if __name__ == "__main__": pypath = "." - _ = lambda x : x + _ = lambda x: x db = FileHandler(None) @@ -1054,19 +1058,19 @@ if __name__ == "__main__": #print db.addPackage("package", "folder" , 1) - pack = db.db.addPackage("package", "folder", 1) + pack = db.db.addPackage("package", "folder", 1) updates = [] for x in range(0, 200): x = str(x) - db.db.addLink("http://somehost.com/hoster/file/download?file_id="+x,x,"BasePlugin", pack) - updates.append( ("new name"+x,0,3, "http://somehost.com/hoster/file/download?file_id="+x) ) + db.db.addLink("http://somehost.com/hoster/file/download?file_id=" + x, x, "BasePlugin", pack) + updates.append(("new name" + x, 0, 3, "http://somehost.com/hoster/file/download?file_id=" + x)) for x in range(0, 100): - updates.append( ("unimportant%s"%x, 0, 3 , "a really long non existent url%s" %x ) ) + updates.append(("unimportant%s" % x, 0, 3, "a really long non existent url%s" % x)) db.db.commit() diff --git a/module/HookManager.py b/module/HookManager.py index 7cd903403..41e4c5ef0 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -41,6 +41,14 @@ class HookManager(): args[0].lock.release() return res return new + + def try_catch(func): + def new(*args): + try: + return func(*args) + except Exception, e: + args[0].log.error(_("Error executing hooks: %s") % str(e)) + return new def createIndex(self): @@ -50,6 +58,7 @@ class HookManager(): #hookClass = getattr(plugin, plugin.__name__) if self.core.config.getPlugin(pluginClass.__name__, "load"): + #@TODO handle in pluginmanager plugin = pluginClass(self.core) plugins.append(plugin) self.log.info(_("%s loaded, activated %s") % (pluginClass.__name__, plugin.isActivated() )) @@ -67,6 +76,8 @@ class HookManager(): plugin.periodical() plugin.lastCall = time() + + @try_catch def coreReady(self): for plugin in self.plugins: if plugin.isActivated(): diff --git a/module/PluginThread.py b/module/PluginThread.py index 2a4312ecd..2bd078e02 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -18,287 +18,289 @@ @author: RaNaN """ -from threading import Thread from Queue import Queue - +from threading import Thread from time import sleep from traceback import print_exc +from module.plugins.Plugin import Abort +from module.plugins.Plugin import Fail +from module.plugins.Plugin import Reconnect +from module.plugins.Plugin import Retry from pycurl import error -from module.plugins.Plugin import Abort, Reconnect, Retry, Fail ######################################################################## class PluginThread(Thread): - """abstract base class for thread types""" + """abstract base class for thread types""" - #---------------------------------------------------------------------- - def __init__(self, manager): - """Constructor""" - Thread.__init__(self) - self.setDaemon(True) - self.m = manager #thread manager + #---------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + Thread.__init__(self) + self.setDaemon(True) + self.m = manager #thread manager ######################################################################## class DownloadThread(PluginThread): - """thread for downloading files from 'real' hoster plugins""" + """thread for downloading files from 'real' hoster plugins""" - #---------------------------------------------------------------------- - def __init__(self, manager): - """Constructor""" - PluginThread.__init__(self, manager) + #---------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + PluginThread.__init__(self, manager) - self.queue = Queue() # job queue - self.active = False + self.queue = Queue() # job queue + self.active = False - self.start() + self.start() - #---------------------------------------------------------------------- - def run(self): - """run method""" + #---------------------------------------------------------------------- + def run(self): + """run method""" - while True: - self.active = self.queue.get() - pyfile = self.active + while True: + self.active = self.queue.get() + pyfile = self.active - if self.active == "quit": - return True + if self.active == "quit": + return True - self.m.log.info(_("Download starts: %s" % pyfile.name)) + self.m.log.info(_("Download starts: %s" % pyfile.name)) - try: - pyfile.plugin.preprocessing(self) + try: + pyfile.plugin.preprocessing(self) - except NotImplementedError: + except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) - continue + self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + continue - except Abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") + except Abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") - self.active = False - pyfile.release() - continue + self.active = False + pyfile.release() + continue - except Reconnect: - self.queue.put(pyfile) - #@TODO - #pyfile.req.clearCookies() + except Reconnect: + self.queue.put(pyfile) + #@TODO + #pyfile.req.clearCookies() - while self.m.reconnecting.isSet(): - sleep(0.5) + while self.m.reconnecting.isSet(): + sleep(0.5) - continue + continue - except Retry: + except Retry: - self.m.log.info(_("Download restarted: %s") % pyfile.name) - self.queue.put(pyfile) - continue + self.m.log.info(_("Download restarted: %s") % pyfile.name) + self.queue.put(pyfile) + continue - except Fail,e : + except Fail, e: - msg = e.args[0] + msg = e.args[0] - if msg == "offline": - pyfile.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % pyfile.name) - else: - pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %s | %s") % (pyfile.name, msg)) - pyfile.error = msg + if msg == "offline": + pyfile.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %s | %s") % (pyfile.name, msg)) + pyfile.error = msg - self.active = False - pyfile.release() - continue + self.active = False + pyfile.release() + continue - except error, e: - code, msg = e - if self.m.core.debug: - print "pycurl error", code, msg - print_exc() + except error, e: + code, msg = e + if self.m.core.debug: + print "pycurl error", code, msg + print_exc() - if code == 7: - self.m.log.warning(_("Couldn't connect to host waiting 1 minute and retry.")) - self.queue.put(pyfile) - continue + if code == 7: + self.m.log.warning(_("Couldn't connect to host waiting 1 minute and retry.")) + self.queue.put(pyfile) + continue - self.active = False - pyfile.release() - continue + self.active = False + pyfile.release() + continue - except Exception, e: - pyfile.setStatus("failed") - self.m.log.error(_("Download failed: %s | %s") % (pyfile.name, str(e))) - pyfile.error = str(e) + except Exception, e: + pyfile.setStatus("failed") + self.m.log.error(_("Download failed: %s | %s") % (pyfile.name, str(e))) + pyfile.error = str(e) - if self.m.core.debug: - print_exc() + if self.m.core.debug: + print_exc() - self.active = False - pyfile.release() - continue + self.active = False + pyfile.release() + continue - finally: - self.m.core.files.save() + finally: + self.m.core.files.save() - self.m.log.info(_("Download finished: %s") % pyfile.name) + self.m.log.info(_("Download finished: %s") % pyfile.name) - self.m.core.hookManager.downloadFinished(pyfile) + self.m.core.hookManager.downloadFinished(pyfile) - self.m.core.files.checkPackageFinished(pyfile) + self.m.core.files.checkPackageFinished(pyfile) - self.active = False - pyfile.finishIfDone() - self.m.core.files.save() + self.active = False + pyfile.finishIfDone() + self.m.core.files.save() - #---------------------------------------------------------------------- - def put(self, job): - """assing job to thread""" - self.queue.put(job) + #---------------------------------------------------------------------- + def put(self, job): + """assing job to thread""" + self.queue.put(job) - #---------------------------------------------------------------------- - def stop(self): - """stops the thread""" - self.put("quit") + #---------------------------------------------------------------------- + def stop(self): + """stops the thread""" + self.put("quit") ######################################################################## class DecrypterThread(PluginThread): - """thread for decrypting""" + """thread for decrypting""" - #---------------------------------------------------------------------- - def __init__(self, manager, pyfile): - """constructor""" - PluginThread.__init__(self, manager) + #---------------------------------------------------------------------- + def __init__(self, manager, pyfile): + """constructor""" + PluginThread.__init__(self, manager) - self.active = pyfile - manager.localThreads.append(self) + self.active = pyfile + manager.localThreads.append(self) - pyfile.setStatus("decrypting") + pyfile.setStatus("decrypting") - self.start() + self.start() - #---------------------------------------------------------------------- - def run(self): - """run method""" + #---------------------------------------------------------------------- + def run(self): + """run method""" - pyfile = self.active + pyfile = self.active - try: - self.m.log.info(_("Decrypting starts: %s") % self.active.name) - self.active.plugin.preprocessing(self) + try: + self.m.log.info(_("Decrypting starts: %s") % self.active.name) + self.active.plugin.preprocessing(self) - except NotImplementedError: + except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) - return + self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) + return - except Fail,e : + except Fail, e: - msg = e.args[0] + msg = e.args[0] - if msg == "offline": - self.active.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % self.active.name) - else: - self.active.setStatus("failed") - self.m.log.warning(_("Decrypting failed: %s | %s") % (self.active.name, msg)) - self.active.error = msg + if msg == "offline": + self.active.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % self.active.name) + else: + self.active.setStatus("failed") + self.m.log.warning(_("Decrypting failed: %s | %s") % (self.active.name, msg)) + self.active.error = msg - return + return - except Exception, e: + except Exception, e: - self.active.setStatus("failed") - self.m.log.error(_("Decrypting failed: %s | %s") % (self.active.name, str(e))) - self.active.error = str(e) + self.active.setStatus("failed") + self.m.log.error(_("Decrypting failed: %s | %s") % (self.active.name, str(e))) + self.active.error = str(e) - if self.m.core.debug: - print_exc() + if self.m.core.debug: + print_exc() - return + return - finally: - self.active.release() - self.active = False - self.m.core.files.save() - self.m.localThreads.remove(self) + finally: + self.active.release() + self.active = False + self.m.core.files.save() + self.m.localThreads.remove(self) - #self.m.core.hookManager.downloadFinished(pyfile) + #self.m.core.hookManager.downloadFinished(pyfile) - #self.m.localThreads.remove(self) - #self.active.finishIfDone() - pyfile.delete() + #self.m.localThreads.remove(self) + #self.active.finishIfDone() + pyfile.delete() ######################################################################## class HookThread(PluginThread): - """thread for hooks""" + """thread for hooks""" - #---------------------------------------------------------------------- - def __init__(self, m, function, pyfile): - """Constructor""" - PluginThread.__init__(self, m) + #---------------------------------------------------------------------- + def __init__(self, m, function, pyfile): + """Constructor""" + PluginThread.__init__(self, m) - self.f = function - self.active = pyfile + self.f = function + self.active = pyfile - m.localThreads.append(self) + m.localThreads.append(self) - pyfile.setStatus("processing") + pyfile.setStatus("processing") - self.start() + self.start() - def run(self): - self.f(self.active) + def run(self): + self.f(self.active) - self.m.localThreads.remove(self) - self.active.finishIfDone() + self.m.localThreads.remove(self) + self.active.finishIfDone() ######################################################################## class InfoThread(PluginThread): - #---------------------------------------------------------------------- - def __init__(self, manager, data, pid): - """Constructor""" - PluginThread.__init__(self, manager) - - self.data = data - self.pid = pid # package id - # [ .. (name, plugin) .. ] - self.start() - - #---------------------------------------------------------------------- - def run(self): - """run method""" - - plugins = {} - - for url, plugin in self.data: - if plugins.has_key(plugin): - plugins[plugin].append(url) - else: - plugins[plugin] = [url] + #---------------------------------------------------------------------- + def __init__(self, manager, data, pid): + """Constructor""" + PluginThread.__init__(self, manager) + + self.data = data + self.pid = pid # package id + # [ .. (name, plugin) .. ] + self.start() + + #---------------------------------------------------------------------- + def run(self): + """run method""" + + plugins = {} + + for url, plugin in self.data: + if plugins.has_key(plugin): + plugins[plugin].append(url) + else: + plugins[plugin] = [url] - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname) - if hasattr(plugin, "getInfo"): - self.m.core.log.debug("Run Info Fetching for %s" % pluginname) - for result in plugin.getInfo(urls): - if not type(result) == list: result = [result] - self.m.core.files.updateFileInfo(result, self.pid) + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname) + if hasattr(plugin, "getInfo"): + self.m.core.log.debug("Run Info Fetching for %s" % pluginname) + for result in plugin.getInfo(urls): + if not type(result) == list: result = [result] + self.m.core.files.updateFileInfo(result, self.pid) - self.m.core.log.debug("Finished Info Fetching for %s" % pluginname) + self.m.core.log.debug("Finished Info Fetching for %s" % pluginname) self.m.core.files.save()
\ No newline at end of file diff --git a/module/SpeedManager.py b/module/SpeedManager.py index c13f4b2dc..e69c641fc 100644 --- a/module/SpeedManager.py +++ b/module/SpeedManager.py @@ -20,7 +20,6 @@ """ from threading import Thread -import traceback from time import sleep, time class SpeedManager(Thread): diff --git a/module/ThreadManager.py b/module/ThreadManager.py index caa8524d4..70ac1c787 100644 --- a/module/ThreadManager.py +++ b/module/ThreadManager.py @@ -18,15 +18,15 @@ @author: RaNaN """ -from threading import Event -from subprocess import Popen from os.path import exists +import re +from subprocess import Popen +from threading import Event from time import sleep from traceback import print_exc -import re -from module.network.Request import getURL import PluginThread +from module.network.Request import getURL ######################################################################## class ThreadManager: @@ -48,7 +48,7 @@ class ThreadManager: self.reconnecting = Event() self.reconnecting.clear() - for i in range(0, self.core.config.get("general","max_downloads") ): + for i in range(0, self.core.config.get("general", "max_downloads")): self.createThread() @@ -78,7 +78,7 @@ class ThreadManager: #---------------------------------------------------------------------- def processingIds(self): """get a id list of all pyfiles processed""" - return [x.active.id for x in self.threads+self.localThreads if x.active and x.active != "quit"] + return [x.active.id for x in self.threads + self.localThreads if x.active and x.active != "quit"] #---------------------------------------------------------------------- @@ -93,7 +93,7 @@ class ThreadManager: def tryReconnect(self): """checks if reconnect needed""" - if not (self.core.server_methods.is_time_reconnect() and self.core.config["reconnect"]["activated"] ): + if not (self.core.server_methods.is_time_reconnect() and self.core.config["reconnect"]["activated"]): return False active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active] @@ -127,11 +127,11 @@ class ThreadManager: sleep(1) ip = "" while ip == "": - try: - ip = re.match(".*Current IP Address: (.*)</body>.*", getURL("http://checkip.dyndns.org/")).group(1) #get new ip - except: - ip = "" - sleep(1) + try: + ip = re.match(".*Current IP Address: (.*)</body>.*", getURL("http://checkip.dyndns.org/")).group(1) #get new ip + except: + ip = "" + sleep(1) self.core.hookManager.afterReconnecting(ip) self.log.info(_("Reconnected, new IP: %s") % ip) @@ -162,7 +162,7 @@ class ThreadManager: - occ = [x.active.pluginname for x in self.threads if x.active and not x.active.plugin.multiDL ] + occ = [x.active.pluginname for x in self.threads if x.active and not x.active.plugin.multiDL] occ.sort() occ = tuple(set(occ)) job = self.core.files.getJob(occ) diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index aa4aff171..610e27173 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -146,8 +146,10 @@ class Plugin(object): return self.config.getPlugin(self.__name__, option) - def setWait(self, seconds): + def setWait(self, seconds, reconnect=False): """ set the wait time to specified seconds """ + if reconnect: + self.wantReconnect = True self.pyfile.waitUntil = time() + int(seconds) def wait(self): diff --git a/module/plugins/accounts/NetloadIn.py b/module/plugins/accounts/NetloadIn.py index 95e77a9cb..e0f66e269 100644 --- a/module/plugins/accounts/NetloadIn.py +++ b/module/plugins/accounts/NetloadIn.py @@ -18,8 +18,6 @@ """ from module.plugins.Account import Account -import re -from time import strptime, mktime class NetloadIn(Account): __name__ = "NetloadIn" diff --git a/module/plugins/hooks/IRCInterface.py b/module/plugins/hooks/IRCInterface.py new file mode 100644 index 000000000..c271cfe89 --- /dev/null +++ b/module/plugins/hooks/IRCInterface.py @@ -0,0 +1,346 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN + @interface-version: 0.2 +""" + +from select import select +import socket +import sys +from threading import Thread +import time +from time import sleep +from traceback import print_exc + +from module.plugins.Hook import Hook + +class IRCInterface(Thread, Hook): + __name__ = "IRCInterface" + __version__ = "0.1" + __description__ = """connect to irc and let owner perform different tasks""" + __config__ = [("activated", "bool", "Activated", "False"), + ("host", "str", "IRC-Server Address", ""), + ("port", "int", "IRC-Server Port", "6667"), + ("ident", "str", "Clients ident", "pyload-irc"), + ("realname", "str", "Realname", "pyload-irc"), + ("nick", "str", "Nickname the Client will take", "pyLoad-IRC"), + ("owner", "str", "Nickname the Client will accept commands from", "Enter your nick here")] + __author_name__ = ("Jeix") + __author_mail__ = ("Jeix@hasnomail.com") + + def __init__(self, core): + Thread.__init__(self) + Hook.__init__(self, core) + self.setDaemon(True) + + def coreReady(self): + self.new_package = {} + + self.abort = False + + self.host = self.getConfig("host") + ":" + str(self.getConfig("port")) + self.owner = self.getConfig("owner") + self.nick = self.getConfig("nick") + + #self.targets = irc_targets # where replies will go to + #if len(self.targets) < 1: + + self.targets = [self.getConfig("owner")] + + self.links_added = 0 + + self.start() + + def run(self): + # connect to IRC etc. + self.sock = socket.socket() + temp = self.host.split(":", 1) + self.sock.connect((temp[0], int(temp[1]))) + self.sock.send("NICK %s\r\n" % self.nick) + self.sock.send("USER %s %s bla :%s\r\n" % (self.nick, self.host, self.nick)) + for t in self.targets: + if t.startswith("#"): + self.sock.send("JOIN %s\r\n" % t) + self.log.info("pyLoadIRC: Connected to %s!" % self.host) + self.log.info("pyLoadIRC: Switching to listening mode!") + try: + self.main_loop() + + except IRCError, ex: + self.sock.send("QUIT :byebye\r\n") + print_exc() + self.sock.close() + sys.exit(1) + + + def main_loop(self): + readbuffer = "" + while True: + sleep(1) + fdset = select([self.sock], [], [], 0) + if self.sock not in fdset[0]: + continue + + if self.abort: + raise IRCError("quit") + + readbuffer += self.sock.recv(1024) + temp = readbuffer.split("\n") + readbuffer = temp.pop() + + for line in temp: + line = line.rstrip() + first = line.split() + + if(first[0] == "PING"): + self.sock.send("PING %s\r\n" % first[1]) + + if first[0] == "ERROR": + raise IRCError(line) + + msg = line.split(None, 3) + if len(msg) < 4: + continue + + msg = { + "origin":msg[0][1:], + "action":msg[1], + "target":msg[2], + "text":msg[3][1:] + } + + self.handle_events(msg) + + + def handle_events(self, msg): + if msg["origin"].split("!", 1)[0] != self.owner: + return + + if msg["target"].split("!", 1)[0] != self.nick: + return + + if msg["action"] != "PRIVMSG": + return + + trigger = "pass" + args = None + + temp = msg["text"].split() + trigger = temp[0] + if len(temp) > 1: + args = temp[1:] + + handler = getattr(self, "event_%s" % trigger, self.event_pass) + try: + handler(args) + except Exception, e: + self.log(repr(e)) + + + def log(self, msg): + print _(msg) + for t in self.targets: + self.sock.send("PRIVMSG %s :%s\r\n" % (t, msg)) + + +#### Events + def event_pass(self, args): + pass + + def event_status(self, args): + downloads = self.core.status_downloads() + if len(downloads) < 1: + self.log("INFO: There are no active downloads currently.") + return + + self.log("ID - Name - Status - Speed - ETA - Progress") + for data in downloads: + self.log("#%d - %s - %s - %s - %s - %s" % + ( + data['id'], + data['name'], + data['status'], + "%d kb/s" % int(data['speed']), + "%d min" % int(data['eta'] / 60), + "%d/%d MB (%d%%)" % ((data['size']-data['kbleft']) / 1024, data['size'] / 1024, data['percent']) + ) + ) + + def event_queue(self, args): + # just forward for now + self.event_status(args) + + def event_collector(self, args): + ps = self.core.get_collector_packages() + if len(ps) == 0: + self.log("INFO: No packages in collector!") + return + + for packdata in ps: + self.log('PACKAGE: Package "%s" with id #%d' % (packdata['package_name'], packdata['id'])) + for fileid in self.core.get_package_files(packdata['id']): + fileinfo = self.core.get_file_info(fileid) + self.log('#%d FILE: %s (#%d)' % (packdata['id'], fileinfo["filename"], fileinfo["id"])) + + def event_links(self, args): + fids = self.core.get_files() + if len(fids) == 0: + self.log("INFO: No links.") + return + + for fid in fids: + info = self.core.get_file_info(fid) + self.log('LINK #%d: %s [%s]' % (fid, info["filename"], info["status_type"])) + + def event_packages(self, args): + pids = self.core.get_packages() + if len(pids) == 0: + self.log("INFO: No packages.") + return + + for pid in pids: + data = self.core.get_package_data(pid) + self.log('PACKAGE #%d: %s (%d links)' % (pid, data["package_name"], len(self.core.get_package_files(pid)))) + + def event_info(self, args): + if not args: + self.log('ERROR: Use info like this: info <id>') + return + + info = self.core.get_file_info(int(args[0])) + self.log('LINK #%d: %s (%d) [%s bytes]' % (info['id'], info['filename'], info['size'], info['status_type'])) + + def event_packinfo(self, args): + if not args: + self.log('ERROR: Use packinfo like this: packinfo <id>') + return + + packdata = self.core.get_package_data(int(args[0])) + self.log('PACKAGE: Package "%s" with id #%d' % (packdata['package_name'], packdata['id'])) + for fileid in self.core.get_package_files(packdata['id']): + fileinfo = self.core.get_file_info(fileid) + self.log('#%d LINK: %s (#%d)' % (packdata['id'], fileinfo["filename"], fileinfo["id"])) + + def event_start(self, args): + if not args: + count = 0 + for packdata in self.core.get_collector_packages(): + self.core.push_package_2_queue(packdata['id']) + count += 1 + + self.log("INFO: %d downloads started." % count) + return + + for val in args: + id = int(val.strip()) + self.core.push_package_2_queue(id) + self.log("INFO: Starting download #%d" % id) + + def event_stop(self, args): + if not args: + self.core.stop_downloads() + self.log("INFO: All downloads stopped.") + return + + for val in args: + id = int(val.strip()) + self.core.stop_download("", id) + self.log("INFO: Download #%d stopped." % id) + + def event_add(self, args): + if len(args) != 2: + self.log('ERROR: Add links like this: "add <package|id> <link>". '\ + 'This will add the link <link> to to the package <package> / the package with id <id>!') + return + + def get_pack_id(pack): + if pack.isdigit(): + pack = int(pack) + for packdata in self.core.get_collector_packages(): + if packdata['id'] == pack: + return pack + return -1 + + for packdata in self.core.get_collector_packages(): + if packdata['package_name'] == pack: + return packdata['id'] + return -1 + + + pack = args[0].strip() + link = args[1].strip() + count_added = 0 + count_failed = 0 + + # verify that we have a valid link + if not self.core.is_valid_link(link): + self.log("ERROR: Your specified link is not supported by pyLoad.") + return + + # get a valid package id (create new package if it doesn't exist) + pack_id = get_pack_id(pack) + if pack_id == -1: + pack_id = self.core.new_package(pack) + + # move link into package + fid = self.core.add_links_to_package(pack_id, [link]) + self.log("INFO: Added %s to Package %s [#%d]" % (link, pack, pack_id)) + + def event_del(self, args): + if len(args) < 2: + self.log("ERROR: Use del command like this: del -p|-l <id> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)") + return + + if args[0] == "-p": + ret = self.core.del_packages(map(int, args[1:])) + self.log("INFO: Deleted %d packages!" % ret) + + elif args[0] == "-l": + ret = self.core.del_links(map(int, args[1:])) + self.log("INFO: Deleted %d links!" % ret) + + else: + self.log("ERROR: Use del command like this: del <-p|-l> <id> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)") + return + + def event_help(self, args): + self.log("The following commands are available:") + self.log("add <package|packid> <link> Adds link to package. (creates new package if it does not exist)") + time.sleep(1) + self.log("collector Shows all packages in collector") + self.log("del -p|-l <id> [...] Deletes all packages|links with the ids specified") + time.sleep(1) + self.log("info <id> Shows info of the link with id <id>") + self.log("help Shows this help file") + time.sleep(1) + self.log("links Shows all links in pyload") + self.log("packages Shows all packages in pyload") + time.sleep(1) + self.log("packinfo <id> Shows info of the package with id <id>") + self.log("queue Shows info about the queue") + time.sleep(1) + self.log("start [<id>...] Starts the package with id <id> or all packages if no id is given") + self.log("status Show general download status") + time.sleep(1) + self.log("stop [<id>...] Stops the package with id <id> or all packages if no id is given") + + +class IRCError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value)
\ No newline at end of file diff --git a/module/plugins/hoster/MegauploadCom.py b/module/plugins/hoster/MegauploadCom.py index a1eaf6cc5..a14c2c76f 100644 --- a/module/plugins/hoster/MegauploadCom.py +++ b/module/plugins/hoster/MegauploadCom.py @@ -1,10 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import os import re -import tempfile -from time import time from module.plugins.Hoster import Hoster diff --git a/module/web/templates/default/settings.html b/module/web/templates/default/settings.html index d7c7f78a4..208549813 100644 --- a/module/web/templates/default/settings.html +++ b/module/web/templates/default/settings.html @@ -66,11 +66,6 @@ {% endblock %} {% block content %} - -{% for message in errors %} -<b>{{message}}</b><br> -{% endfor %} - <ul id="toptabs" class="tabs"> {% for configname, config in conf.iteritems %} @@ -128,6 +123,11 @@ <input class="submit" type="submit" value="{% trans "Submit" %}" /> </form> +<br> +{% for message in errors %} +<b>{{message}}</b><br> +{% endfor %} + {% endif %} {% endblock %} diff --git a/pyLoadCli.py b/pyLoadCli.py index 5105599cd..36ab0c4f7 100755 --- a/pyLoadCli.py +++ b/pyLoadCli.py @@ -20,6 +20,7 @@ from getopt import GetoptError from getopt import getopt import gettext +from itertools import islice import os import os.path from os.path import join @@ -29,10 +30,8 @@ import threading import time from time import sleep import xmlrpclib -from itertools import islice from module import InitHomeDir - from module.ConfigParser import ConfigParser if sys.stdout.encoding.lower().startswith("utf"): @@ -220,7 +219,7 @@ class pyLoadCli: self.println(line, _("Type d(number of package) to delete a package, r to restart, or w/o d,r to look into it.")) line += 1 i = 0 - for id, value in islice(pack.iteritems(), self.pos[2], self.pos[2] + 5 ): + for id, value in islice(pack.iteritems(), self.pos[2], self.pos[2] + 5): try: self.println(line, mag(conv(id)) + ": " + value['name']) line += 1 diff --git a/pyLoadCore.py b/pyLoadCore.py index 644ccf925..1db514d5d 100755 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -29,10 +29,9 @@ from imp import find_module import logging import logging.handlers from os import _exit -from os import getcwd from os import execv +from os import getcwd from os import makedirs -from os import name as platform from os import remove from os import sep from os.path import exists @@ -44,28 +43,25 @@ from sys import argv from sys import executable from sys import exit from sys import stdout -from tempfile import NamedTemporaryFile +from sys import platform import thread import time from time import sleep -from xmlrpclib import Binary from traceback import print_exc - +from xmlrpclib import Binary from module import InitHomeDir - -from module.ConfigParser import ConfigParser -import module.remote.SecureXMLRPCServer as Server -from module.web.ServerThread import WebServer - -from module.ThreadManager import ThreadManager +from module.AccountManager import AccountManager from module.CaptchaManager import CaptchaManager +from module.ConfigParser import ConfigParser +from module.FileDatabase import FileHandler from module.HookManager import HookManager -from module.PullEvents import PullManager from module.PluginManager import PluginManager -from module.FileDatabase import FileHandler +from module.PullEvents import PullManager from module.RequestFactory import RequestFactory -from module.AccountManager import AccountManager +from module.ThreadManager import ThreadManager +import module.remote.SecureXMLRPCServer as Server +from module.web.ServerThread import WebServer class Core(object): """ pyLoad Core """ @@ -177,7 +173,7 @@ class Core(object): self.do_kill = False self.do_restart = False - self.log.info(_("Using home directory: %s") % getcwd() ) + self.log.info(_("Using home directory: %s") % getcwd()) #@TODO refractor @@ -261,8 +257,7 @@ class Core(object): except Exception, e: self.log.error(_("Failed starting XMLRPC server CLI and GUI will not be available: %s") % str(e)) if self.debug: - import traceback - traceback.print_exc() + print_exc() def init_webserver(self): if self.config['webinterface']['activated']: @@ -292,7 +287,7 @@ class Core(object): pipe = subprocess.PIPE subprocess.Popen(check_name, stdout=pipe, stderr=pipe) except: - self.log.info( _("Install %s") % legend ) + self.log.info(_("Install %s") % legend) if essential: exit() def check_file(self, check_names, description="", folder=False, empty=True, essential=False, quiet=False): @@ -324,7 +319,7 @@ class Core(object): pass else: if not empty: - self.log.warning( _("could not find %s: %s") % (description, tmp_name) ) + self.log.warning(_("could not find %s: %s") % (description, tmp_name)) else: print _("could not create %s: %s") % (description, tmp_name) if essential: @@ -363,18 +358,18 @@ class Core(object): for pyfile in pyfiles: pyfile.abortDownload() - - -# self.requestFactory.clean() + except: if self.debug: print_exc() self.log.info(_("error while shutting down")) + finally: self.files.syncSave() - def path(self, *args): - return join(pypath, *args) + + def path(self, * args): + return join(pypath, * args) def freeSpace(self): folder = self.config['general']['download_folder'] @@ -389,7 +384,7 @@ class Core(object): return s.f_bsize * s.f_bavail / 1024 / 1024 #megabyte - #################################### +#################################### ########## XMLRPC Methods ########## #################################### @@ -426,7 +421,7 @@ class ServerMethods(): if sec == "core": return self.core.config[cat][var] elif sec == "plugin": - return self.core.config.getPlugin(cat,var) + return self.core.config.getPlugin(cat, var) def set_conf_val(self, cat, opt, val, sec="core"): """ set config value """ @@ -482,7 +477,7 @@ class ServerMethods(): self.core.files.addLinks(links, pid) - self.core.log.info(_("Added package %s containing %s links") % (name, len(links) ) ) + self.core.log.info(_("Added package %s containing %s links") % (name, len(links))) self.core.files.save() @@ -537,7 +532,7 @@ class ServerMethods(): self.core.files.restartFile(int(fileid)) def upload_container(self, filename, content): - th = open(join(self.core.config["general"]["download_folder"] , "tmp_" + filename), "wb") + th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb") th.write(str(content)) th.close() @@ -551,7 +546,7 @@ class ServerMethods(): fh.close() return lines except: - return ('No log available',) + return ('No log available', ) def stop_downloads(self): pyfiles = self.core.files.cache.values() |